{"as_of":"2026-08-13T02:45:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:5b594018e139cbca39518febfa64759a34040fadc51d92a115010a8ea65cbd04","coverage":[{"denominator":80,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":80,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T11:30:27.995986Z","state":"measured"},{"denominator":80,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":80,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.18151/citation-record","integrity":"/paper/2411.18151/integrity","json":"/paper/2411.18151/citation-record.json","paper":"/paper/2411.18151"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.580872Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.580872Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:aa1116c15e043a43d9996e071a3709cc91b4dfd27d50fd350536a4194a84cb4c","observation_id":"e2887da4-0e8c-4225-873a-7ab27235200e","resolution":{"observed_at":"2026-08-12T11:30:27.580872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3377814","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.137187Z","title":"Ahmed, Nisheeth Srivastava, Renuka Sindhgatta, and Amey Karkare","venue":null,"work_id":"1ddc47a7-1bdb-4fff-ba98-1271373d77a8","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.586029Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:beb8e0b2da7dd00fc1d0a36cc71f9489c019b64d91c429c033807d4084a01622","observation_id":"e033df61-614f-4bae-8ae4-26bf9e84f9c7","resolution":{"observed_at":"2026-08-12T11:30:28.142580Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.21061/ctete-rms.v1.c.1","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.119624Z","title":null,"venue":null,"work_id":"5560b2bc-605f-488b-8f04-8771d656d6f4","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.591276Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:1eb0dcb16e2c5d8cb79558f1eecf3d0a8a9ce89229276a4355c12f4cbe8f8df6","observation_id":"b075a8eb-7cdf-42bc-8d20-14a63aa85539","resolution":{"observed_at":"2026-08-12T11:30:28.125581Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.595937Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.595937Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:7c8266637e8c73beedd11c42cae28e52560e87d1ab1cb003bfa48a8aa0d2230d","observation_id":"5a58db14-67d7-4581-b011-810456b066c9","resolution":{"observed_at":"2026-08-12T11:30:27.595937Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.600605Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.600605Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:958ba7ab2f8ce1ae606354d4e226ce72c7f80c423366fe3769e548ca3c040c97","observation_id":"fdc7d5e0-257f-49db-ad7c-66497dfaf47e","resolution":{"observed_at":"2026-08-12T11:30:27.600605Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0969.36111","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.328268Z","title":null,"venue":null,"work_id":"5aff724e-e2dc-4337-acf8-bfa677f2666a","year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.605855Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:0c729b89bdf6a4bb7a98ea63dad2e2d18bd720094d847c24d6ad56eff6e55c94","observation_id":"b8558598-1afb-4892-b264-3157c3d48771","resolution":{"observed_at":"2026-08-12T11:30:32.335459Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.610863Z","title":null,"venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.610863Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2a9aeacd2e41886ad29bfdd48a101b2691fbcb5cd8aac6f54a44b011dabafb75","observation_id":"0f300b67-252e-40b7-bb81-2847ecc1b5ad","resolution":{"observed_at":"2026-08-12T11:30:27.610863Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.615271Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.615271Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:72abdcb8f15c6ddcac764b54147b99864fd71b25a91678c03055234e7e27253f","observation_id":"350c4baa-de3a-4bcb-9fa5-bd153a69af6d","resolution":{"observed_at":"2026-08-12T11:30:27.615271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.619568Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.619568Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:9ed3ee28d11a4ada88a70ecf1df3ee0d69545f9f173567c8320bed5ffbce1e5d","observation_id":"ae96dc2c-639e-4e17-8a58-d517c3b1a528","resolution":{"observed_at":"2026-08-12T11:30:27.619568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.623788Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.623788Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:3a3e45e15bbf5eb29d192758b366e6a54afddddb02ef8de0fd5c7dc4428f4cd3","observation_id":"63b06e05-c23c-4c36-85b6-cef3bb8cb188","resolution":{"observed_at":"2026-08-12T11:30:27.623788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.641488Z","title":"Becker, James Finnie-Ansley, Arto Hellas, Juho Leinonen, Andrew Luxton-Reilly, Brent N","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.641488Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ed9cbcdb352788e7ad243da9f9b41cb643a9bacd4a71894396ba2c481cb6878c","observation_id":"ebf653e9-3bd1-4ff0-a598-12576604280c","resolution":{"observed_at":"2026-08-12T11:30:27.641488Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.732430Z","title":"Becker, Catherine Mooney, John Homer, Zachary C Albrecht, and Garrett B","venue":null,"work_id":"9dd434f7-945c-47c6-abee-68a1fa400b5c","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.646147Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:beb97360fa51df5b324d26af9260b9d73f09453de937655131f88849913b6f01","observation_id":"55428469-3402-4099-bd03-d7af87c2aafb","resolution":{"observed_at":"2026-08-12T11:30:32.737238Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"1276.23612","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.677141Z","title":"Falkner and Katrina E","venue":null,"work_id":"ab2d5b62-60cd-40c0-ab7c-7e7c954e5529","year":2012},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.656180Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b76802aeb1b52e9acd5845533236a8e1b4c17c28269d456235c18e30cbba338c","observation_id":"3259f061-7343-4314-96dc-496a11cf7193","resolution":{"observed_at":"2026-08-12T11:30:31.684371Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.660861Z","title":"Fiannaca, Chinmay Kulkarni, Carrie J Cai, and Michael Terry","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.660861Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f9d0ce2b1c9cf20a5807472215bbd8b81303270cf4231ed9969c9421557bab4a","observation_id":"128bf2ba-094d-4fdc-b5d6-edb88a95ae69","resolution":{"observed_at":"2026-08-12T11:30:27.660861Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.717292Z","title":null,"venue":null,"work_id":"9179dfae-5daf-4b7c-9e40-cf9884a4e261","year":2005},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.666187Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:28ec2c58d4b67bf0d440b489aff6adb312f7a0082ea894e5f90666cbb7e40248","observation_id":"502daedb-2b28-4276-b7c3-b7cf731e3e37","resolution":{"observed_at":"2026-08-12T11:30:32.722233Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.701808Z","title":null,"venue":null,"work_id":"f80cd2e4-7192-4047-89af-0919695df2fa","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.670854Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:deb22c51a4e3d48318e6e43b3a351c3d8c67f26611bcd6066acb5831c77e1e3f","observation_id":"c70e0d49-eeaa-4d85-a027-0c47fe048d28","resolution":{"observed_at":"2026-08-12T11:30:32.707092Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.675508Z","title":"Glassman, Aaron Lin, Carrie J","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.675508Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5178d86a1212a6c3ea70bd4c2b711fa86b19d145408d7d153d989cd253901325","observation_id":"caa45737-8407-4d33-97f8-9dccdafb5fb8","resolution":{"observed_at":"2026-08-12T11:30:27.675508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"6527.34067","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.441741Z","title":"Guo, Julia M","venue":null,"work_id":"f0e08905-db5d-4f7b-af01-5dca52a2877e","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.680369Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:09d60fc36bdbcc06e2d383ece6fe2fa87883d8fd7477fb8275899cb6c54c7f4e","observation_id":"a42b8d7d-b2aa-4466-bbee-f5a52910584a","resolution":{"observed_at":"2026-08-12T11:30:31.450088Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2017.81204","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.336861Z","title":null,"venue":null,"work_id":"8ac92cdd-5061-46d9-b147-a73d289de4cb","year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.684804Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:82fceba51fd228f1b21aefaa48303cfb1eac9c3307261d6f775beb717e27b31a","observation_id":"e01261a7-c138-4f4f-839e-904df9e79271","resolution":{"observed_at":"2026-08-12T11:30:31.346742Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"7091.32058","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.221222Z","title":"Ajanovski, Mirela Gutica, Timo Hynninen, Antti Knutas, Juho Leinonen, Chris Messom, and Soohyun Nam Liao","venue":null,"work_id":"47f19c26-b8df-49fd-8bd1-eb94403bfeb2","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.689558Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:dcd28b2c37a29792c4d8465273ec2c10db26b9b1fa91b5e501034765c909c95e","observation_id":"b57ce87c-0a21-4a59-8811-adcab921a9da","resolution":{"observed_at":"2026-08-12T11:30:31.231311Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.686460Z","title":"2024.Predicting Results of Social Science Experiments Using Large Language Models","venue":null,"work_id":"4593df07-71d3-4cf7-bd47-78e6027b5408","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.699118Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:433c6410a998922682ba7c7cc13e2edf3ef55381e6ff2b2e179de5b3487d2b27","observation_id":"3cf5202d-8683-4f2a-89c0-d20b34387f2b","resolution":{"observed_at":"2026-08-12T11:30:32.691282Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.671714Z","title":null,"venue":null,"work_id":"e53e07ad-fdb1-4cb1-a609-9c9f490216a5","year":1995},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.703538Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:442c6fd63443c73a09df3e3a658f863dcdff7cff33ce42945a617f1a924207c5","observation_id":"78d8315f-0ea7-4c15-b9cd-2a016d47304d","resolution":{"observed_at":"2026-08-12T11:30:32.676204Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.707850Z","title":"Hutto and Eric Gilbert","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.707850Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:59729d4b57b997c7ec208767f136ea35f64972d7c0c17687e05b9b0eb20dd5e3","observation_id":"cdafeb7c-8f4a-496c-aee6-8cb4be7beb70","resolution":{"observed_at":"2026-08-12T11:30:27.707850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.712309Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.712309Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:201228c298014a3b29f83aacb43d4868f4970be88c7e5835388efba85d45b6e2","observation_id":"ab92ac17-8e2c-455e-b2c4-6a6d273ead20","resolution":{"observed_at":"2026-08-12T11:30:27.712309Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.717800Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.717800Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8996a556384b8a8dfc74198550d34cb6cdba96a2337d94aca982f13ac450bc4b","observation_id":"54d4dcaa-9036-4e64-b034-0b4559baec24","resolution":{"observed_at":"2026-08-12T11:30:27.717800Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2023.22422","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:30.841156Z","title":null,"venue":null,"work_id":"4ee87397-795f-4721-b3ba-556298ae8bfa","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.722750Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ac28d69ac97f796bf5cae925cdad09cff17516926bea12046269b1fb1441f286","observation_id":"1cf38f08-12a7-4418-a77d-f6a4a0447241","resolution":{"observed_at":"2026-08-12T11:30:30.849444Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10694","last_updated":"2023-09-30T07:22:41Z","snapshot_observed_at":"2026-08-09T11:58:54.049598Z","submitted_at":"2023-09-19T15:29:12Z","title":"\"With Great Power Comes Great Responsibility!\": Student and Instructor Perspectives on the influence of LLMs on Undergraduate Engineering Education","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10694","snapshot_observed_at":"2026-08-12T11:30:27.727526Z","title":"With Great Power Comes Great Respon- sibility!","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.727526Z"},"links":{"cited_paper":"/paper/2309.10694","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:79b91ca09b947b51d7a777dbc3958fed83e136156a689d6ce05a3c31122c7d90","observation_id":"e8756c65-a449-4b04-a586-75a1193ad08e","resolution":{"observed_at":"2026-08-12T11:30:27.727526Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.732282Z","title":"Ericson, David Weintrop, and Tovi Grossman","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.732282Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5c04015097c619ae73a1e22e9b071c03261bf1d8c3bbe25866727b64369b6e9b","observation_id":"dc9ab964-d753-4f28-a6ce-6f00f669f372","resolution":{"observed_at":"2026-08-12T11:30:27.732282Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.736826Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.736826Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:98d9e526cdb597bedb468006985075470a8e337fcea24210b2fa92d123524163","observation_id":"260a696e-7097-4549-b6e3-1c85fcc01c95","resolution":{"observed_at":"2026-08-12T11:30:27.736826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.741550Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.741550Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:1a80e17e23f92d3bce7b748a100a80636af3bed0aec1a8b1afab845bc219bc5a","observation_id":"283b7718-1bb0-494a-b123-b1d5e8ad5702","resolution":{"observed_at":"2026-08-12T11:30:27.741550Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.746218Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.746218Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:0318523a585332a18a7904d1282448c6e9111ef8f4709a451492ed4292187529","observation_id":"258e90de-da70-4024-8282-cd671bf3f74a","resolution":{"observed_at":"2026-08-12T11:30:27.746218Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08572","last_updated":"2023-08-15T19:48:31Z","snapshot_observed_at":"2026-07-06T16:06:59.216785Z","submitted_at":"2023-08-15T19:48:31Z","title":"Large Language Models in Introductory Programming Education: ChatGPT's Performance and Implications for Assessments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08572","snapshot_observed_at":"2026-08-12T11:30:27.750926Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.750926Z"},"links":{"cited_paper":"/paper/2308.08572","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c7dcafdcd16f1f012d427cd27a4cfa8ef1e3463a2dbf18aeea00847b2b377212","observation_id":"7e4ceb67-f775-4ca6-902f-c7eec62d2c95","resolution":{"observed_at":"2026-08-12T11:30:27.750926Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.756198Z","title":null,"venue":null,"work_id":null,"year":1975},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.756198Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:1f7935a2a55f820bd5e9f8610ef3e2d040ddf718795a430aa11dbaff010fd437","observation_id":"8efe8acc-1dfc-4ae9-8a68-458dec23b7da","resolution":{"observed_at":"2026-08-12T11:30:27.756198Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.645655Z","title":null,"venue":null,"work_id":"91fdef8a-a8fd-4cfc-a1cf-74062fb8f14e","year":2016},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.761650Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:09b4927df7ff315c09102a6775e6dcf967f70b2b152f125bd2587338a99b16f1","observation_id":"e3ab300c-637e-4b94-954d-358570a3bfb3","resolution":{"observed_at":"2026-08-12T11:30:32.651783Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04873","last_updated":"2024-11-22T01:13:13Z","snapshot_observed_at":"2026-08-12T23:27:51.864697Z","submitted_at":"2024-07-05T21:44:11Z","title":"Evaluating Language Models for Generating and Judging Programming Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04873","snapshot_observed_at":"2026-08-12T11:30:27.766081Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.766081Z"},"links":{"cited_paper":"/paper/2407.04873","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c2e42e85f99091b17ff5752c890ac72ca4d39373deecaf457349cdba5af498be","observation_id":"3e3600b4-774a-474f-a049-d7c17480b0ba","resolution":{"observed_at":"2026-08-12T11:30:27.766081Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.776293Z","title":"Ban It Till We Understand It","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.776293Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:d27d662350d43de1e7ed1c7aa974a5da383b7e440adf27bec2f02d189a7626ad","observation_id":"7767876f-8d09-4983-b4b4-8962a11064ab","resolution":{"observed_at":"2026-08-12T11:30:27.776293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.631572Z","title":null,"venue":null,"work_id":"caec3336-dff5-43c7-a0d1-98313031496a","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.780949Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:7f4892c6a32570b6f4ac6ac61b5c8ef1bae0c2efb5bbe4789a6e5dad78b0793b","observation_id":"002a46f1-21fa-498d-86cb-dc9b82ac1149","resolution":{"observed_at":"2026-08-12T11:30:32.636005Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.794774Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.794774Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:73187c654ec4227e897ec2350876619d0425649b62be0bd00e2284190a27735a","observation_id":"97be0ac3-4651-4154-81da-87984be30248","resolution":{"observed_at":"2026-08-12T11:30:27.794774Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.617192Z","title":null,"venue":null,"work_id":"15ddda4d-808f-4b71-b419-a40630f5bace","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.799392Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f6b8d216710ba575eae2ddefb6ea6081aab9cbf20298b0cc0318da0d7dfe1568","observation_id":"39c78a17-9e8a-4fb0-8e8e-2ee2c7dcc839","resolution":{"observed_at":"2026-08-12T11:30:32.621703Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.804217Z","title":"Let Them Try to Figure It Out First","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.804217Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c16774d578496f741ab6179a81152dacfdeea851c12c8103944605e21765da81","observation_id":"2aeb9b1e-3827-4788-be03-cf27effe1a62","resolution":{"observed_at":"2026-08-12T11:30:27.804217Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.601543Z","title":null,"venue":null,"work_id":"5abc755b-0c9d-44bb-bf72-b53613fe46fa","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.808450Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:997a39055dfbf5af10f3498c401fea86aab4c8b4a2cb300ce05b540877a988e7","observation_id":"845fbdc3-ecda-48b5-98fd-ad8163b6b065","resolution":{"observed_at":"2026-08-12T11:30:32.606679Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0969.36109","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:30.130170Z","title":null,"venue":null,"work_id":"d143488a-07cb-4f1f-9650-7d6fa7955d61","year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.818467Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f3fd09f039d3b9fcf7db89c3420942e753a540c2427214dbda893d566c1d273a","observation_id":"d6c88357-5e80-47fd-a947-3510e34f444d","resolution":{"observed_at":"2026-08-12T11:30:30.138032Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.585837Z","title":"Ok Pal, We Have to Code That Now","venue":null,"work_id":"0c074c91-aeda-4cdc-9a85-1b588fb4c408","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.824069Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:16d077f0ca70358b9af99d421cd1723c3419131df4cf3aa5d49fc71e7a411928","observation_id":"60ca70de-188f-4fa2-a7de-b4e04d89ef65","resolution":{"observed_at":"2026-08-12T11:30:32.590967Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.813325Z","title":"In Proceedings of the 54th ACM Technical Symposium on Computer Science Education V","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.813325Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:e94c5294fb6b4ab65349a3c9620433caead6c6f224d1f867a3255faedb27e9a6","observation_id":"8573eb57-3a88-44c3-ba0e-f5cf12236af9","resolution":{"observed_at":"2026-08-12T11:30:27.813325Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.835342Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.835342Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:db336c6f340970c3820b7c82a0f1d349256c3a86336de92275f28e25fe1b4a2f","observation_id":"5d120c56-646c-4bf8-8e11-4c44f3475c3f","resolution":{"observed_at":"2026-08-12T11:30:27.835342Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.840475Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.840475Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:3197bde4773c275aeb9c5dbfaf421ef88022c5b5e99834f9334d8ab19e630613","observation_id":"c85326dc-50bb-4107-8d9e-6dc9fe4782c7","resolution":{"observed_at":"2026-08-12T11:30:27.840475Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.829681Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.829681Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:056ffa080dd7f3560332ae26fd57ea5cccbb6077f1e01c5cc5062c37a309257e","observation_id":"3de0fd73-dd4a-4f56-a354-6b9e254348ac","resolution":{"observed_at":"2026-08-12T11:30:27.829681Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.570405Z","title":null,"venue":null,"work_id":"f3b206a7-bd38-4d43-8f31-015df5a270d9","year":2015},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.850832Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a1fc95fe5365b5545d115ef542a5bf3e83e172d102e3324f81b6b3320fc55fbd","observation_id":"9d4c9aa0-d5b2-4dae-8017-f10fd8a97eed","resolution":{"observed_at":"2026-08-12T11:30:32.575663Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.855457Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.855457Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:0794fa3243ee8f7049486bfdbf792d173e04b04eec63ffe539d866d6ad31d160","observation_id":"b55b55f6-1809-4a24-987c-0b8b20bd7dec","resolution":{"observed_at":"2026-08-12T11:30:27.855457Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.845664Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.845664Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:d9c5310b106bdc9fb2c7072ce4b3eeb153b69a14b0d23dec35fad117805f3681","observation_id":"0badc109-39ff-4ace-8cfb-2f2de304ea8d","resolution":{"observed_at":"2026-08-12T11:30:27.845664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.864974Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.864974Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b8f66d8930c88499a7e60ae9d441fd83f03f8cec593336d95d0d7ab0226e6525","observation_id":"fad38ca3-fc09-4220-af08-a45df4a1ba42","resolution":{"observed_at":"2026-08-12T11:30:27.864974Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.869591Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.869591Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b635de9af707ecc4c0bc60ceea2110a1ae41b2ae9c4008263fc49e8b47ac059b","observation_id":"ba92dd5f-86af-4eae-a5dc-2be776d8b849","resolution":{"observed_at":"2026-08-12T11:30:27.869591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.860474Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.860474Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:803c9cdd63cda5ab6ec990a547e320c4cada934898ccc518d17b1791bdef5a89","observation_id":"8aca14a4-5e93-45ba-a183-18479f1f07ef","resolution":{"observed_at":"2026-08-12T11:30:27.860474Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.878783Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.878783Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:25936d101081c9b6f8810eebc26f84aa897958d1b1a2758f45046186f4a05b84","observation_id":"9b5f3ed2-99e5-46e9-822c-6184844171d9","resolution":{"observed_at":"2026-08-12T11:30:27.878783Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2012.66535","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:29.446352Z","title":null,"venue":null,"work_id":"deed239f-ed75-48cc-aaf8-85c1580bb154","year":2012},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.883859Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a0dcea6dd078a9150efdd088e80e14f07b43212a041fa49394330b7f4981c781","observation_id":"8a11d673-7fa8-47b8-966d-be0274a97459","resolution":{"observed_at":"2026-08-12T11:30:29.453686Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.874527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.874527Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:55b4f4db08afde2efe98d2c639070962e515fd5bbac0fe11ede8d00aa1fc6c8f","observation_id":"29d41e53-a56d-45ec-a108-7928ef136fc8","resolution":{"observed_at":"2026-08-12T11:30:27.874527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.554027Z","title":"Becker, Robert Nix, Brent N","venue":null,"work_id":"3c5c486c-a034-4d43-9577-916cef46ee63","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.893325Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a49744c9b600f9e594f928b36d7ee440daf1a1fac21d2f1e98753ce127507cc2","observation_id":"5e096b9a-ec56-4610-a6dc-ae30737fb2dc","resolution":{"observed_at":"2026-08-12T11:30:32.559852Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0977.32309","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:29.287469Z","title":null,"venue":null,"work_id":"2cc06ab7-bc84-47db-92d1-b0700a9a11f0","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.903223Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:d1766dba6cc6acdba9c73627b67cfef1a211c348c9e0460a8f21597fe66418c6","observation_id":"520347c2-8c49-41ec-8ea9-2024fa994ae2","resolution":{"observed_at":"2026-08-12T11:30:29.294844Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.888690Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.888690Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2e1ed0e0eef65aba55838f70c69ff0523bc6962b43c878c9deffe2b754c2ab52","observation_id":"6b3afdcd-63a0-4597-b997-233068345fbf","resolution":{"observed_at":"2026-08-12T11:30:27.888690Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.913197Z","title":"It’s Weird That it Knows What I Want","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.913197Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a40d91a77a7152071ec047b78291d4a67da8081b0007844873d605a365d6278c","observation_id":"642e95e8-e8c7-4005-9495-2d948719d26b","resolution":{"observed_at":"2026-08-12T11:30:27.913197Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.898445Z","title":"In Proceedings of the 54th ACM Technical Symposium on Computer Science Education V","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.898445Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f801bd39eaf2274315f6dfa0c21a81b792c058a57b81e3403e6fdfd4fd9be20b","observation_id":"0762d504-54d2-4793-bc71-6d97e968ad95","resolution":{"observed_at":"2026-08-12T11:30:27.898445Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.923300Z","title":"Price, Samiha Marwan, and Joseph Jay Williams","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.923300Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2ee220253050ba8a2e74e0860b05b810c836f17e7e20cbf4644dad5e16b62810","observation_id":"4b879563-1418-48b4-a809-15bc9eeb3355","resolution":{"observed_at":"2026-08-12T11:30:27.923300Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.908275Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.908275Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b618e7840e7168e9cd1f6c72920e20e37465f1d99c205d128a91df35bcb052c2","observation_id":"3cd10ca0-a1e4-4964-8bb5-3e01903cf36d","resolution":{"observed_at":"2026-08-12T11:30:27.908275Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"1833.35618","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.961268Z","title":null,"venue":null,"work_id":"05cc0cb9-5224-47d7-b17f-9aeb7af5b40a","year":2022},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.933509Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f7e7981091cd82d1ad9901c9bdb41992d3443e0ab33094bb9d6f4a027b57f782","observation_id":"55544338-25ad-4635-a40f-a4f12ea0886f","resolution":{"observed_at":"2026-08-12T11:30:28.969555Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.918367Z","title":"Becker, Bailey Kimmel, Jared Wright, and Ben Briggs","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.918367Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a760d5b802078447ffe29c0b038087b0cb864306d2ba1de946d736eaf1689b28","observation_id":"2c8d6d53-c0f4-482a-8f7f-db8b4782cc55","resolution":{"observed_at":"2026-08-12T11:30:27.918367Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.539670Z","title":"Price, Samiha Marwan, Michael Winters, and Joseph Jay Williams","venue":null,"work_id":"985d4ccc-cd32-47c8-826a-38c40be79bf4","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.928074Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6a2a5cac95b980265e063e52fae3f1da1fa7bc5c2122ac21b23d6f7833f2951c","observation_id":"ed01883d-9245-4224-a096-196fe164d436","resolution":{"observed_at":"2026-08-12T11:30:32.544289Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.20792","last_updated":"2024-07-30T12:55:42Z","snapshot_observed_at":"2026-08-12T23:12:03.607386Z","submitted_at":"2024-07-30T12:55:42Z","title":"How Novice Programmers Use and Experience ChatGPT when Solving Programming Exercises in an Introductory Course","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.20792","snapshot_observed_at":"2026-08-12T11:30:27.952170Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.952170Z"},"links":{"cited_paper":"/paper/2407.20792","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b023b5b8b127519ca85781c72953c8233ad30f2b5dfbc97cf40feb6e8f79b9ea","observation_id":"aa3f8c6c-f504-4640-baa6-eee803b0b1de","resolution":{"observed_at":"2026-08-12T11:30:27.952170Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.524763Z","title":null,"venue":null,"work_id":"bad5133b-a824-4748-9377-d0b311c03ace","year":2006},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.938338Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:dbab0b92440870f571170edcb87a21e658db5b2f9ec34ab5b5bade49c0b90d0d","observation_id":"1bf578ff-6020-4ba7-b3d2-5164531dd5d3","resolution":{"observed_at":"2026-08-12T11:30:32.530048Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.963619Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.963619Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b0786b7297daa4b679d0dbe317242c6ae9c3bd557681c51dc9ea68c73be19adf","observation_id":"cb53221f-2e75-4dac-91b8-951000c20b2a","resolution":{"observed_at":"2026-08-12T11:30:27.963619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2019.16020","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.806070Z","title":null,"venue":null,"work_id":"7ba59d12-413b-45f0-b74c-fb3828fd9ac9","year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.947627Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:87839ad15b1c4c4498c801788ab6d475a7a6ac86cdf12a62085fd4c0b066ff6d","observation_id":"8aad0103-980a-4f90-b3b1-434d5e5ad1df","resolution":{"observed_at":"2026-08-12T11:30:28.813852Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.972494Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.972494Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b0f322069c90b93e46689706faabbeca778899c9ea02b11c35736da117fba632","observation_id":"c3416b10-484a-4121-9ca1-2c6ecbb527e3","resolution":{"observed_at":"2026-08-12T11:30:27.972494Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.958729Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.958729Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:697c0d72f3e2f08cc10c57ad70121e1a143d69fde0918fc3e1dd5828d0437fb5","observation_id":"ce1cda8d-2c69-4e77-909c-ea84027c014b","resolution":{"observed_at":"2026-08-12T11:30:27.958729Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.981810Z","title":"Wiggins, Fahmid M","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.981810Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:cbc0dce1478c7bf9ac9af556828cffb98989673a0a282b4466ea6514cd42d590","observation_id":"c6bb094c-cda7-472d-81a9-b2148958f760","resolution":{"observed_at":"2026-08-12T11:30:27.981810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.968087Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.968087Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8694aae9304e583aa159cb76b693176e6098f38633a1c04c84fe0c87f150d71d","observation_id":"faf24bd6-7717-42fe-82c0-d9296e96e760","resolution":{"observed_at":"2026-08-12T11:30:27.968087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.991617Z","title":"Bai, Robert Tairas, and Yu Huang","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.991617Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:1db30565eac434f32e6fa63668c0c6cdaa134ac9f6a19ceecf60c46ef0496982","observation_id":"3a16b9ce-e337-46bb-816c-5ebd4432e74d","resolution":{"observed_at":"2026-08-12T11:30:27.991617Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.977093Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.977093Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5b7dc62af337320f97244d6373bd65a2640952531d5085d80846ebe3c76abd92","observation_id":"f7a308c9-3677-4888-9622-3c91dc3d9b93","resolution":{"observed_at":"2026-08-12T11:30:27.977093Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.986744Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.986744Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2cce7364857814b6bf425275b2ba10339daf81b6ea01ff6545459425ffa07c82","observation_id":"17414038-65f2-4153-8c93-cd2830e2fb2b","resolution":{"observed_at":"2026-08-12T11:30:27.986744Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.995986Z","title":"Zamfirescu-Pereira, Richmond Y","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.995986Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:9e10ee512c4e86040cdd12e6a8eaa759f6ec49a7f539ea71db6f89fb0b93fe67","observation_id":"1a7018da-4c37-40a5-8d2f-8cd7fec552be","resolution":{"observed_at":"2026-08-12T11:30:27.995986Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.650787Z","title":"In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems (Yokohama, Japan) (CHI ’21)","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.650787Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b2f48cb3d62031fb6052e1ca2cb4cdabbaa9742d05fd270abb8dc97d53f97b66","observation_id":"588635e6-980d-4115-bef7-b8bb5215a244","resolution":{"observed_at":"2026-08-12T11:30:27.650787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.785836Z","title":"In Proceedings of the 2023 33 Brown et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.785836Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:152428765bab05dcdf041f719ce6ab67af0a3be9c1780da3128dfee6f7855667","observation_id":"4572e063-bf9f-4733-8dd0-f20aa1bc95f4","resolution":{"observed_at":"2026-08-12T11:30:27.785836Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","latest_version":1,"primary_category":"cs.CY","snapshot_observed_at":"2026-08-12T22:44:16.833640Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers"},"reference_resolution":{"displayed":80,"state_counts":{"malformed_identifier":3,"metadata_mismatch":11,"parse_uncertain":0,"unresolved":59,"verified_exact":2,"verified_fuzzy":5},"total_outbound_references":80},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 80 of 80 outbound references and 0 inbound Pith citation observations for arXiv:2411.18151."}