{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:S3SMFEZANBNPO5I436BW3PHFQN","short_pith_number":"pith:S3SMFEZA","schema_version":"1.0","canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","source":{"kind":"arxiv","id":"2404.12994","version":2},"attestation_state":"computed","paper":{"title":"Rethinking the Evaluation of Dialogue Systems: Effects of User Feedback on Crowdworkers and LLMs","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Clemencia Siro, Maarten de Rijke, Mohammad Aliannejadi","submitted_at":"2024-04-19T16:45:50Z","abstract_excerpt":"In ad-hoc retrieval, evaluation relies heavily on user actions, including implicit feedback. In a conversational setting such signals are usually unavailable due to the nature of the interactions, and, instead, the evaluation often relies on crowdsourced evaluation labels. The role of user feedback in annotators' assessment of turns in a conversational perception has been little studied. We focus on how the evaluation of task-oriented dialogue systems (TDSs), is affected by considering user feedback, explicit or implicit, as provided through the follow-up utterance of a turn being evaluated. W"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.12994","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"ad451ff720f81b07f8b8bc46471c5d96a27d648d3789f16a213919d8bea4a972","abstract_canon_sha256":"34603770be802b0e351a7b8d17ee737357b8c68a4dae1814a9b5fd8cd08d2cd2"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:13:33.424700Z","signature_b64":"Eih5Jlv1J1PHSUhz3oiHlex3nRI3pyhamAU3AKyB4ndgvTv0Z+rqjq4lUeFtBzagFPclpSCSuUEMOJf7B/VoDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","last_reissued_at":"2026-07-05T08:13:33.424177Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:13:33.424177Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Rethinking the Evaluation of Dialogue Systems: Effects of User Feedback on Crowdworkers and LLMs","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Clemencia Siro, Maarten de Rijke, Mohammad Aliannejadi","submitted_at":"2024-04-19T16:45:50Z","abstract_excerpt":"In ad-hoc retrieval, evaluation relies heavily on user actions, including implicit feedback. In a conversational setting such signals are usually unavailable due to the nature of the interactions, and, instead, the evaluation often relies on crowdsourced evaluation labels. The role of user feedback in annotators' assessment of turns in a conversational perception has been little studied. We focus on how the evaluation of task-oriented dialogue systems (TDSs), is affected by considering user feedback, explicit or implicit, as provided through the follow-up utterance of a turn being evaluated. W"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.12994","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.12994/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.12994","created_at":"2026-07-05T08:13:33.424240+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.12994v2","created_at":"2026-07-05T08:13:33.424240+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.12994","created_at":"2026-07-05T08:13:33.424240+00:00"},{"alias_kind":"pith_short_12","alias_value":"S3SMFEZANBNP","created_at":"2026-07-05T08:13:33.424240+00:00"},{"alias_kind":"pith_short_16","alias_value":"S3SMFEZANBNPO5I4","created_at":"2026-07-05T08:13:33.424240+00:00"},{"alias_kind":"pith_short_8","alias_value":"S3SMFEZA","created_at":"2026-07-05T08:13:33.424240+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN","json":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN.json","graph_json":"https://pith.science/api/pith-number/S3SMFEZANBNPO5I436BW3PHFQN/graph.json","events_json":"https://pith.science/api/pith-number/S3SMFEZANBNPO5I436BW3PHFQN/events.json","paper":"https://pith.science/paper/S3SMFEZA"},"agent_actions":{"view_html":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN","download_json":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN.json","view_paper":"https://pith.science/paper/S3SMFEZA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.12994&json=true","fetch_graph":"https://pith.science/api/pith-number/S3SMFEZANBNPO5I436BW3PHFQN/graph.json","fetch_events":"https://pith.science/api/pith-number/S3SMFEZANBNPO5I436BW3PHFQN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/action/storage_attestation","attest_author":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/action/author_attestation","sign_citation":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/action/citation_signature","submit_replication":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/action/replication_record"}},"created_at":"2026-07-05T08:13:33.424240+00:00","updated_at":"2026-07-05T08:13:33.424240+00:00"}