{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:S3SMFEZANBNPO5I436BW3PHFQN","short_pith_number":"pith:S3SMFEZA","canonical_record":{"source":{"id":"2404.12994","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"ad451ff720f81b07f8b8bc46471c5d96a27d648d3789f16a213919d8bea4a972","abstract_canon_sha256":"34603770be802b0e351a7b8d17ee737357b8c68a4dae1814a9b5fd8cd08d2cd2"},"schema_version":"1.0"},"canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","source":{"kind":"arxiv","id":"2404.12994","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.12994","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"arxiv_version","alias_value":"2404.12994v2","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.12994","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_12","alias_value":"S3SMFEZANBNP","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_16","alias_value":"S3SMFEZANBNPO5I4","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_8","alias_value":"S3SMFEZA","created_at":"2026-07-05T08:13:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:S3SMFEZANBNPO5I436BW3PHFQN","target":"record","payload":{"canonical_record":{"source":{"id":"2404.12994","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"ad451ff720f81b07f8b8bc46471c5d96a27d648d3789f16a213919d8bea4a972","abstract_canon_sha256":"34603770be802b0e351a7b8d17ee737357b8c68a4dae1814a9b5fd8cd08d2cd2"},"schema_version":"1.0"},"canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:13:33.424700Z","signature_b64":"Eih5Jlv1J1PHSUhz3oiHlex3nRI3pyhamAU3AKyB4ndgvTv0Z+rqjq4lUeFtBzagFPclpSCSuUEMOJf7B/VoDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","last_reissued_at":"2026-07-05T08:13:33.424177Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:13:33.424177Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.12994","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:13:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UPbMjce+F4+jU52NT4Wpucj58MrYQZnB5wifuGOGJKnl2YqL/UIsfJrxzS3hARRQru7+jsUdjG4xcAnC0W3dAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T17:12:15.474503Z"},"content_sha256":"2df8369a630fd7ac415bb88f0e5c3e9c329da8d8ed2c98fed9cf61b04263a6be","schema_version":"1.0","event_id":"sha256:2df8369a630fd7ac415bb88f0e5c3e9c329da8d8ed2c98fed9cf61b04263a6be"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:S3SMFEZANBNPO5I436BW3PHFQN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Rethinking the Evaluation of Dialogue Systems: Effects of User Feedback on Crowdworkers and LLMs","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Clemencia Siro, Maarten de Rijke, Mohammad Aliannejadi","submitted_at":"2024-04-19T16:45:50Z","abstract_excerpt":"In ad-hoc retrieval, evaluation relies heavily on user actions, including implicit feedback. In a conversational setting such signals are usually unavailable due to the nature of the interactions, and, instead, the evaluation often relies on crowdsourced evaluation labels. The role of user feedback in annotators' assessment of turns in a conversational perception has been little studied. We focus on how the evaluation of task-oriented dialogue systems (TDSs), is affected by considering user feedback, explicit or implicit, as provided through the follow-up utterance of a turn being evaluated. W"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.12994","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.12994/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:13:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xiM/6LWdxPAdRswRuUEHO7U3eweQ3EK6m/Hl5u0lXkJ+JP68Qkwqh9zIkme74ok3gRSnMr5mJfQWVOlWOA7gAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T17:12:15.475364Z"},"content_sha256":"2e0d9ec10785e50c473442bbbb91f18c9e5385ae06fa6642a3cfc6cee40a2344","schema_version":"1.0","event_id":"sha256:2e0d9ec10785e50c473442bbbb91f18c9e5385ae06fa6642a3cfc6cee40a2344"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/bundle.json","state_url":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S3SMFEZANBNPO5I436BW3PHFQN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T17:12:15Z","links":{"resolver":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN","bundle":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/bundle.json","state":"https://pith.science/pith/S3SMFEZANBNPO5I436BW3PHFQN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S3SMFEZANBNPO5I436BW3PHFQN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:S3SMFEZANBNPO5I436BW3PHFQN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34603770be802b0e351a7b8d17ee737357b8c68a4dae1814a9b5fd8cd08d2cd2","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","title_canon_sha256":"ad451ff720f81b07f8b8bc46471c5d96a27d648d3789f16a213919d8bea4a972"},"schema_version":"1.0","source":{"id":"2404.12994","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.12994","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"arxiv_version","alias_value":"2404.12994v2","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.12994","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_12","alias_value":"S3SMFEZANBNP","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_16","alias_value":"S3SMFEZANBNPO5I4","created_at":"2026-07-05T08:13:33Z"},{"alias_kind":"pith_short_8","alias_value":"S3SMFEZA","created_at":"2026-07-05T08:13:33Z"}],"graph_snapshots":[{"event_id":"sha256:2e0d9ec10785e50c473442bbbb91f18c9e5385ae06fa6642a3cfc6cee40a2344","target":"graph","created_at":"2026-07-05T08:13:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.12994/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In ad-hoc retrieval, evaluation relies heavily on user actions, including implicit feedback. In a conversational setting such signals are usually unavailable due to the nature of the interactions, and, instead, the evaluation often relies on crowdsourced evaluation labels. The role of user feedback in annotators' assessment of turns in a conversational perception has been little studied. We focus on how the evaluation of task-oriented dialogue systems (TDSs), is affected by considering user feedback, explicit or implicit, as provided through the follow-up utterance of a turn being evaluated. W","authors_text":"Clemencia Siro, Maarten de Rijke, Mohammad Aliannejadi","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","title":"Rethinking the Evaluation of Dialogue Systems: Effects of User Feedback on Crowdworkers and LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.12994","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2df8369a630fd7ac415bb88f0e5c3e9c329da8d8ed2c98fed9cf61b04263a6be","target":"record","created_at":"2026-07-05T08:13:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34603770be802b0e351a7b8d17ee737357b8c68a4dae1814a9b5fd8cd08d2cd2","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2024-04-19T16:45:50Z","title_canon_sha256":"ad451ff720f81b07f8b8bc46471c5d96a27d648d3789f16a213919d8bea4a972"},"schema_version":"1.0","source":{"id":"2404.12994","kind":"arxiv","version":2}},"canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"96e4c29320685af7751cdf836dbce5834bebe6ae956648399cfb18b55c2a6cd0","first_computed_at":"2026-07-05T08:13:33.424177Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:13:33.424177Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Eih5Jlv1J1PHSUhz3oiHlex3nRI3pyhamAU3AKyB4ndgvTv0Z+rqjq4lUeFtBzagFPclpSCSuUEMOJf7B/VoDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:13:33.424700Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.12994","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2df8369a630fd7ac415bb88f0e5c3e9c329da8d8ed2c98fed9cf61b04263a6be","sha256:2e0d9ec10785e50c473442bbbb91f18c9e5385ae06fa6642a3cfc6cee40a2344"],"state_sha256":"6bf8cca86fbf1de7919cf175d6fb39f1474f8350e45b0ec04638d642e8ffe74c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kf7HEmHoWL6eZA5qeieqaTnMIHLgb5CmSqyAu3e6hjjI2HG1Zl3r/oNg/p+6ywFaLC2RqQ8fx5Y7kgJhnWJHBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T17:12:15.480681Z","bundle_sha256":"2dde1ae5a48b4bd2edc2c60687f518ac66727f8cb5f4ea5161b8c108b1a6f58c"}}