{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:KL245GWRJXK73NXV7RSU26ZYOK","short_pith_number":"pith:KL245GWR","canonical_record":{"source":{"id":"1902.10236","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-02-26T21:33:48Z","cross_cats_sorted":[],"title_canon_sha256":"83fe2038ee177ac158a946a961f58de8859142bc667f8c7b5c94d50f3e8209d8","abstract_canon_sha256":"1ed8081f47ee207f431aeb551df315eeb0d23917a81936496e382e2818cf53db"},"schema_version":"1.0"},"canonical_sha256":"52f5ce9ad14dd5fdb6f5fc654d7b3872ab82e1fb2771178f2d3c4c1cd3791953","source":{"kind":"arxiv","id":"1902.10236","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1902.10236","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"arxiv_version","alias_value":"1902.10236v2","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1902.10236","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"pith_short_12","alias_value":"KL245GWRJXK7","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_16","alias_value":"KL245GWRJXK73NXV","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_8","alias_value":"KL245GWR","created_at":"2026-05-18T12:33:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:KL245GWRJXK73NXV7RSU26ZYOK","target":"record","payload":{"canonical_record":{"source":{"id":"1902.10236","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-02-26T21:33:48Z","cross_cats_sorted":[],"title_canon_sha256":"83fe2038ee177ac158a946a961f58de8859142bc667f8c7b5c94d50f3e8209d8","abstract_canon_sha256":"1ed8081f47ee207f431aeb551df315eeb0d23917a81936496e382e2818cf53db"},"schema_version":"1.0"},"canonical_sha256":"52f5ce9ad14dd5fdb6f5fc654d7b3872ab82e1fb2771178f2d3c4c1cd3791953","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:49:27.061918Z","signature_b64":"T2IcsO0rTc6p9eQl7DGhC+/Ob1dmBwk6AxPI+95ylNeVT1aJ8ONzB+KgydzwUfFh8ghgVOzMoGh/Xzi1FIDhBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"52f5ce9ad14dd5fdb6f5fc654d7b3872ab82e1fb2771178f2d3c4c1cd3791953","last_reissued_at":"2026-05-17T23:49:27.061148Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:49:27.061148Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1902.10236","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:49:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xxe9yelILFLP6OCjUXtF3aNIIV5jg3PYDInx1XlD1aI2owTn08IIyjYWRMpwgY8SgFW4zzPCC3LV9b+PzKeFBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-29T09:08:45.829010Z"},"content_sha256":"72a809af00edb5e281771780f50315e957ff143fc5bcaa1a9ea2595cd5d461e9","schema_version":"1.0","event_id":"sha256:72a809af00edb5e281771780f50315e957ff143fc5bcaa1a9ea2595cd5d461e9"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:KL245GWRJXK73NXV7RSU26ZYOK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Learning When Not to Answer: A Ternary Reward Structure for Reinforcement Learning based Question Answering","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Anjishnu Kumar, Arpit Mittal, Fr\\'ederic Godin","submitted_at":"2019-02-26T21:33:48Z","abstract_excerpt":"In this paper, we investigate the challenges of using reinforcement learning agents for question-answering over knowledge graphs for real-world applications. We examine the performance metrics used by state-of-the-art systems and determine that they are inadequate for such settings. More specifically, they do not evaluate the systems correctly for situations when there is no answer available and thus agents optimized for these metrics are poor at modeling confidence. We introduce a simple new performance metric for evaluating question-answering agents that is more representative of practical u"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1902.10236","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:49:27Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aUJuNT56tKbSf3v8JOiIw/s1audM5xbUVnstDdoiL2RQHIFYqzlLQlxE5JgUmy2Q0L5C9QZAOyZ1GXTGv3wXBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-29T09:08:45.829366Z"},"content_sha256":"2c1813232c8dc7525fbf01dc8960bf46181be88e16e55a09666c5d667ff909a9","schema_version":"1.0","event_id":"sha256:2c1813232c8dc7525fbf01dc8960bf46181be88e16e55a09666c5d667ff909a9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KL245GWRJXK73NXV7RSU26ZYOK/bundle.json","state_url":"https://pith.science/pith/KL245GWRJXK73NXV7RSU26ZYOK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KL245GWRJXK73NXV7RSU26ZYOK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-29T09:08:45Z","links":{"resolver":"https://pith.science/pith/KL245GWRJXK73NXV7RSU26ZYOK","bundle":"https://pith.science/pith/KL245GWRJXK73NXV7RSU26ZYOK/bundle.json","state":"https://pith.science/pith/KL245GWRJXK73NXV7RSU26ZYOK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KL245GWRJXK73NXV7RSU26ZYOK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:KL245GWRJXK73NXV7RSU26ZYOK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1ed8081f47ee207f431aeb551df315eeb0d23917a81936496e382e2818cf53db","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-02-26T21:33:48Z","title_canon_sha256":"83fe2038ee177ac158a946a961f58de8859142bc667f8c7b5c94d50f3e8209d8"},"schema_version":"1.0","source":{"id":"1902.10236","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1902.10236","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"arxiv_version","alias_value":"1902.10236v2","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1902.10236","created_at":"2026-05-17T23:49:27Z"},{"alias_kind":"pith_short_12","alias_value":"KL245GWRJXK7","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_16","alias_value":"KL245GWRJXK73NXV","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_8","alias_value":"KL245GWR","created_at":"2026-05-18T12:33:21Z"}],"graph_snapshots":[{"event_id":"sha256:2c1813232c8dc7525fbf01dc8960bf46181be88e16e55a09666c5d667ff909a9","target":"graph","created_at":"2026-05-17T23:49:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"In this paper, we investigate the challenges of using reinforcement learning agents for question-answering over knowledge graphs for real-world applications. We examine the performance metrics used by state-of-the-art systems and determine that they are inadequate for such settings. More specifically, they do not evaluate the systems correctly for situations when there is no answer available and thus agents optimized for these metrics are poor at modeling confidence. We introduce a simple new performance metric for evaluating question-answering agents that is more representative of practical u","authors_text":"Anjishnu Kumar, Arpit Mittal, Fr\\'ederic Godin","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-02-26T21:33:48Z","title":"Learning When Not to Answer: A Ternary Reward Structure for Reinforcement Learning based Question Answering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1902.10236","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:72a809af00edb5e281771780f50315e957ff143fc5bcaa1a9ea2595cd5d461e9","target":"record","created_at":"2026-05-17T23:49:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1ed8081f47ee207f431aeb551df315eeb0d23917a81936496e382e2818cf53db","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-02-26T21:33:48Z","title_canon_sha256":"83fe2038ee177ac158a946a961f58de8859142bc667f8c7b5c94d50f3e8209d8"},"schema_version":"1.0","source":{"id":"1902.10236","kind":"arxiv","version":2}},"canonical_sha256":"52f5ce9ad14dd5fdb6f5fc654d7b3872ab82e1fb2771178f2d3c4c1cd3791953","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"52f5ce9ad14dd5fdb6f5fc654d7b3872ab82e1fb2771178f2d3c4c1cd3791953","first_computed_at":"2026-05-17T23:49:27.061148Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:49:27.061148Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"T2IcsO0rTc6p9eQl7DGhC+/Ob1dmBwk6AxPI+95ylNeVT1aJ8ONzB+KgydzwUfFh8ghgVOzMoGh/Xzi1FIDhBg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:49:27.061918Z","signed_message":"canonical_sha256_bytes"},"source_id":"1902.10236","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:72a809af00edb5e281771780f50315e957ff143fc5bcaa1a9ea2595cd5d461e9","sha256:2c1813232c8dc7525fbf01dc8960bf46181be88e16e55a09666c5d667ff909a9"],"state_sha256":"a338ec08f50bb10beb9e0bd94c6b1ca15382397643cd3dc66068b5e54c4f20dc"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ytaSN4bYh7DeQDQPMLUcWXHV1MnxAWOzyX4HEMm0CX5gB2OK1iK+bsJ4qDxQk9KXL5+s2H6bvK4dpS6OasEaCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-29T09:08:45.831215Z","bundle_sha256":"8d6e18033ec28b29fcfb655b4aa8f3f17aeffe650462fe901fb775e5bdbf635d"}}