{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2019:MVBCTSRS4QTNZRCV5MPJ5EBFO6","short_pith_number":"pith:MVBCTSRS","schema_version":"1.0","canonical_sha256":"654229ca32e426dcc455eb1e9e902577be825265c60d0f01b29d4a4131e38d4b","source":{"kind":"arxiv","id":"1906.09317","version":1},"attestation_state":"computed","paper":{"title":"Identification of Tasks, Datasets, Evaluation Metrics, and Numeric Scores for Scientific Leaderboards Construction","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Charles Jochim, Debasis Ganguly, Francesca Bonin, Martin Gleize, Yufang Hou","submitted_at":"2019-06-21T20:55:57Z","abstract_excerpt":"While the fast-paced inception of novel tasks and new datasets helps foster active research in a community towards interesting directions, keeping track of the abundance of research activity in different areas on different datasets is likely to become increasingly difficult. The community could greatly benefit from an automatic system able to summarize scientific results, e.g., in the form of a leaderboard. In this paper we build two datasets and develop a framework (TDMS-IE) aimed at automatically extracting task, dataset, metric and score from NLP papers, towards the automatic construction o"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1906.09317","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-06-21T20:55:57Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"38e69495dad3db7bcee7edb60f5436aed5e6a957659ce8ed7c4291549fc6fe81","abstract_canon_sha256":"d7d921c759ecb65024c19332e81324e21e452225643311e92fc195163e8ad404"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:42:38.539618Z","signature_b64":"x2YMjpmJ+lL2onUEtccOFwTL6S40THPTBqhvLQvIQzj0kiKZNnl+AT55b8JPDW0y9YUEU8GOXM/lM1tIAZfHAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"654229ca32e426dcc455eb1e9e902577be825265c60d0f01b29d4a4131e38d4b","last_reissued_at":"2026-05-17T23:42:38.538910Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:42:38.538910Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Identification of Tasks, Datasets, Evaluation Metrics, and Numeric Scores for Scientific Leaderboards Construction","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Charles Jochim, Debasis Ganguly, Francesca Bonin, Martin Gleize, Yufang Hou","submitted_at":"2019-06-21T20:55:57Z","abstract_excerpt":"While the fast-paced inception of novel tasks and new datasets helps foster active research in a community towards interesting directions, keeping track of the abundance of research activity in different areas on different datasets is likely to become increasingly difficult. The community could greatly benefit from an automatic system able to summarize scientific results, e.g., in the form of a leaderboard. In this paper we build two datasets and develop a framework (TDMS-IE) aimed at automatically extracting task, dataset, metric and score from NLP papers, towards the automatic construction o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1906.09317","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1906.09317","created_at":"2026-05-17T23:42:38.539028+00:00"},{"alias_kind":"arxiv_version","alias_value":"1906.09317v1","created_at":"2026-05-17T23:42:38.539028+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1906.09317","created_at":"2026-05-17T23:42:38.539028+00:00"},{"alias_kind":"pith_short_12","alias_value":"MVBCTSRS4QTN","created_at":"2026-05-18T12:33:24.271573+00:00"},{"alias_kind":"pith_short_16","alias_value":"MVBCTSRS4QTNZRCV","created_at":"2026-05-18T12:33:24.271573+00:00"},{"alias_kind":"pith_short_8","alias_value":"MVBCTSRS","created_at":"2026-05-18T12:33:24.271573+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6","json":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6.json","graph_json":"https://pith.science/api/pith-number/MVBCTSRS4QTNZRCV5MPJ5EBFO6/graph.json","events_json":"https://pith.science/api/pith-number/MVBCTSRS4QTNZRCV5MPJ5EBFO6/events.json","paper":"https://pith.science/paper/MVBCTSRS"},"agent_actions":{"view_html":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6","download_json":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6.json","view_paper":"https://pith.science/paper/MVBCTSRS","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1906.09317&json=true","fetch_graph":"https://pith.science/api/pith-number/MVBCTSRS4QTNZRCV5MPJ5EBFO6/graph.json","fetch_events":"https://pith.science/api/pith-number/MVBCTSRS4QTNZRCV5MPJ5EBFO6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6/action/storage_attestation","attest_author":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6/action/author_attestation","sign_citation":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6/action/citation_signature","submit_replication":"https://pith.science/pith/MVBCTSRS4QTNZRCV5MPJ5EBFO6/action/replication_record"}},"created_at":"2026-05-17T23:42:38.539028+00:00","updated_at":"2026-05-17T23:42:38.539028+00:00"}