{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:I233GLQYHIYE6GDK5P6W35UPVC","short_pith_number":"pith:I233GLQY","canonical_record":{"source":{"id":"2411.05224","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-07T22:36:19Z","cross_cats_sorted":[],"title_canon_sha256":"eaf4270b4972cb6caf2d5ef376e89c400d7dd886fb2525cbea366a7b2a550d07","abstract_canon_sha256":"3a7b96358ebf7859afe0ce51eaa092a5ed6d65b734d3036e7dfdf8d6dd0f60ee"},"schema_version":"1.0"},"canonical_sha256":"46b7b32e183a304f186aebfd6df68fa8856621a7739af9879e918ce91e84c7e1","source":{"kind":"arxiv","id":"2411.05224","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.05224","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"arxiv_version","alias_value":"2411.05224v1","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05224","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_12","alias_value":"I233GLQYHIYE","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_16","alias_value":"I233GLQYHIYE6GDK","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_8","alias_value":"I233GLQY","created_at":"2026-07-05T09:32:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:I233GLQYHIYE6GDK5P6W35UPVC","target":"record","payload":{"canonical_record":{"source":{"id":"2411.05224","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-07T22:36:19Z","cross_cats_sorted":[],"title_canon_sha256":"eaf4270b4972cb6caf2d5ef376e89c400d7dd886fb2525cbea366a7b2a550d07","abstract_canon_sha256":"3a7b96358ebf7859afe0ce51eaa092a5ed6d65b734d3036e7dfdf8d6dd0f60ee"},"schema_version":"1.0"},"canonical_sha256":"46b7b32e183a304f186aebfd6df68fa8856621a7739af9879e918ce91e84c7e1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:32:56.378037Z","signature_b64":"CYFZnpqfFtTPbY2LhouTelitNebEQh5pUNrKHhXWjCGHJVv4sSoUiAsR+wn7Sg/5cb3bwawWIcxuIY72agzkCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"46b7b32e183a304f186aebfd6df68fa8856621a7739af9879e918ce91e84c7e1","last_reissued_at":"2026-07-05T09:32:56.377562Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:32:56.377562Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.05224","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:32:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vPa/Vbr0oTdGByoONkadUGuP9i7/g+SxV6I6gZOM22A5WTcdWEccJlXK/hF67aLWOsixr2i/zK92uHmQyjD2CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T19:25:50.230964Z"},"content_sha256":"86cf8d7fc7936714f018eae55d015d0c70bb186f98f62cad9eda5f5de078fdf7","schema_version":"1.0","event_id":"sha256:86cf8d7fc7936714f018eae55d015d0c70bb186f98f62cad9eda5f5de078fdf7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:I233GLQYHIYE6GDK5P6W35UPVC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond the Numbers: Transparency in Relation Extraction Benchmark Creation and Leaderboards","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Allan Hanbury, Varvara Arzt","submitted_at":"2024-11-07T22:36:19Z","abstract_excerpt":"This paper investigates the transparency in the creation of benchmarks and the use of leaderboards for measuring progress in NLP, with a focus on the relation extraction (RE) task. Existing RE benchmarks often suffer from insufficient documentation, lacking crucial details such as data sources, inter-annotator agreement, the algorithms used for the selection of instances for datasets, and information on potential biases like dataset imbalance. Progress in RE is frequently measured by leaderboards that rank systems based on evaluation methods, typically limited to aggregate metrics like F1-scor"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05224","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.05224/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:32:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZGGZnmIg0OCgOhaQmAIGH4dpLNLDEYi3yHQBfSyrIHNToFEUww72wZ3i+tPuSseQ4LraBXN74U3JbpEYP4tUCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T19:25:50.231457Z"},"content_sha256":"89987c50b8e77c56af925ce5cb0a0e60fae7bf79e1ecf1ee4a9bc7c27c70e986","schema_version":"1.0","event_id":"sha256:89987c50b8e77c56af925ce5cb0a0e60fae7bf79e1ecf1ee4a9bc7c27c70e986"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I233GLQYHIYE6GDK5P6W35UPVC/bundle.json","state_url":"https://pith.science/pith/I233GLQYHIYE6GDK5P6W35UPVC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I233GLQYHIYE6GDK5P6W35UPVC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-23T19:25:50Z","links":{"resolver":"https://pith.science/pith/I233GLQYHIYE6GDK5P6W35UPVC","bundle":"https://pith.science/pith/I233GLQYHIYE6GDK5P6W35UPVC/bundle.json","state":"https://pith.science/pith/I233GLQYHIYE6GDK5P6W35UPVC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I233GLQYHIYE6GDK5P6W35UPVC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:I233GLQYHIYE6GDK5P6W35UPVC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3a7b96358ebf7859afe0ce51eaa092a5ed6d65b734d3036e7dfdf8d6dd0f60ee","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-07T22:36:19Z","title_canon_sha256":"eaf4270b4972cb6caf2d5ef376e89c400d7dd886fb2525cbea366a7b2a550d07"},"schema_version":"1.0","source":{"id":"2411.05224","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.05224","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"arxiv_version","alias_value":"2411.05224v1","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05224","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_12","alias_value":"I233GLQYHIYE","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_16","alias_value":"I233GLQYHIYE6GDK","created_at":"2026-07-05T09:32:56Z"},{"alias_kind":"pith_short_8","alias_value":"I233GLQY","created_at":"2026-07-05T09:32:56Z"}],"graph_snapshots":[{"event_id":"sha256:89987c50b8e77c56af925ce5cb0a0e60fae7bf79e1ecf1ee4a9bc7c27c70e986","target":"graph","created_at":"2026-07-05T09:32:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.05224/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper investigates the transparency in the creation of benchmarks and the use of leaderboards for measuring progress in NLP, with a focus on the relation extraction (RE) task. Existing RE benchmarks often suffer from insufficient documentation, lacking crucial details such as data sources, inter-annotator agreement, the algorithms used for the selection of instances for datasets, and information on potential biases like dataset imbalance. Progress in RE is frequently measured by leaderboards that rank systems based on evaluation methods, typically limited to aggregate metrics like F1-scor","authors_text":"Allan Hanbury, Varvara Arzt","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-07T22:36:19Z","title":"Beyond the Numbers: Transparency in Relation Extraction Benchmark Creation and Leaderboards"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05224","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:86cf8d7fc7936714f018eae55d015d0c70bb186f98f62cad9eda5f5de078fdf7","target":"record","created_at":"2026-07-05T09:32:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3a7b96358ebf7859afe0ce51eaa092a5ed6d65b734d3036e7dfdf8d6dd0f60ee","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-07T22:36:19Z","title_canon_sha256":"eaf4270b4972cb6caf2d5ef376e89c400d7dd886fb2525cbea366a7b2a550d07"},"schema_version":"1.0","source":{"id":"2411.05224","kind":"arxiv","version":1}},"canonical_sha256":"46b7b32e183a304f186aebfd6df68fa8856621a7739af9879e918ce91e84c7e1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"46b7b32e183a304f186aebfd6df68fa8856621a7739af9879e918ce91e84c7e1","first_computed_at":"2026-07-05T09:32:56.377562Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:32:56.377562Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"CYFZnpqfFtTPbY2LhouTelitNebEQh5pUNrKHhXWjCGHJVv4sSoUiAsR+wn7Sg/5cb3bwawWIcxuIY72agzkCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:32:56.378037Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.05224","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:86cf8d7fc7936714f018eae55d015d0c70bb186f98f62cad9eda5f5de078fdf7","sha256:89987c50b8e77c56af925ce5cb0a0e60fae7bf79e1ecf1ee4a9bc7c27c70e986"],"state_sha256":"1b627a1441250d218ffca97c047b08d7a4f60e9952ec8a5de4ee910d38ffa3bc"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rQb/JTmcwHbgyr1gTC3csNWvmC/6EHZWmNlyPz6hx7/3VAae+yUSAzL5trg6TaiY6kku28c62kY6O7BXsJOeCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-23T19:25:50.236555Z","bundle_sha256":"4182f3f387c3d66d573d2f1d51f18f73b5d56cbea7a904addad261b05c3739ef"}}