{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:MGMM2HIQPLRABPUWW6RHXGB4AM","short_pith_number":"pith:MGMM2HIQ","canonical_record":{"source":{"id":"2307.02732","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-07-06T02:31:38Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"e9cdcc7fc769cf859396fbad622c3e201755272fc6e2eb53a13c0c5cfc9d26e1","abstract_canon_sha256":"1524c61f6ea8dd5d5ac9277bbd2341723dd0a3a70047c6e22f5bf7d50760a77a"},"schema_version":"1.0"},"canonical_sha256":"6198cd1d107ae200be96b7a27b983c0338b39b2808b9318acddd7cbe3c3e7756","source":{"kind":"arxiv","id":"2307.02732","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.02732","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"arxiv_version","alias_value":"2307.02732v1","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.02732","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_12","alias_value":"MGMM2HIQPLRA","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_16","alias_value":"MGMM2HIQPLRABPUW","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_8","alias_value":"MGMM2HIQ","created_at":"2026-07-05T06:28:23Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:MGMM2HIQPLRABPUWW6RHXGB4AM","target":"record","payload":{"canonical_record":{"source":{"id":"2307.02732","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-07-06T02:31:38Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"e9cdcc7fc769cf859396fbad622c3e201755272fc6e2eb53a13c0c5cfc9d26e1","abstract_canon_sha256":"1524c61f6ea8dd5d5ac9277bbd2341723dd0a3a70047c6e22f5bf7d50760a77a"},"schema_version":"1.0"},"canonical_sha256":"6198cd1d107ae200be96b7a27b983c0338b39b2808b9318acddd7cbe3c3e7756","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:28:23.540334Z","signature_b64":"zxTw+KdB8XRGuQ7KsvRZpOkOQi/+QQsDlpaTCmnWeZq2PQLxnHiaCpOzvK12KNRNkrY/CSXfplj1p0mrK+H6AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6198cd1d107ae200be96b7a27b983c0338b39b2808b9318acddd7cbe3c3e7756","last_reissued_at":"2026-07-05T06:28:23.539931Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:28:23.539931Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2307.02732","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:28:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5Q0LHdPnXG+RECxfgYtRz9W7X2I/la3ZLgwqZoTxSm5bW0A1O9UHTtUDJ97uIpQQELTcM2ifK8QB3xbTJNhTAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:33:08.468239Z"},"content_sha256":"3aad9da3ae6ef547cfde2ce8f284961e799bed393ee39854d9df05a67a7f574e","schema_version":"1.0","event_id":"sha256:3aad9da3ae6ef547cfde2ce8f284961e799bed393ee39854d9df05a67a7f574e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:MGMM2HIQPLRABPUWW6RHXGB4AM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating the Evaluators: Are Current Few-Shot Learning Benchmarks Fit for Purpose?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Henry Gouk, Lu\\'isa Shimabucoro, Timothy Hospedales","submitted_at":"2023-07-06T02:31:38Z","abstract_excerpt":"Numerous benchmarks for Few-Shot Learning have been proposed in the last decade. However all of these benchmarks focus on performance averaged over many tasks, and the question of how to reliably evaluate and tune models trained for individual tasks in this regime has not been addressed. This paper presents the first investigation into task-level evaluation -- a fundamental step when deploying a model. We measure the accuracy of performance estimators in the few-shot setting, consider strategies for model selection, and examine the reasons for the failure of evaluators usually thought of as be"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.02732","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2307.02732/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:28:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7xXUT4BqbYy2576apd3D5e/h2Z/ke2gBVJPvnS/asgaM6AkLQp/HWKl4+PrNe+cuY9/pzGN2/mwcoQ9qV7ExDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:33:08.468636Z"},"content_sha256":"fab30b8892b987f6f2d3f2934ed63e3ee0e62e220588eed9468e9450ca973989","schema_version":"1.0","event_id":"sha256:fab30b8892b987f6f2d3f2934ed63e3ee0e62e220588eed9468e9450ca973989"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/bundle.json","state_url":"https://pith.science/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T12:33:08Z","links":{"resolver":"https://pith.science/pith/MGMM2HIQPLRABPUWW6RHXGB4AM","bundle":"https://pith.science/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/bundle.json","state":"https://pith.science/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MGMM2HIQPLRABPUWW6RHXGB4AM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:MGMM2HIQPLRABPUWW6RHXGB4AM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1524c61f6ea8dd5d5ac9277bbd2341723dd0a3a70047c6e22f5bf7d50760a77a","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-07-06T02:31:38Z","title_canon_sha256":"e9cdcc7fc769cf859396fbad622c3e201755272fc6e2eb53a13c0c5cfc9d26e1"},"schema_version":"1.0","source":{"id":"2307.02732","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.02732","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"arxiv_version","alias_value":"2307.02732v1","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.02732","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_12","alias_value":"MGMM2HIQPLRA","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_16","alias_value":"MGMM2HIQPLRABPUW","created_at":"2026-07-05T06:28:23Z"},{"alias_kind":"pith_short_8","alias_value":"MGMM2HIQ","created_at":"2026-07-05T06:28:23Z"}],"graph_snapshots":[{"event_id":"sha256:fab30b8892b987f6f2d3f2934ed63e3ee0e62e220588eed9468e9450ca973989","target":"graph","created_at":"2026-07-05T06:28:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2307.02732/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Numerous benchmarks for Few-Shot Learning have been proposed in the last decade. However all of these benchmarks focus on performance averaged over many tasks, and the question of how to reliably evaluate and tune models trained for individual tasks in this regime has not been addressed. This paper presents the first investigation into task-level evaluation -- a fundamental step when deploying a model. We measure the accuracy of performance estimators in the few-shot setting, consider strategies for model selection, and examine the reasons for the failure of evaluators usually thought of as be","authors_text":"Henry Gouk, Lu\\'isa Shimabucoro, Timothy Hospedales","cross_cats":["stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-07-06T02:31:38Z","title":"Evaluating the Evaluators: Are Current Few-Shot Learning Benchmarks Fit for Purpose?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.02732","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3aad9da3ae6ef547cfde2ce8f284961e799bed393ee39854d9df05a67a7f574e","target":"record","created_at":"2026-07-05T06:28:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1524c61f6ea8dd5d5ac9277bbd2341723dd0a3a70047c6e22f5bf7d50760a77a","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-07-06T02:31:38Z","title_canon_sha256":"e9cdcc7fc769cf859396fbad622c3e201755272fc6e2eb53a13c0c5cfc9d26e1"},"schema_version":"1.0","source":{"id":"2307.02732","kind":"arxiv","version":1}},"canonical_sha256":"6198cd1d107ae200be96b7a27b983c0338b39b2808b9318acddd7cbe3c3e7756","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6198cd1d107ae200be96b7a27b983c0338b39b2808b9318acddd7cbe3c3e7756","first_computed_at":"2026-07-05T06:28:23.539931Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:28:23.539931Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zxTw+KdB8XRGuQ7KsvRZpOkOQi/+QQsDlpaTCmnWeZq2PQLxnHiaCpOzvK12KNRNkrY/CSXfplj1p0mrK+H6AA==","signature_status":"signed_v1","signed_at":"2026-07-05T06:28:23.540334Z","signed_message":"canonical_sha256_bytes"},"source_id":"2307.02732","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3aad9da3ae6ef547cfde2ce8f284961e799bed393ee39854d9df05a67a7f574e","sha256:fab30b8892b987f6f2d3f2934ed63e3ee0e62e220588eed9468e9450ca973989"],"state_sha256":"88ed46082702d51781ce4c6dea509ca8671af953c481491f68f908198d6776cc"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vfPE8k9KtR9hXugcu8k2W0uoodSIv0Q5nwFuyCDLSJFkOHlEo8/VRC9ND9mlSe+2z0Lu9S09m22oSXhTJgbbCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T12:33:08.470614Z","bundle_sha256":"1fa31c648129b4001c33d372fa4f682f4732b2679697c1c868ca9cc0c90d389c"}}