{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:IMJBDS23QR5BYMQS6JBBUZE4MD","short_pith_number":"pith:IMJBDS23","canonical_record":{"source":{"id":"2409.04041","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-09-06T06:20:11Z","cross_cats_sorted":[],"title_canon_sha256":"5cd474ef0ad4832d38b41771ace9c69d5b472644b43121dbc2ada864664eda13","abstract_canon_sha256":"29230af40d161ff57405a6cd64c90a701453649e8d1757b12d366387916cd182"},"schema_version":"1.0"},"canonical_sha256":"431211cb5b847a1c3212f2421a649c60f143a78b8ea1c166585730a84b61f453","source":{"kind":"arxiv","id":"2409.04041","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.04041","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"arxiv_version","alias_value":"2409.04041v1","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.04041","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_12","alias_value":"IMJBDS23QR5B","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_16","alias_value":"IMJBDS23QR5BYMQS","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_8","alias_value":"IMJBDS23","created_at":"2026-07-05T09:04:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:IMJBDS23QR5BYMQS6JBBUZE4MD","target":"record","payload":{"canonical_record":{"source":{"id":"2409.04041","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-09-06T06:20:11Z","cross_cats_sorted":[],"title_canon_sha256":"5cd474ef0ad4832d38b41771ace9c69d5b472644b43121dbc2ada864664eda13","abstract_canon_sha256":"29230af40d161ff57405a6cd64c90a701453649e8d1757b12d366387916cd182"},"schema_version":"1.0"},"canonical_sha256":"431211cb5b847a1c3212f2421a649c60f143a78b8ea1c166585730a84b61f453","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:04:00.719962Z","signature_b64":"PqA+iC/k+3Mk0JKb2ifw+owY2ulGV2JqUUr/kdSnvA9//wPs+5gPGVymlCCk+w5zT7Wp6mChssQ6aB4VeGZZAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"431211cb5b847a1c3212f2421a649c60f143a78b8ea1c166585730a84b61f453","last_reissued_at":"2026-07-05T09:04:00.719509Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:04:00.719509Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.04041","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:04:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yqO61xubAVbf3YfqVQFMdTCQcl+KsXkB1xVOyjIYSX9rOGeGvXON8ZSCdlNBy5v5bgzwDGEM2vVsPSxYNCsNDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T00:20:55.198595Z"},"content_sha256":"023b534e181aada6ef1d2708a71cd3d44c869fdd75568d705e940529c0bff3f3","schema_version":"1.0","event_id":"sha256:023b534e181aada6ef1d2708a71cd3d44c869fdd75568d705e940529c0bff3f3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:IMJBDS23QR5BYMQS6JBBUZE4MD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On Evaluation of Vision Datasets and Models using Human Competency Frameworks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CV","authors_text":"Charchit Sharma, Deepak Vijaykeerthy, Rahul Ramachandran, Tejal Kulkarni, Vineeth N Balasubramanian","submitted_at":"2024-09-06T06:20:11Z","abstract_excerpt":"Evaluating models and datasets in computer vision remains a challenging task, with most leaderboards relying solely on accuracy. While accuracy is a popular metric for model evaluation, it provides only a coarse assessment by considering a single model's score on all dataset items. This paper explores Item Response Theory (IRT), a framework that infers interpretable latent parameters for an ensemble of models and each dataset item, enabling richer evaluation and analysis beyond the single accuracy number. Leveraging IRT, we assess model calibration, select informative data subsets, and demonst"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.04041","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.04041/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:04:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0jio0hLTM/31mOvBoS9P+kWwgIWSzZkYfBFKD3t+KEZL9tc76VH0gsUVpu6dBPhqngfedhL0fwJVHRFC8j79DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T00:20:55.199457Z"},"content_sha256":"8162f9526a996cb29de170b5013730535b51929c5ca399cedde823ba479217fd","schema_version":"1.0","event_id":"sha256:8162f9526a996cb29de170b5013730535b51929c5ca399cedde823ba479217fd"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/bundle.json","state_url":"https://pith.science/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T00:20:55Z","links":{"resolver":"https://pith.science/pith/IMJBDS23QR5BYMQS6JBBUZE4MD","bundle":"https://pith.science/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/bundle.json","state":"https://pith.science/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IMJBDS23QR5BYMQS6JBBUZE4MD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IMJBDS23QR5BYMQS6JBBUZE4MD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"29230af40d161ff57405a6cd64c90a701453649e8d1757b12d366387916cd182","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-09-06T06:20:11Z","title_canon_sha256":"5cd474ef0ad4832d38b41771ace9c69d5b472644b43121dbc2ada864664eda13"},"schema_version":"1.0","source":{"id":"2409.04041","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.04041","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"arxiv_version","alias_value":"2409.04041v1","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.04041","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_12","alias_value":"IMJBDS23QR5B","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_16","alias_value":"IMJBDS23QR5BYMQS","created_at":"2026-07-05T09:04:00Z"},{"alias_kind":"pith_short_8","alias_value":"IMJBDS23","created_at":"2026-07-05T09:04:00Z"}],"graph_snapshots":[{"event_id":"sha256:8162f9526a996cb29de170b5013730535b51929c5ca399cedde823ba479217fd","target":"graph","created_at":"2026-07-05T09:04:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.04041/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating models and datasets in computer vision remains a challenging task, with most leaderboards relying solely on accuracy. While accuracy is a popular metric for model evaluation, it provides only a coarse assessment by considering a single model's score on all dataset items. This paper explores Item Response Theory (IRT), a framework that infers interpretable latent parameters for an ensemble of models and each dataset item, enabling richer evaluation and analysis beyond the single accuracy number. Leveraging IRT, we assess model calibration, select informative data subsets, and demonst","authors_text":"Charchit Sharma, Deepak Vijaykeerthy, Rahul Ramachandran, Tejal Kulkarni, Vineeth N Balasubramanian","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-09-06T06:20:11Z","title":"On Evaluation of Vision Datasets and Models using Human Competency Frameworks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.04041","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:023b534e181aada6ef1d2708a71cd3d44c869fdd75568d705e940529c0bff3f3","target":"record","created_at":"2026-07-05T09:04:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"29230af40d161ff57405a6cd64c90a701453649e8d1757b12d366387916cd182","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2024-09-06T06:20:11Z","title_canon_sha256":"5cd474ef0ad4832d38b41771ace9c69d5b472644b43121dbc2ada864664eda13"},"schema_version":"1.0","source":{"id":"2409.04041","kind":"arxiv","version":1}},"canonical_sha256":"431211cb5b847a1c3212f2421a649c60f143a78b8ea1c166585730a84b61f453","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"431211cb5b847a1c3212f2421a649c60f143a78b8ea1c166585730a84b61f453","first_computed_at":"2026-07-05T09:04:00.719509Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:04:00.719509Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"PqA+iC/k+3Mk0JKb2ifw+owY2ulGV2JqUUr/kdSnvA9//wPs+5gPGVymlCCk+w5zT7Wp6mChssQ6aB4VeGZZAA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:04:00.719962Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.04041","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:023b534e181aada6ef1d2708a71cd3d44c869fdd75568d705e940529c0bff3f3","sha256:8162f9526a996cb29de170b5013730535b51929c5ca399cedde823ba479217fd"],"state_sha256":"045bd71cc73d2513ff62dc59f54b971e8067cc5f22d4a1badbf2b6ef71929a43"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4/pCRwMR+wKOML24AsOsz/uMNAmaowVPfrE/S5CJMpIT4TO/qrnvSccW9LFyIFn+UKooarxgIjutGJE2kdNjDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T00:20:55.207552Z","bundle_sha256":"cfffd824f6f4f535d19a5b5382134aaa292ba4c822959ef5228fad6d23c11ccb"}}