{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:VLAP3GB3PGG4I4H7TKLU4ID36U","short_pith_number":"pith:VLAP3GB3","canonical_record":{"source":{"id":"2506.21182","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-26T12:40:48Z","cross_cats_sorted":["cs.AI","cs.SE"],"title_canon_sha256":"feba7ff61f14bb7e461455ab8bc6cc667122f63cb7851bc9e533a03b47eacdc4","abstract_canon_sha256":"441624673519a59ce0f99f48ba11527cde52ba52241148235699886ca46b7775"},"schema_version":"1.0"},"canonical_sha256":"aac0fd983b798dc470ff9a974e207bf52f89b1754f177d0641593a3adc375636","source":{"kind":"arxiv","id":"2506.21182","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.21182","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"arxiv_version","alias_value":"2506.21182v1","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.21182","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_12","alias_value":"VLAP3GB3PGG4","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_16","alias_value":"VLAP3GB3PGG4I4H7","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_8","alias_value":"VLAP3GB3","created_at":"2026-07-05T11:27:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:VLAP3GB3PGG4I4H7TKLU4ID36U","target":"record","payload":{"canonical_record":{"source":{"id":"2506.21182","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-26T12:40:48Z","cross_cats_sorted":["cs.AI","cs.SE"],"title_canon_sha256":"feba7ff61f14bb7e461455ab8bc6cc667122f63cb7851bc9e533a03b47eacdc4","abstract_canon_sha256":"441624673519a59ce0f99f48ba11527cde52ba52241148235699886ca46b7775"},"schema_version":"1.0"},"canonical_sha256":"aac0fd983b798dc470ff9a974e207bf52f89b1754f177d0641593a3adc375636","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:27:41.163500Z","signature_b64":"JdaviqcdOHPzecIyIyTI3GoVM/dzqR/t6ut38GKcJMXmgelDAHPw3lRD6dCTgXubwsKS9I727l1s8xxPy6YyAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"aac0fd983b798dc470ff9a974e207bf52f89b1754f177d0641593a3adc375636","last_reissued_at":"2026-07-05T11:27:41.163050Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:27:41.163050Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.21182","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ws3IUVvwAy4Mahq9NS0UuMrOO5OvR2FnMG7tk6SPiGUbivhEnD2W+v+w/4kcuqKxQmhfyjyjooHRA448D//GDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T03:09:22.498653Z"},"content_sha256":"fbc974874af1053aaebb1ccbd08159867ead23535785bf42e76f6e170a7f5fbe","schema_version":"1.0","event_id":"sha256:fbc974874af1053aaebb1ccbd08159867ead23535785bf42e76f6e170a7f5fbe"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:VLAP3GB3PGG4I4H7TKLU4ID36U","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Maintaining MTEB: Towards Long Term Usability and Reproducibility of Embedding Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.SE"],"primary_cat":"cs.CL","authors_text":"Imene Kerboua, Isaac Chung, Kenneth Enevoldsen, Marton Kardos, Roman Solomatin","submitted_at":"2025-06-26T12:40:48Z","abstract_excerpt":"The Massive Text Embedding Benchmark (MTEB) has become a standard evaluation platform for text embedding models. While previous work has established the core benchmark methodology, this paper focuses on the engineering aspects that ensure MTEB's continued reproducibility and extensibility. We present our approach to maintaining robust continuous integration pipelines that validate dataset integrity, automate test execution, and assess benchmark results' generalizability. We detail the design choices that collectively enhance reproducibility and usability. Furthermore, we discuss our strategies"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.21182","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.21182/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mrpOwL2mzAhUi/+P2dyZjEkW/eAmvygb1/D8h4twUfu60UHk+DYGn/kFsWIcH5nnC6Q0KZZlSRfMpPjIDUikCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T03:09:22.499040Z"},"content_sha256":"3645856a9313f9a62044888d2ca095ca94767ea63eb7c01cfbc0f90d2ca7c186","schema_version":"1.0","event_id":"sha256:3645856a9313f9a62044888d2ca095ca94767ea63eb7c01cfbc0f90d2ca7c186"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/bundle.json","state_url":"https://pith.science/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T03:09:22Z","links":{"resolver":"https://pith.science/pith/VLAP3GB3PGG4I4H7TKLU4ID36U","bundle":"https://pith.science/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/bundle.json","state":"https://pith.science/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VLAP3GB3PGG4I4H7TKLU4ID36U/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:VLAP3GB3PGG4I4H7TKLU4ID36U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"441624673519a59ce0f99f48ba11527cde52ba52241148235699886ca46b7775","cross_cats_sorted":["cs.AI","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-26T12:40:48Z","title_canon_sha256":"feba7ff61f14bb7e461455ab8bc6cc667122f63cb7851bc9e533a03b47eacdc4"},"schema_version":"1.0","source":{"id":"2506.21182","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.21182","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"arxiv_version","alias_value":"2506.21182v1","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.21182","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_12","alias_value":"VLAP3GB3PGG4","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_16","alias_value":"VLAP3GB3PGG4I4H7","created_at":"2026-07-05T11:27:41Z"},{"alias_kind":"pith_short_8","alias_value":"VLAP3GB3","created_at":"2026-07-05T11:27:41Z"}],"graph_snapshots":[{"event_id":"sha256:3645856a9313f9a62044888d2ca095ca94767ea63eb7c01cfbc0f90d2ca7c186","target":"graph","created_at":"2026-07-05T11:27:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.21182/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The Massive Text Embedding Benchmark (MTEB) has become a standard evaluation platform for text embedding models. While previous work has established the core benchmark methodology, this paper focuses on the engineering aspects that ensure MTEB's continued reproducibility and extensibility. We present our approach to maintaining robust continuous integration pipelines that validate dataset integrity, automate test execution, and assess benchmark results' generalizability. We detail the design choices that collectively enhance reproducibility and usability. Furthermore, we discuss our strategies","authors_text":"Imene Kerboua, Isaac Chung, Kenneth Enevoldsen, Marton Kardos, Roman Solomatin","cross_cats":["cs.AI","cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-26T12:40:48Z","title":"Maintaining MTEB: Towards Long Term Usability and Reproducibility of Embedding Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.21182","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fbc974874af1053aaebb1ccbd08159867ead23535785bf42e76f6e170a7f5fbe","target":"record","created_at":"2026-07-05T11:27:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"441624673519a59ce0f99f48ba11527cde52ba52241148235699886ca46b7775","cross_cats_sorted":["cs.AI","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-26T12:40:48Z","title_canon_sha256":"feba7ff61f14bb7e461455ab8bc6cc667122f63cb7851bc9e533a03b47eacdc4"},"schema_version":"1.0","source":{"id":"2506.21182","kind":"arxiv","version":1}},"canonical_sha256":"aac0fd983b798dc470ff9a974e207bf52f89b1754f177d0641593a3adc375636","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"aac0fd983b798dc470ff9a974e207bf52f89b1754f177d0641593a3adc375636","first_computed_at":"2026-07-05T11:27:41.163050Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:27:41.163050Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JdaviqcdOHPzecIyIyTI3GoVM/dzqR/t6ut38GKcJMXmgelDAHPw3lRD6dCTgXubwsKS9I727l1s8xxPy6YyAA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:27:41.163500Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.21182","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fbc974874af1053aaebb1ccbd08159867ead23535785bf42e76f6e170a7f5fbe","sha256:3645856a9313f9a62044888d2ca095ca94767ea63eb7c01cfbc0f90d2ca7c186"],"state_sha256":"8931ead623f63f3fed7561c5fcd0e758acb635a08a0b6821255ab168d5e1dc57"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"r6DpXMxhzCky7lHks+zIbsec7fIGYg5Y1rqCYlORhaOOmTVku1Z4p5/fcQi7fxwriPhGiQCKnlvmXxe8TUonCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T03:09:22.501923Z","bundle_sha256":"19211aaa16720e5e1795cdbfeb57f2650ea7691c8aeb3604e184a3b43a1267d7"}}