{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:OCBFZ65DI5Z6Q4FUVPZMVXPJWN","short_pith_number":"pith:OCBFZ65D","canonical_record":{"source":{"id":"2312.12604","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2023-12-19T21:18:14Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"da5f64fab25bb45bde09526069d752ca0b5011cb6f668bcca4485b1ec6bda5a8","abstract_canon_sha256":"015025646689ad5e6b52f444a81159e4614bdbcee5f28ab18fbd3df0a5bfccef"},"schema_version":"1.0"},"canonical_sha256":"70825cfba34773e870b4abf2cadde9b3483ad53f749c51e07c04d808295bce76","source":{"kind":"arxiv","id":"2312.12604","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.12604","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"arxiv_version","alias_value":"2312.12604v2","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.12604","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_12","alias_value":"OCBFZ65DI5Z6","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_16","alias_value":"OCBFZ65DI5Z6Q4FU","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_8","alias_value":"OCBFZ65D","created_at":"2026-07-05T08:43:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:OCBFZ65DI5Z6Q4FUVPZMVXPJWN","target":"record","payload":{"canonical_record":{"source":{"id":"2312.12604","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2023-12-19T21:18:14Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"da5f64fab25bb45bde09526069d752ca0b5011cb6f668bcca4485b1ec6bda5a8","abstract_canon_sha256":"015025646689ad5e6b52f444a81159e4614bdbcee5f28ab18fbd3df0a5bfccef"},"schema_version":"1.0"},"canonical_sha256":"70825cfba34773e870b4abf2cadde9b3483ad53f749c51e07c04d808295bce76","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:43:25.113038Z","signature_b64":"/tXLulPXi+YBcW2CebJrBhXDeVyoFgIf+LWHvBQdy3uKVnViKiyu7XkhWlafRJdE/HjDLusHV2Qqb0gHJi54DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"70825cfba34773e870b4abf2cadde9b3483ad53f749c51e07c04d808295bce76","last_reissued_at":"2026-07-05T08:43:25.112438Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:43:25.112438Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2312.12604","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:43:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XsS6ajTE4IGvHv9aCuKJc9IHeBtnbW5LVteLIn14y9oyL731+3cgilNasw3v+Q4X3Nh8CDSwfM3uGwIymOvVDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T14:52:04.480509Z"},"content_sha256":"5112a02534f6cec1df8452d99fee4962b1bcd3d760ba872635b56d25ed2e8b57","schema_version":"1.0","event_id":"sha256:5112a02534f6cec1df8452d99fee4962b1bcd3d760ba872635b56d25ed2e8b57"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:OCBFZ65DI5Z6Q4FUVPZMVXPJWN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An empirical study of testing machine learning in the wild","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.SE","authors_text":"Ahmed E. Hassan, Armstrong Foundjem, Foutse Khomh, Moses Openja, Mouna Abidi, Zhen Ming (Jack) Jiang","submitted_at":"2023-12-19T21:18:14Z","abstract_excerpt":"Recently, machine and deep learning (ML/DL) algorithms have been increasingly adopted in many software systems. Due to their inductive nature, ensuring the quality of these systems remains a significant challenge for the research community. Unlike traditional software built deductively by writing explicit rules, ML/DL systems infer rules from training data. Recent research in ML/DL quality assurance has adapted concepts from traditional software testing, such as mutation testing, to improve reliability. However, it is unclear if these proposed testing techniques are adopted in practice, or if "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.12604","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2312.12604/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:43:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ExGruhX1J02jPRNIzAlKyfmN46JXEyIkW6Od0bKPwjU5zKK3popNKDT919WJIzAymesYZbQ4DQhJOJdqaKtGAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T14:52:04.480878Z"},"content_sha256":"6cd29f1cf143d55e518fce93c6d8be80db73e9e82a233b9235f4b6789e159d6b","schema_version":"1.0","event_id":"sha256:6cd29f1cf143d55e518fce93c6d8be80db73e9e82a233b9235f4b6789e159d6b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/bundle.json","state_url":"https://pith.science/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T14:52:04Z","links":{"resolver":"https://pith.science/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN","bundle":"https://pith.science/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/bundle.json","state":"https://pith.science/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OCBFZ65DI5Z6Q4FUVPZMVXPJWN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:OCBFZ65DI5Z6Q4FUVPZMVXPJWN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"015025646689ad5e6b52f444a81159e4614bdbcee5f28ab18fbd3df0a5bfccef","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2023-12-19T21:18:14Z","title_canon_sha256":"da5f64fab25bb45bde09526069d752ca0b5011cb6f668bcca4485b1ec6bda5a8"},"schema_version":"1.0","source":{"id":"2312.12604","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2312.12604","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"arxiv_version","alias_value":"2312.12604v2","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2312.12604","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_12","alias_value":"OCBFZ65DI5Z6","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_16","alias_value":"OCBFZ65DI5Z6Q4FU","created_at":"2026-07-05T08:43:25Z"},{"alias_kind":"pith_short_8","alias_value":"OCBFZ65D","created_at":"2026-07-05T08:43:25Z"}],"graph_snapshots":[{"event_id":"sha256:6cd29f1cf143d55e518fce93c6d8be80db73e9e82a233b9235f4b6789e159d6b","target":"graph","created_at":"2026-07-05T08:43:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2312.12604/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recently, machine and deep learning (ML/DL) algorithms have been increasingly adopted in many software systems. Due to their inductive nature, ensuring the quality of these systems remains a significant challenge for the research community. Unlike traditional software built deductively by writing explicit rules, ML/DL systems infer rules from training data. Recent research in ML/DL quality assurance has adapted concepts from traditional software testing, such as mutation testing, to improve reliability. However, it is unclear if these proposed testing techniques are adopted in practice, or if ","authors_text":"Ahmed E. Hassan, Armstrong Foundjem, Foutse Khomh, Moses Openja, Mouna Abidi, Zhen Ming (Jack) Jiang","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2023-12-19T21:18:14Z","title":"An empirical study of testing machine learning in the wild"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2312.12604","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5112a02534f6cec1df8452d99fee4962b1bcd3d760ba872635b56d25ed2e8b57","target":"record","created_at":"2026-07-05T08:43:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"015025646689ad5e6b52f444a81159e4614bdbcee5f28ab18fbd3df0a5bfccef","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2023-12-19T21:18:14Z","title_canon_sha256":"da5f64fab25bb45bde09526069d752ca0b5011cb6f668bcca4485b1ec6bda5a8"},"schema_version":"1.0","source":{"id":"2312.12604","kind":"arxiv","version":2}},"canonical_sha256":"70825cfba34773e870b4abf2cadde9b3483ad53f749c51e07c04d808295bce76","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"70825cfba34773e870b4abf2cadde9b3483ad53f749c51e07c04d808295bce76","first_computed_at":"2026-07-05T08:43:25.112438Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:43:25.112438Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/tXLulPXi+YBcW2CebJrBhXDeVyoFgIf+LWHvBQdy3uKVnViKiyu7XkhWlafRJdE/HjDLusHV2Qqb0gHJi54DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:43:25.113038Z","signed_message":"canonical_sha256_bytes"},"source_id":"2312.12604","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5112a02534f6cec1df8452d99fee4962b1bcd3d760ba872635b56d25ed2e8b57","sha256:6cd29f1cf143d55e518fce93c6d8be80db73e9e82a233b9235f4b6789e159d6b"],"state_sha256":"3970c906144b5f8c17ddfd6fe871cf83ba91e48964c5cac93a232a105c4385b1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BQwZRxCMbJF9tv55vbSs4qukS4CfTg7f4iQJQ9HY8MmVZH+NsdQr4oSo4QSXQe4759RCxiufumxKBPwbkT5RDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T14:52:04.482822Z","bundle_sha256":"b5b90683708939eb190e7a25c1d74eb296de4b41cee8acf45d33580f3b88bdcd"}}