{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:Z2QXSBA7RNHELRRSNRUIK6F7LR","short_pith_number":"pith:Z2QXSBA7","canonical_record":{"source":{"id":"2010.13447","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-10-26T09:36:15Z","cross_cats_sorted":[],"title_canon_sha256":"5285eb02c193f420dbaaf0d5019034ca032402a089d882d2f78f6e382ad24a79","abstract_canon_sha256":"c91cd8e2c51322a1415e281a3c792585e45c88e7e53561a31e69382eff6dbed9"},"schema_version":"1.0"},"canonical_sha256":"cea179041f8b4e45c6326c688578bf5c4a12972b76c7c0557a5da2a36b8f1dff","source":{"kind":"arxiv","id":"2010.13447","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.13447","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"arxiv_version","alias_value":"2010.13447v1","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.13447","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_12","alias_value":"Z2QXSBA7RNHE","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_16","alias_value":"Z2QXSBA7RNHELRRS","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_8","alias_value":"Z2QXSBA7","created_at":"2026-07-05T01:46:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:Z2QXSBA7RNHELRRSNRUIK6F7LR","target":"record","payload":{"canonical_record":{"source":{"id":"2010.13447","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-10-26T09:36:15Z","cross_cats_sorted":[],"title_canon_sha256":"5285eb02c193f420dbaaf0d5019034ca032402a089d882d2f78f6e382ad24a79","abstract_canon_sha256":"c91cd8e2c51322a1415e281a3c792585e45c88e7e53561a31e69382eff6dbed9"},"schema_version":"1.0"},"canonical_sha256":"cea179041f8b4e45c6326c688578bf5c4a12972b76c7c0557a5da2a36b8f1dff","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:46:06.435124Z","signature_b64":"/80PdFT+8Bbt/G0sGbLR02tldIbeXHGVcW2u7b0nipUyZCn6624qDULwgWzjr7cOQXj+b+8Upp0sYFQG3uugCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cea179041f8b4e45c6326c688578bf5c4a12972b76c7c0557a5da2a36b8f1dff","last_reissued_at":"2026-07-05T01:46:06.434741Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:46:06.434741Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2010.13447","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"htpuT3tszi7SM6zI0ZzoZkspRKME7beH4ylVk+itYWOYP0+dzXBDrjBFDAgI4L76Bj41iETTAjA76JVCGsNiDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T20:45:10.345984Z"},"content_sha256":"cf3b8faeae42afa1845f5a8bd2240982262b27f67517d6ef7d85a89beb284142","schema_version":"1.0","event_id":"sha256:cf3b8faeae42afa1845f5a8bd2240982262b27f67517d6ef7d85a89beb284142"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:Z2QXSBA7RNHELRRSNRUIK6F7LR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How to Measure the Reproducibility of System-oriented IR Experiments","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Ian Soboroff, Maria Maistro, Nicola Ferro, Norbert Fuhr, Philipp Schaer, Tetsuya Sakai, Timo Breuer","submitted_at":"2020-10-26T09:36:15Z","abstract_excerpt":"Replicability and reproducibility of experimental results are primary concerns in all the areas of science and IR is not an exception. Besides the problem of moving the field towards more reproducible experimental practices and protocols, we also face a severe methodological issue: we do not have any means to assess when reproduced is reproduced. Moreover, we lack any reproducibility-oriented dataset, which would allow us to develop such methods. To address these issues, we compare several measures to objectively quantify to what extent we have replicated or reproduced a system-oriented IR exp"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.13447","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2010.13447/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:46:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kQ7moBM6EWva5ELZxm26HvjXeVB/6m0aWRS5+W3EZgQpyjhmVjMq404VtKSkMNU/ehmkj76nQTEEyOmbqWXgCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T20:45:10.346366Z"},"content_sha256":"420c1c85254bd9949d096de4e318385ebf93a8b1a0ea86e7162bbdaf38cf4de7","schema_version":"1.0","event_id":"sha256:420c1c85254bd9949d096de4e318385ebf93a8b1a0ea86e7162bbdaf38cf4de7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/bundle.json","state_url":"https://pith.science/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T20:45:10Z","links":{"resolver":"https://pith.science/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR","bundle":"https://pith.science/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/bundle.json","state":"https://pith.science/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Z2QXSBA7RNHELRRSNRUIK6F7LR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:Z2QXSBA7RNHELRRSNRUIK6F7LR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c91cd8e2c51322a1415e281a3c792585e45c88e7e53561a31e69382eff6dbed9","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-10-26T09:36:15Z","title_canon_sha256":"5285eb02c193f420dbaaf0d5019034ca032402a089d882d2f78f6e382ad24a79"},"schema_version":"1.0","source":{"id":"2010.13447","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2010.13447","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"arxiv_version","alias_value":"2010.13447v1","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2010.13447","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_12","alias_value":"Z2QXSBA7RNHE","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_16","alias_value":"Z2QXSBA7RNHELRRS","created_at":"2026-07-05T01:46:06Z"},{"alias_kind":"pith_short_8","alias_value":"Z2QXSBA7","created_at":"2026-07-05T01:46:06Z"}],"graph_snapshots":[{"event_id":"sha256:420c1c85254bd9949d096de4e318385ebf93a8b1a0ea86e7162bbdaf38cf4de7","target":"graph","created_at":"2026-07-05T01:46:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2010.13447/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Replicability and reproducibility of experimental results are primary concerns in all the areas of science and IR is not an exception. Besides the problem of moving the field towards more reproducible experimental practices and protocols, we also face a severe methodological issue: we do not have any means to assess when reproduced is reproduced. Moreover, we lack any reproducibility-oriented dataset, which would allow us to develop such methods. To address these issues, we compare several measures to objectively quantify to what extent we have replicated or reproduced a system-oriented IR exp","authors_text":"Ian Soboroff, Maria Maistro, Nicola Ferro, Norbert Fuhr, Philipp Schaer, Tetsuya Sakai, Timo Breuer","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-10-26T09:36:15Z","title":"How to Measure the Reproducibility of System-oriented IR Experiments"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2010.13447","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cf3b8faeae42afa1845f5a8bd2240982262b27f67517d6ef7d85a89beb284142","target":"record","created_at":"2026-07-05T01:46:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c91cd8e2c51322a1415e281a3c792585e45c88e7e53561a31e69382eff6dbed9","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-10-26T09:36:15Z","title_canon_sha256":"5285eb02c193f420dbaaf0d5019034ca032402a089d882d2f78f6e382ad24a79"},"schema_version":"1.0","source":{"id":"2010.13447","kind":"arxiv","version":1}},"canonical_sha256":"cea179041f8b4e45c6326c688578bf5c4a12972b76c7c0557a5da2a36b8f1dff","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cea179041f8b4e45c6326c688578bf5c4a12972b76c7c0557a5da2a36b8f1dff","first_computed_at":"2026-07-05T01:46:06.434741Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:46:06.434741Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/80PdFT+8Bbt/G0sGbLR02tldIbeXHGVcW2u7b0nipUyZCn6624qDULwgWzjr7cOQXj+b+8Upp0sYFQG3uugCw==","signature_status":"signed_v1","signed_at":"2026-07-05T01:46:06.435124Z","signed_message":"canonical_sha256_bytes"},"source_id":"2010.13447","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cf3b8faeae42afa1845f5a8bd2240982262b27f67517d6ef7d85a89beb284142","sha256:420c1c85254bd9949d096de4e318385ebf93a8b1a0ea86e7162bbdaf38cf4de7"],"state_sha256":"9bb79433c1241f3f82d4b8cd4b6ca77eaa322f128bca12f38cbc4e179c27bc51"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7n5BnuvLQw0EOQtSy6MslRcKujan3fovOAmuqbISVffcoomV4106h6AciZOeX8356AUSDZZT/2ydWWxWVwgRBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T20:45:10.348280Z","bundle_sha256":"a84dd8cd9cf2efa0ea4769b351f90cf720a47ec6be9c65970674e0e7c3e32b75"}}