{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:I2K3H75DXQYUIQE2YKHJ2UMMMQ","short_pith_number":"pith:I2K3H75D","canonical_record":{"source":{"id":"2107.07229","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2021-07-15T10:08:18Z","cross_cats_sorted":[],"title_canon_sha256":"fbd13da8e68e4fb5e9d1c67aacf4226d207b1189d1591096b9ccdfa0830d388a","abstract_canon_sha256":"3ff84b4a3e5821dbaa9a9c08765ebe3912eccd5c619d6b3e1659227d5628c8bf"},"schema_version":"1.0"},"canonical_sha256":"4695b3ffa3bc3144409ac28e9d518c6405b02402989dd8df6ee270b1e3c44372","source":{"kind":"arxiv","id":"2107.07229","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.07229","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"arxiv_version","alias_value":"2107.07229v1","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.07229","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_12","alias_value":"I2K3H75DXQYU","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_16","alias_value":"I2K3H75DXQYUIQE2","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_8","alias_value":"I2K3H75D","created_at":"2026-07-05T02:58:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:I2K3H75DXQYUIQE2YKHJ2UMMMQ","target":"record","payload":{"canonical_record":{"source":{"id":"2107.07229","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2021-07-15T10:08:18Z","cross_cats_sorted":[],"title_canon_sha256":"fbd13da8e68e4fb5e9d1c67aacf4226d207b1189d1591096b9ccdfa0830d388a","abstract_canon_sha256":"3ff84b4a3e5821dbaa9a9c08765ebe3912eccd5c619d6b3e1659227d5628c8bf"},"schema_version":"1.0"},"canonical_sha256":"4695b3ffa3bc3144409ac28e9d518c6405b02402989dd8df6ee270b1e3c44372","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:58:12.106367Z","signature_b64":"eu/H6WINK+lmaVpmegFlJkpZyiNRr+th+TQ9zpmGjmWft7v/9OqgEY3CBkX1bS0hSKuR1owlOoVLFQfF8qzlAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4695b3ffa3bc3144409ac28e9d518c6405b02402989dd8df6ee270b1e3c44372","last_reissued_at":"2026-07-05T02:58:12.106005Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:58:12.106005Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.07229","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xc1bDtq6AzFq2R3xWS09f9teg3/aTtPuDFF2n1/o5r7M3WKW/gN4Z7n9FyLQwqKsjL2i9IphJmU1ajh/VsByDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T21:53:45.187735Z"},"content_sha256":"8a6f4c47ac58282fca5e94f559d848eb3d741d32b3d019d19dd9edcdacdd7e91","schema_version":"1.0","event_id":"sha256:8a6f4c47ac58282fca5e94f559d848eb3d741d32b3d019d19dd9edcdacdd7e91"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:I2K3H75DXQYUIQE2YKHJ2UMMMQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Trusting RoBERTa over BERT: Insights from CheckListing the Natural Language Inference Task","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Ishan Tarunesh, Monojit Choudhury, Somak Aditya","submitted_at":"2021-07-15T10:08:18Z","abstract_excerpt":"The recent state-of-the-art natural language understanding (NLU) systems often behave unpredictably, failing on simpler reasoning examples. Despite this, there has been limited focus on quantifying progress towards systems with more predictable behavior. We think that reasoning capability-wise behavioral summary is a step towards bridging this gap. We create a CheckList test-suite (184K examples) for the Natural Language Inference (NLI) task, a representative NLU task. We benchmark state-of-the-art NLI systems on this test-suite, which reveals fine-grained insights into the reasoning abilities"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.07229","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.07229/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pKt+AL0IKubpGV8gGhjeCMzJYuv6hJGs8/54diQ7s+4krDRmRWhS5aOnC7AZMVslTOFV9GZouRkvFW2SoFM0Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T21:53:45.188226Z"},"content_sha256":"5f19fb5516fe3005cfc482b1efa7c0cb7001ecb7d5cc5b2cb891541d5bf8f48e","schema_version":"1.0","event_id":"sha256:5f19fb5516fe3005cfc482b1efa7c0cb7001ecb7d5cc5b2cb891541d5bf8f48e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/bundle.json","state_url":"https://pith.science/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T21:53:45Z","links":{"resolver":"https://pith.science/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ","bundle":"https://pith.science/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/bundle.json","state":"https://pith.science/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I2K3H75DXQYUIQE2YKHJ2UMMMQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:I2K3H75DXQYUIQE2YKHJ2UMMMQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3ff84b4a3e5821dbaa9a9c08765ebe3912eccd5c619d6b3e1659227d5628c8bf","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2021-07-15T10:08:18Z","title_canon_sha256":"fbd13da8e68e4fb5e9d1c67aacf4226d207b1189d1591096b9ccdfa0830d388a"},"schema_version":"1.0","source":{"id":"2107.07229","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.07229","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"arxiv_version","alias_value":"2107.07229v1","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.07229","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_12","alias_value":"I2K3H75DXQYU","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_16","alias_value":"I2K3H75DXQYUIQE2","created_at":"2026-07-05T02:58:12Z"},{"alias_kind":"pith_short_8","alias_value":"I2K3H75D","created_at":"2026-07-05T02:58:12Z"}],"graph_snapshots":[{"event_id":"sha256:5f19fb5516fe3005cfc482b1efa7c0cb7001ecb7d5cc5b2cb891541d5bf8f48e","target":"graph","created_at":"2026-07-05T02:58:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.07229/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The recent state-of-the-art natural language understanding (NLU) systems often behave unpredictably, failing on simpler reasoning examples. Despite this, there has been limited focus on quantifying progress towards systems with more predictable behavior. We think that reasoning capability-wise behavioral summary is a step towards bridging this gap. We create a CheckList test-suite (184K examples) for the Natural Language Inference (NLI) task, a representative NLU task. We benchmark state-of-the-art NLI systems on this test-suite, which reveals fine-grained insights into the reasoning abilities","authors_text":"Ishan Tarunesh, Monojit Choudhury, Somak Aditya","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2021-07-15T10:08:18Z","title":"Trusting RoBERTa over BERT: Insights from CheckListing the Natural Language Inference Task"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.07229","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8a6f4c47ac58282fca5e94f559d848eb3d741d32b3d019d19dd9edcdacdd7e91","target":"record","created_at":"2026-07-05T02:58:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3ff84b4a3e5821dbaa9a9c08765ebe3912eccd5c619d6b3e1659227d5628c8bf","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2021-07-15T10:08:18Z","title_canon_sha256":"fbd13da8e68e4fb5e9d1c67aacf4226d207b1189d1591096b9ccdfa0830d388a"},"schema_version":"1.0","source":{"id":"2107.07229","kind":"arxiv","version":1}},"canonical_sha256":"4695b3ffa3bc3144409ac28e9d518c6405b02402989dd8df6ee270b1e3c44372","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4695b3ffa3bc3144409ac28e9d518c6405b02402989dd8df6ee270b1e3c44372","first_computed_at":"2026-07-05T02:58:12.106005Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:58:12.106005Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"eu/H6WINK+lmaVpmegFlJkpZyiNRr+th+TQ9zpmGjmWft7v/9OqgEY3CBkX1bS0hSKuR1owlOoVLFQfF8qzlAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T02:58:12.106367Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.07229","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8a6f4c47ac58282fca5e94f559d848eb3d741d32b3d019d19dd9edcdacdd7e91","sha256:5f19fb5516fe3005cfc482b1efa7c0cb7001ecb7d5cc5b2cb891541d5bf8f48e"],"state_sha256":"5412add3a2eeefbea932e905a0c4fd7494deeb252d5d0f4de702f213e127c911"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6EHUt79TQPBjGacd8XxCpXwh8S/j6/1XFQLnL6rYqjJwmyVUOh+lzCsTRi2kGMGQnsN2JKciQRHnTxjvVekxBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T21:53:45.191957Z","bundle_sha256":"ddbbc6d7a84eb720341ef96444183f7b1c3fc5fd947e7192ed6338d9b4aeb998"}}