{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:F6F3YNDU7UZQQGNCKLTUPQZCUT","short_pith_number":"pith:F6F3YNDU","canonical_record":{"source":{"id":"2411.14103","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-21T13:09:36Z","cross_cats_sorted":[],"title_canon_sha256":"9e7bf0047f90b776c0f6dee996ccfb289c6126dec9b3249592970e1010253ab2","abstract_canon_sha256":"94eab88ed574d7feb8c7db15cf4fde4599c01ce9a5cc53441344861e0fae1fc0"},"schema_version":"1.0"},"canonical_sha256":"2f8bbc3474fd330819a252e747c322a4f1174520e943239e2d3cb982fc47ae7e","source":{"kind":"arxiv","id":"2411.14103","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.14103","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"arxiv_version","alias_value":"2411.14103v1","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.14103","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_12","alias_value":"F6F3YNDU7UZQ","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_16","alias_value":"F6F3YNDU7UZQQGNC","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_8","alias_value":"F6F3YNDU","created_at":"2026-07-05T09:38:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:F6F3YNDU7UZQQGNCKLTUPQZCUT","target":"record","payload":{"canonical_record":{"source":{"id":"2411.14103","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-21T13:09:36Z","cross_cats_sorted":[],"title_canon_sha256":"9e7bf0047f90b776c0f6dee996ccfb289c6126dec9b3249592970e1010253ab2","abstract_canon_sha256":"94eab88ed574d7feb8c7db15cf4fde4599c01ce9a5cc53441344861e0fae1fc0"},"schema_version":"1.0"},"canonical_sha256":"2f8bbc3474fd330819a252e747c322a4f1174520e943239e2d3cb982fc47ae7e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:38:41.698160Z","signature_b64":"1pL0Lj+njqXdDzYAlfzWar0j3IqbJpGq182ujRHxrziufsmD2ToN2hWhyCAYpjFSjIVQPqzGiqioYMrY1hdZBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f8bbc3474fd330819a252e747c322a4f1174520e943239e2d3cb982fc47ae7e","last_reissued_at":"2026-07-05T09:38:41.697764Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:38:41.697764Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.14103","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:38:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wOy4XdbpubfxO1zoyWTEDN15fNvRFwqOE5qfJ3NGVURxdDQzm9wR/FFGRjXH8gXzjVzk2R7RVrkOVdO3rnwMDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T16:50:50.616597Z"},"content_sha256":"6f33ef125d604844386af7656fda47df3d4ed6fcbec34c451d6a86a69755f186","schema_version":"1.0","event_id":"sha256:6f33ef125d604844386af7656fda47df3d4ed6fcbec34c451d6a86a69755f186"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:F6F3YNDU7UZQQGNCKLTUPQZCUT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Lost in Inference: Rediscovering the Role of Natural Language Inference for Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Barbara Plank, David Esiobu, Dieuwke Hupkes, Lovish Madaan, Pontus Stenetorp","submitted_at":"2024-11-21T13:09:36Z","abstract_excerpt":"In the recent past, a popular way of evaluating natural language understanding (NLU), was to consider a model's ability to perform natural language inference (NLI) tasks. In this paper, we investigate if NLI tasks, that are rarely used for LLM evaluation, can still be informative for evaluating LLMs. Focusing on five different NLI benchmarks across six models of different scales, we investigate if they are able to discriminate models of different size and quality and how their accuracies develop during training. Furthermore, we investigate the extent to which the softmax distributions of model"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.14103","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.14103/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:38:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8k7lF72mZReBVkCc8NZY0cXYMEcbZsF2poDEG8OthqEStYdla/lGrX44d1tJ4CyAQaiJoFLAH1Ei+IXq2+KyAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T16:50:50.617128Z"},"content_sha256":"2d00df8ef24235d7625c4d4c8b079bd38b6924ed2b48aa9f1db3c5e5098427f1","schema_version":"1.0","event_id":"sha256:2d00df8ef24235d7625c4d4c8b079bd38b6924ed2b48aa9f1db3c5e5098427f1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/bundle.json","state_url":"https://pith.science/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T16:50:50Z","links":{"resolver":"https://pith.science/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT","bundle":"https://pith.science/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/bundle.json","state":"https://pith.science/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F6F3YNDU7UZQQGNCKLTUPQZCUT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:F6F3YNDU7UZQQGNCKLTUPQZCUT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"94eab88ed574d7feb8c7db15cf4fde4599c01ce9a5cc53441344861e0fae1fc0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-21T13:09:36Z","title_canon_sha256":"9e7bf0047f90b776c0f6dee996ccfb289c6126dec9b3249592970e1010253ab2"},"schema_version":"1.0","source":{"id":"2411.14103","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.14103","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"arxiv_version","alias_value":"2411.14103v1","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.14103","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_12","alias_value":"F6F3YNDU7UZQ","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_16","alias_value":"F6F3YNDU7UZQQGNC","created_at":"2026-07-05T09:38:41Z"},{"alias_kind":"pith_short_8","alias_value":"F6F3YNDU","created_at":"2026-07-05T09:38:41Z"}],"graph_snapshots":[{"event_id":"sha256:2d00df8ef24235d7625c4d4c8b079bd38b6924ed2b48aa9f1db3c5e5098427f1","target":"graph","created_at":"2026-07-05T09:38:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.14103/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In the recent past, a popular way of evaluating natural language understanding (NLU), was to consider a model's ability to perform natural language inference (NLI) tasks. In this paper, we investigate if NLI tasks, that are rarely used for LLM evaluation, can still be informative for evaluating LLMs. Focusing on five different NLI benchmarks across six models of different scales, we investigate if they are able to discriminate models of different size and quality and how their accuracies develop during training. Furthermore, we investigate the extent to which the softmax distributions of model","authors_text":"Barbara Plank, David Esiobu, Dieuwke Hupkes, Lovish Madaan, Pontus Stenetorp","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-21T13:09:36Z","title":"Lost in Inference: Rediscovering the Role of Natural Language Inference for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.14103","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6f33ef125d604844386af7656fda47df3d4ed6fcbec34c451d6a86a69755f186","target":"record","created_at":"2026-07-05T09:38:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"94eab88ed574d7feb8c7db15cf4fde4599c01ce9a5cc53441344861e0fae1fc0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-21T13:09:36Z","title_canon_sha256":"9e7bf0047f90b776c0f6dee996ccfb289c6126dec9b3249592970e1010253ab2"},"schema_version":"1.0","source":{"id":"2411.14103","kind":"arxiv","version":1}},"canonical_sha256":"2f8bbc3474fd330819a252e747c322a4f1174520e943239e2d3cb982fc47ae7e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2f8bbc3474fd330819a252e747c322a4f1174520e943239e2d3cb982fc47ae7e","first_computed_at":"2026-07-05T09:38:41.697764Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:38:41.697764Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1pL0Lj+njqXdDzYAlfzWar0j3IqbJpGq182ujRHxrziufsmD2ToN2hWhyCAYpjFSjIVQPqzGiqioYMrY1hdZBw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:38:41.698160Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.14103","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6f33ef125d604844386af7656fda47df3d4ed6fcbec34c451d6a86a69755f186","sha256:2d00df8ef24235d7625c4d4c8b079bd38b6924ed2b48aa9f1db3c5e5098427f1"],"state_sha256":"ea713e49eb53cd213612f694bca50f02127d857af1b60994afde0104e3ed4b63"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FEvtvy1yjkS0XCIvzaTOkTHNy4kN/sDLQ87loZ411kwoSzGU30wLyLEiVFs69RsIA9qKDNh1alrMjoaT9FosCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T16:50:50.621096Z","bundle_sha256":"19d9364ab9d4ec1c4474b4ef4f6c079f0d7b6296519fe82efbae7b6e9bacc4a6"}}