{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:MN5D2QMUOQV3AAD6X4UL63FDFC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4528c82dff9aaa0944a525bb2c02047de65f97c0f9fb474837f504e4ce7461ea","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-27T00:05:12Z","title_canon_sha256":"c0ce173197108a0968d0df3fc72cf4b275073e4243f8a54d822572d00c994d20"},"schema_version":"1.0","source":{"id":"2501.15708","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.15708","created_at":"2026-07-05T10:50:46Z"},{"alias_kind":"arxiv_version","alias_value":"2501.15708v3","created_at":"2026-07-05T10:50:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.15708","created_at":"2026-07-05T10:50:46Z"},{"alias_kind":"pith_short_12","alias_value":"MN5D2QMUOQV3","created_at":"2026-07-05T10:50:46Z"},{"alias_kind":"pith_short_16","alias_value":"MN5D2QMUOQV3AAD6","created_at":"2026-07-05T10:50:46Z"},{"alias_kind":"pith_short_8","alias_value":"MN5D2QMU","created_at":"2026-07-05T10:50:46Z"}],"graph_snapshots":[{"event_id":"sha256:61dd7faac60f17bed67b6b7c599106f8304ba3a09ef6e9f217aa84abea24e801","target":"graph","created_at":"2026-07-05T10:50:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.15708/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Classification tasks are widely investigated in the In-Context Learning (ICL) paradigm. However, current efforts are evaluated on disjoint benchmarks and settings, while their performances are significantly influenced by some trivial variables, such as prompt templates, data sampling, instructions, etc., which leads to significant inconsistencies in the results reported across various literature, preventing fair comparison or meta-analysis across different papers. Therefore, this paper proposes a standardized and easy-to-use evaluation toolkit (StaICC) for in-context classification. Including,","authors_text":"Hakaze Cho, Naoya Inoue","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-27T00:05:12Z","title":"StaICC: Standardized Evaluation for Classification Task in In-context Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.15708","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d00dd94af4958628c74cae474fb9c8f03e7d3dd0339a12d7f4193c4a3571ddf1","target":"record","created_at":"2026-07-05T10:50:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4528c82dff9aaa0944a525bb2c02047de65f97c0f9fb474837f504e4ce7461ea","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-27T00:05:12Z","title_canon_sha256":"c0ce173197108a0968d0df3fc72cf4b275073e4243f8a54d822572d00c994d20"},"schema_version":"1.0","source":{"id":"2501.15708","kind":"arxiv","version":3}},"canonical_sha256":"637a3d4194742bb0007ebf28bf6ca328acabf9e2e27cf50839f8ee174f54a11a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"637a3d4194742bb0007ebf28bf6ca328acabf9e2e27cf50839f8ee174f54a11a","first_computed_at":"2026-07-05T10:50:46.862445Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:50:46.862445Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gBOixmPyZ0+5OmiKB39Z4TraeOmEqsy+6AprdCXInd8eHnIaMJsMNi5xXpfiMoiUBzFOrLLMU9iCDw6FcQhDAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:50:46.862869Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.15708","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d00dd94af4958628c74cae474fb9c8f03e7d3dd0339a12d7f4193c4a3571ddf1","sha256:61dd7faac60f17bed67b6b7c599106f8304ba3a09ef6e9f217aa84abea24e801"],"state_sha256":"ea16f938f9685ec3afc80dbf7f4034b45d0e08705d22fd70191fbc2e45499317"}