{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:MN5D2QMUOQV3AAD6X4UL63FDFC","short_pith_number":"pith:MN5D2QMU","schema_version":"1.0","canonical_sha256":"637a3d4194742bb0007ebf28bf6ca328acabf9e2e27cf50839f8ee174f54a11a","source":{"kind":"arxiv","id":"2501.15708","version":3},"attestation_state":"computed","paper":{"title":"StaICC: Standardized Evaluation for Classification Task in In-context Learning","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hakaze Cho, Naoya Inoue","submitted_at":"2025-01-27T00:05:12Z","abstract_excerpt":"Classification tasks are widely investigated in the In-Context Learning (ICL) paradigm. However, current efforts are evaluated on disjoint benchmarks and settings, while their performances are significantly influenced by some trivial variables, such as prompt templates, data sampling, instructions, etc., which leads to significant inconsistencies in the results reported across various literature, preventing fair comparison or meta-analysis across different papers. Therefore, this paper proposes a standardized and easy-to-use evaluation toolkit (StaICC) for in-context classification. Including,"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2501.15708","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-27T00:05:12Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c0ce173197108a0968d0df3fc72cf4b275073e4243f8a54d822572d00c994d20","abstract_canon_sha256":"4528c82dff9aaa0944a525bb2c02047de65f97c0f9fb474837f504e4ce7461ea"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:50:46.862869Z","signature_b64":"gBOixmPyZ0+5OmiKB39Z4TraeOmEqsy+6AprdCXInd8eHnIaMJsMNi5xXpfiMoiUBzFOrLLMU9iCDw6FcQhDAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"637a3d4194742bb0007ebf28bf6ca328acabf9e2e27cf50839f8ee174f54a11a","last_reissued_at":"2026-07-05T10:50:46.862445Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:50:46.862445Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"StaICC: Standardized Evaluation for Classification Task in In-context Learning","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hakaze Cho, Naoya Inoue","submitted_at":"2025-01-27T00:05:12Z","abstract_excerpt":"Classification tasks are widely investigated in the In-Context Learning (ICL) paradigm. However, current efforts are evaluated on disjoint benchmarks and settings, while their performances are significantly influenced by some trivial variables, such as prompt templates, data sampling, instructions, etc., which leads to significant inconsistencies in the results reported across various literature, preventing fair comparison or meta-analysis across different papers. Therefore, this paper proposes a standardized and easy-to-use evaluation toolkit (StaICC) for in-context classification. Including,"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.15708","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.15708/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2501.15708","created_at":"2026-07-05T10:50:46.862504+00:00"},{"alias_kind":"arxiv_version","alias_value":"2501.15708v3","created_at":"2026-07-05T10:50:46.862504+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.15708","created_at":"2026-07-05T10:50:46.862504+00:00"},{"alias_kind":"pith_short_12","alias_value":"MN5D2QMUOQV3","created_at":"2026-07-05T10:50:46.862504+00:00"},{"alias_kind":"pith_short_16","alias_value":"MN5D2QMUOQV3AAD6","created_at":"2026-07-05T10:50:46.862504+00:00"},{"alias_kind":"pith_short_8","alias_value":"MN5D2QMU","created_at":"2026-07-05T10:50:46.862504+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC","json":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC.json","graph_json":"https://pith.science/api/pith-number/MN5D2QMUOQV3AAD6X4UL63FDFC/graph.json","events_json":"https://pith.science/api/pith-number/MN5D2QMUOQV3AAD6X4UL63FDFC/events.json","paper":"https://pith.science/paper/MN5D2QMU"},"agent_actions":{"view_html":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC","download_json":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC.json","view_paper":"https://pith.science/paper/MN5D2QMU","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2501.15708&json=true","fetch_graph":"https://pith.science/api/pith-number/MN5D2QMUOQV3AAD6X4UL63FDFC/graph.json","fetch_events":"https://pith.science/api/pith-number/MN5D2QMUOQV3AAD6X4UL63FDFC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC/action/storage_attestation","attest_author":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC/action/author_attestation","sign_citation":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC/action/citation_signature","submit_replication":"https://pith.science/pith/MN5D2QMUOQV3AAD6X4UL63FDFC/action/replication_record"}},"created_at":"2026-07-05T10:50:46.862504+00:00","updated_at":"2026-07-05T10:50:46.862504+00:00"}