{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:Y53AQM7LTMTVDHXVL2XKILL5MU","short_pith_number":"pith:Y53AQM7L","canonical_record":{"source":{"id":"2210.12563","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-22T22:12:06Z","cross_cats_sorted":[],"title_canon_sha256":"1b3fa68ff6ac18be6364c3e46526707110ffee3c5284495d451bb9ca35958764","abstract_canon_sha256":"45ba60763be324de9ac43b7cbb3a139bc92c824dc8bd13ec55d6eb89fa311b80"},"schema_version":"1.0"},"canonical_sha256":"c7760833eb9b27519ef55eaea42d7d65362d80975ebf9ed8b41ba1bfc4580e65","source":{"kind":"arxiv","id":"2210.12563","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.12563","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"arxiv_version","alias_value":"2210.12563v1","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.12563","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_12","alias_value":"Y53AQM7LTMTV","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_16","alias_value":"Y53AQM7LTMTVDHXV","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_8","alias_value":"Y53AQM7L","created_at":"2026-07-05T05:09:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:Y53AQM7LTMTVDHXVL2XKILL5MU","target":"record","payload":{"canonical_record":{"source":{"id":"2210.12563","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-22T22:12:06Z","cross_cats_sorted":[],"title_canon_sha256":"1b3fa68ff6ac18be6364c3e46526707110ffee3c5284495d451bb9ca35958764","abstract_canon_sha256":"45ba60763be324de9ac43b7cbb3a139bc92c824dc8bd13ec55d6eb89fa311b80"},"schema_version":"1.0"},"canonical_sha256":"c7760833eb9b27519ef55eaea42d7d65362d80975ebf9ed8b41ba1bfc4580e65","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:09:29.853518Z","signature_b64":"qht4An45o6cYJhjyGVLAUmNBEmLQJimtUU/CrhepC/usfIsVTxX1T41J4PlwZqOQv7nU7Og06coD2fmF1liwCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c7760833eb9b27519ef55eaea42d7d65362d80975ebf9ed8b41ba1bfc4580e65","last_reissued_at":"2026-07-05T05:09:29.853034Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:09:29.853034Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.12563","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:09:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QohippXdnA+Y0sFUHeib4F+COCwpEVYzxStY8U2pOGB1eK3VJDwM61/82hrQOc6zDwYqsH8VnVFPGe4C+A0LDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:05:33.935567Z"},"content_sha256":"4aac3a621acdec0283ae32ba0832f1653d554bf383389b29a9a4eb8a2ddabfff","schema_version":"1.0","event_id":"sha256:4aac3a621acdec0283ae32ba0832f1653d554bf383389b29a9a4eb8a2ddabfff"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:Y53AQM7LTMTVDHXVL2XKILL5MU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Limitations of Reference-Free Evaluations of Generated Text","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Daniel Deutsch, Dan Roth, Rotem Dror","submitted_at":"2022-10-22T22:12:06Z","abstract_excerpt":"There is significant interest in developing evaluation metrics which accurately estimate the quality of generated text without the aid of a human-written reference text, which can be time consuming and expensive to collect or entirely unavailable in online applications. However, in this work, we demonstrate that these reference-free metrics are inherently biased and limited in their ability to evaluate generated text, and we argue that they should not be used to measure progress on tasks like machine translation or summarization. We show how reference-free metrics are equivalent to using one g"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.12563","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.12563/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:09:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"amIKWpMluhP2pHxzW1l4MazZqsgyQY1dZQ5gKlpFDVubss4n4Hutw3/GCM31nEI2oWegOp8XinSR1liQElt/Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:05:33.936525Z"},"content_sha256":"df6ea548676e7fcd94cee22000e93c504e402f34026fcb4ed81d0a7985e476b3","schema_version":"1.0","event_id":"sha256:df6ea548676e7fcd94cee22000e93c504e402f34026fcb4ed81d0a7985e476b3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/bundle.json","state_url":"https://pith.science/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T13:05:33Z","links":{"resolver":"https://pith.science/pith/Y53AQM7LTMTVDHXVL2XKILL5MU","bundle":"https://pith.science/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/bundle.json","state":"https://pith.science/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Y53AQM7LTMTVDHXVL2XKILL5MU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:Y53AQM7LTMTVDHXVL2XKILL5MU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"45ba60763be324de9ac43b7cbb3a139bc92c824dc8bd13ec55d6eb89fa311b80","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-22T22:12:06Z","title_canon_sha256":"1b3fa68ff6ac18be6364c3e46526707110ffee3c5284495d451bb9ca35958764"},"schema_version":"1.0","source":{"id":"2210.12563","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.12563","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"arxiv_version","alias_value":"2210.12563v1","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.12563","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_12","alias_value":"Y53AQM7LTMTV","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_16","alias_value":"Y53AQM7LTMTVDHXV","created_at":"2026-07-05T05:09:29Z"},{"alias_kind":"pith_short_8","alias_value":"Y53AQM7L","created_at":"2026-07-05T05:09:29Z"}],"graph_snapshots":[{"event_id":"sha256:df6ea548676e7fcd94cee22000e93c504e402f34026fcb4ed81d0a7985e476b3","target":"graph","created_at":"2026-07-05T05:09:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.12563/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"There is significant interest in developing evaluation metrics which accurately estimate the quality of generated text without the aid of a human-written reference text, which can be time consuming and expensive to collect or entirely unavailable in online applications. However, in this work, we demonstrate that these reference-free metrics are inherently biased and limited in their ability to evaluate generated text, and we argue that they should not be used to measure progress on tasks like machine translation or summarization. We show how reference-free metrics are equivalent to using one g","authors_text":"Daniel Deutsch, Dan Roth, Rotem Dror","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-22T22:12:06Z","title":"On the Limitations of Reference-Free Evaluations of Generated Text"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.12563","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4aac3a621acdec0283ae32ba0832f1653d554bf383389b29a9a4eb8a2ddabfff","target":"record","created_at":"2026-07-05T05:09:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"45ba60763be324de9ac43b7cbb3a139bc92c824dc8bd13ec55d6eb89fa311b80","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-22T22:12:06Z","title_canon_sha256":"1b3fa68ff6ac18be6364c3e46526707110ffee3c5284495d451bb9ca35958764"},"schema_version":"1.0","source":{"id":"2210.12563","kind":"arxiv","version":1}},"canonical_sha256":"c7760833eb9b27519ef55eaea42d7d65362d80975ebf9ed8b41ba1bfc4580e65","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c7760833eb9b27519ef55eaea42d7d65362d80975ebf9ed8b41ba1bfc4580e65","first_computed_at":"2026-07-05T05:09:29.853034Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:09:29.853034Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qht4An45o6cYJhjyGVLAUmNBEmLQJimtUU/CrhepC/usfIsVTxX1T41J4PlwZqOQv7nU7Og06coD2fmF1liwCA==","signature_status":"signed_v1","signed_at":"2026-07-05T05:09:29.853518Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.12563","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4aac3a621acdec0283ae32ba0832f1653d554bf383389b29a9a4eb8a2ddabfff","sha256:df6ea548676e7fcd94cee22000e93c504e402f34026fcb4ed81d0a7985e476b3"],"state_sha256":"1e14b68512d3a89c1eae9cd7629691d0dd425436bafec679f12b88de5089a182"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6YN5xxKJ5c9xwljQ1EW82EVreBszU5TYemrgoPxlpbdQOutX4W32/TUX2KpA48gIQrMZzNq8R1Y3Dt/wEqW8Aw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T13:05:33.943225Z","bundle_sha256":"2264037dfb510cbad7f67aa06f8d02b535bbb0db2b1eb23df001444a77def3f7"}}