{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:HRARFLBGOQ4NYFH54XGCNMMCDM","short_pith_number":"pith:HRARFLBG","canonical_record":{"source":{"id":"2411.07668","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-11-12T09:35:23Z","cross_cats_sorted":[],"title_canon_sha256":"d56380cd05e49718d0c2d48c4d0bd66703f8c4cce2144d57a817a57d6b9c6415","abstract_canon_sha256":"aa45ef442897a6f8eff0cdc9e91e133567f1bd1a927bac651f9dc3c2fbfa532f"},"schema_version":"1.0"},"canonical_sha256":"3c4112ac267438dc14fde5cc26b1821b021455cfaf393fd6c13ff6e399841e25","source":{"kind":"arxiv","id":"2411.07668","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.07668","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"arxiv_version","alias_value":"2411.07668v3","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.07668","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_12","alias_value":"HRARFLBGOQ4N","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_16","alias_value":"HRARFLBGOQ4NYFH5","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_8","alias_value":"HRARFLBG","created_at":"2026-07-05T10:09:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:HRARFLBGOQ4NYFH54XGCNMMCDM","target":"record","payload":{"canonical_record":{"source":{"id":"2411.07668","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-11-12T09:35:23Z","cross_cats_sorted":[],"title_canon_sha256":"d56380cd05e49718d0c2d48c4d0bd66703f8c4cce2144d57a817a57d6b9c6415","abstract_canon_sha256":"aa45ef442897a6f8eff0cdc9e91e133567f1bd1a927bac651f9dc3c2fbfa532f"},"schema_version":"1.0"},"canonical_sha256":"3c4112ac267438dc14fde5cc26b1821b021455cfaf393fd6c13ff6e399841e25","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:09:11.895771Z","signature_b64":"op2vn/xiSiWYBTviHXmDOy+2kr58mcBPTCerfvVnAp0paMc4BUs54UmgawuaeHSKNRhSkJWS9ZoI6fq5OgXhAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3c4112ac267438dc14fde5cc26b1821b021455cfaf393fd6c13ff6e399841e25","last_reissued_at":"2026-07-05T10:09:11.895299Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:09:11.895299Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.07668","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Gx8Zjp8dwHSDDYoC6c16yzDJj4VQnS/91+ewRxnCG1Qf1jADmWGZr84E6CUUfA5KJIIQgRs5Jq1k/rBTAhDwBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T11:22:03.228065Z"},"content_sha256":"72cc4ebf8e79304c653b7fc854a62acfaec0d9985d041998e9444051678573a3","schema_version":"1.0","event_id":"sha256:72cc4ebf8e79304c653b7fc854a62acfaec0d9985d041998e9444051678573a3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:HRARFLBGOQ4NYFH54XGCNMMCDM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards Evaluation Guidelines for Empirical Studies involving LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Davide Falessi, Marvin Mu\\~noz Bar\\'on, Sebastian Baltes, Stefan Wagner","submitted_at":"2024-11-12T09:35:23Z","abstract_excerpt":"In the short period since the release of ChatGPT, large language models (LLMs) have changed the software engineering research landscape. While there are numerous opportunities to use LLMs for supporting research or software engineering tasks, solid science needs rigorous empirical evaluations. However, so far, there are no specific guidelines for conducting and assessing studies involving LLMs in software engineering research. Our focus is on empirical studies that either use LLMs as part of the research process or studies that evaluate existing or new tools that are based on LLMs. This paper "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.07668","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.07668/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"exK9DLhrEI/Rj8tAw1pER9LOYEVchpKuc0NL2ipCm0Beo1PGyYEdt46utP7JpSkoj7quFRWmTyZmlkeWwyNbAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T11:22:03.228954Z"},"content_sha256":"0be3eb53223003b6860d5f41bac1ef72e9bf9cf172c6e8b04561ae3750c14f75","schema_version":"1.0","event_id":"sha256:0be3eb53223003b6860d5f41bac1ef72e9bf9cf172c6e8b04561ae3750c14f75"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/bundle.json","state_url":"https://pith.science/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T11:22:03Z","links":{"resolver":"https://pith.science/pith/HRARFLBGOQ4NYFH54XGCNMMCDM","bundle":"https://pith.science/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/bundle.json","state":"https://pith.science/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HRARFLBGOQ4NYFH54XGCNMMCDM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HRARFLBGOQ4NYFH54XGCNMMCDM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"aa45ef442897a6f8eff0cdc9e91e133567f1bd1a927bac651f9dc3c2fbfa532f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-11-12T09:35:23Z","title_canon_sha256":"d56380cd05e49718d0c2d48c4d0bd66703f8c4cce2144d57a817a57d6b9c6415"},"schema_version":"1.0","source":{"id":"2411.07668","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.07668","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"arxiv_version","alias_value":"2411.07668v3","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.07668","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_12","alias_value":"HRARFLBGOQ4N","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_16","alias_value":"HRARFLBGOQ4NYFH5","created_at":"2026-07-05T10:09:11Z"},{"alias_kind":"pith_short_8","alias_value":"HRARFLBG","created_at":"2026-07-05T10:09:11Z"}],"graph_snapshots":[{"event_id":"sha256:0be3eb53223003b6860d5f41bac1ef72e9bf9cf172c6e8b04561ae3750c14f75","target":"graph","created_at":"2026-07-05T10:09:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.07668/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In the short period since the release of ChatGPT, large language models (LLMs) have changed the software engineering research landscape. While there are numerous opportunities to use LLMs for supporting research or software engineering tasks, solid science needs rigorous empirical evaluations. However, so far, there are no specific guidelines for conducting and assessing studies involving LLMs in software engineering research. Our focus is on empirical studies that either use LLMs as part of the research process or studies that evaluate existing or new tools that are based on LLMs. This paper ","authors_text":"Davide Falessi, Marvin Mu\\~noz Bar\\'on, Sebastian Baltes, Stefan Wagner","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-11-12T09:35:23Z","title":"Towards Evaluation Guidelines for Empirical Studies involving LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.07668","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:72cc4ebf8e79304c653b7fc854a62acfaec0d9985d041998e9444051678573a3","target":"record","created_at":"2026-07-05T10:09:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"aa45ef442897a6f8eff0cdc9e91e133567f1bd1a927bac651f9dc3c2fbfa532f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-11-12T09:35:23Z","title_canon_sha256":"d56380cd05e49718d0c2d48c4d0bd66703f8c4cce2144d57a817a57d6b9c6415"},"schema_version":"1.0","source":{"id":"2411.07668","kind":"arxiv","version":3}},"canonical_sha256":"3c4112ac267438dc14fde5cc26b1821b021455cfaf393fd6c13ff6e399841e25","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3c4112ac267438dc14fde5cc26b1821b021455cfaf393fd6c13ff6e399841e25","first_computed_at":"2026-07-05T10:09:11.895299Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:09:11.895299Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"op2vn/xiSiWYBTviHXmDOy+2kr58mcBPTCerfvVnAp0paMc4BUs54UmgawuaeHSKNRhSkJWS9ZoI6fq5OgXhAw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:09:11.895771Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.07668","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:72cc4ebf8e79304c653b7fc854a62acfaec0d9985d041998e9444051678573a3","sha256:0be3eb53223003b6860d5f41bac1ef72e9bf9cf172c6e8b04561ae3750c14f75"],"state_sha256":"083cb1c9b06739dbc3dc9e21bf062b14b71189f2675a0be1e21ddf43dc3096da"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zUyYls1nOuNipFeobkx4WoymIdGgP5hEtjGACGgw8N3qM2z1Mu2j0L7ZDa0GBsZY+vWc0ymY3sAZFm2uks4UCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T11:22:03.285079Z","bundle_sha256":"311fb748a8fdab7f5fda7e722deec963deeab46b4065529a1fb9a5a7255dc9e8"}}