From 4ae47fa7e8f95be17d4ff9c317a1193bbb4a3998 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Paluri Date: Mon, 14 Oct 2024 08:09:47 -0500 Subject: [PATCH 01/12] x86/virt: Move SEV-specific parsing into arch/x86/virt/svm Move SEV-specific kernel command line option parsing support from arch/x86/coco/sev/core.c to arch/x86/virt/svm/cmdline.c so that both host and guest related SEV command line options can be supported. No functional changes intended. Signed-off-by: Pavan Kumar Paluri Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241014130948.1476946-2-papaluri@amd.com --- arch/x86/coco/sev/core.c | 44 ------------------------------- arch/x86/include/asm/sev-common.h | 27 +++++++++++++++++++ arch/x86/virt/svm/Makefile | 1 + arch/x86/virt/svm/cmdline.c | 33 +++++++++++++++++++++++ 4 files changed, 61 insertions(+), 44 deletions(-) create mode 100644 arch/x86/virt/svm/cmdline.c diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index de1df0cb45da..ff19e805e7a1 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -141,33 +141,6 @@ static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); static DEFINE_PER_CPU(u64, svsm_caa_pa); -struct sev_config { - __u64 debug : 1, - - /* - * Indicates when the per-CPU GHCB has been created and registered - * and thus can be used by the BSP instead of the early boot GHCB. - * - * For APs, the per-CPU GHCB is created before they are started - * and registered upon startup, so this flag can be used globally - * for the BSP and APs. - */ - ghcbs_initialized : 1, - - /* - * Indicates when the per-CPU SVSM CA is to be used instead of the - * boot SVSM CA. - * - * For APs, the per-CPU SVSM CA is created as part of the AP - * bringup, so this flag can be used globally for the BSP and APs. - */ - use_cas : 1, - - __reserved : 61; -}; - -static struct sev_config sev_cfg __read_mostly; - static __always_inline bool on_vc_stack(struct pt_regs *regs) { unsigned long sp = regs->sp; @@ -2374,23 +2347,6 @@ static int __init report_snp_info(void) } arch_initcall(report_snp_info); -static int __init init_sev_config(char *str) -{ - char *s; - - while ((s = strsep(&str, ","))) { - if (!strcmp(s, "debug")) { - sev_cfg.debug = true; - continue; - } - - pr_info("SEV command-line option '%s' was not recognized\n", s); - } - - return 1; -} -__setup("sev=", init_sev_config); - static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input) { /* If (new) lengths have been returned, propagate them up */ diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h index 98726c2b04f8..50f5666938c0 100644 --- a/arch/x86/include/asm/sev-common.h +++ b/arch/x86/include/asm/sev-common.h @@ -220,4 +220,31 @@ struct snp_psc_desc { #define GHCB_ERR_INVALID_INPUT 5 #define GHCB_ERR_INVALID_EVENT 6 +struct sev_config { + __u64 debug : 1, + + /* + * Indicates when the per-CPU GHCB has been created and registered + * and thus can be used by the BSP instead of the early boot GHCB. + * + * For APs, the per-CPU GHCB is created before they are started + * and registered upon startup, so this flag can be used globally + * for the BSP and APs. + */ + ghcbs_initialized : 1, + + /* + * Indicates when the per-CPU SVSM CA is to be used instead of the + * boot SVSM CA. + * + * For APs, the per-CPU SVSM CA is created as part of the AP + * bringup, so this flag can be used globally for the BSP and APs. + */ + use_cas : 1, + + __reserved : 61; +}; + +extern struct sev_config sev_cfg; + #endif diff --git a/arch/x86/virt/svm/Makefile b/arch/x86/virt/svm/Makefile index ef2a31bdcc70..eca6d71355fa 100644 --- a/arch/x86/virt/svm/Makefile +++ b/arch/x86/virt/svm/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_KVM_AMD_SEV) += sev.o +obj-$(CONFIG_CPU_SUP_AMD) += cmdline.o diff --git a/arch/x86/virt/svm/cmdline.c b/arch/x86/virt/svm/cmdline.c new file mode 100644 index 000000000000..add4bae3ebef --- /dev/null +++ b/arch/x86/virt/svm/cmdline.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD SVM-SEV command line parsing support + * + * Copyright (C) 2023 - 2024 Advanced Micro Devices, Inc. + * + * Author: Michael Roth + */ + +#include +#include +#include + +#include + +struct sev_config sev_cfg __read_mostly; + +static int __init init_sev_config(char *str) +{ + char *s; + + while ((s = strsep(&str, ","))) { + if (!strcmp(s, "debug")) { + sev_cfg.debug = true; + continue; + } + + pr_info("SEV command-line option '%s' was not recognized\n", s); + } + + return 1; +} +__setup("sev=", init_sev_config); From 2db67aaca578ec4998b78dc85e2af214bc2e2770 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Paluri Date: Mon, 14 Oct 2024 08:09:48 -0500 Subject: [PATCH 02/12] x86/virt: Provide "nosnp" boot option for sev kernel command line Provide a "nosnp" kernel command line option to prevent enabling of the RMP and SEV-SNP features in the host/hypervisor. Not initializing the RMP removes system overhead associated with RMP checks. [ bp: Actually make it a HV-only cmdline option. ] Co-developed-by: Eric Van Tassell Signed-off-by: Eric Van Tassell Signed-off-by: Pavan Kumar Paluri Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241014130948.1476946-3-papaluri@amd.com --- Documentation/arch/x86/x86_64/boot-options.rst | 5 +++++ arch/x86/virt/svm/cmdline.c | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Documentation/arch/x86/x86_64/boot-options.rst b/Documentation/arch/x86/x86_64/boot-options.rst index 98d4805f0823..d69e3cfbdba5 100644 --- a/Documentation/arch/x86/x86_64/boot-options.rst +++ b/Documentation/arch/x86/x86_64/boot-options.rst @@ -305,3 +305,8 @@ The available options are: debug Enable debug messages. + + nosnp + Do not enable SEV-SNP (applies to host/hypervisor only). Setting + 'nosnp' avoids the RMP check overhead in memory accesses when + users do not want to run SEV-SNP guests. diff --git a/arch/x86/virt/svm/cmdline.c b/arch/x86/virt/svm/cmdline.c index add4bae3ebef..affa2759fa20 100644 --- a/arch/x86/virt/svm/cmdline.c +++ b/arch/x86/virt/svm/cmdline.c @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -25,6 +26,17 @@ static int __init init_sev_config(char *str) continue; } + if (!strcmp(s, "nosnp")) { + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) { + setup_clear_cpu_cap(X86_FEATURE_SEV_SNP); + cc_platform_clear(CC_ATTR_HOST_SEV_SNP); + continue; + } else { + goto warn; + } + } + +warn: pr_info("SEV command-line option '%s' was not recognized\n", s); } From f3476bc77057db0adf90c0a141a3599dd11c56a0 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:32 +0530 Subject: [PATCH 03/12] virt: sev-guest: Use AES GCM crypto library The sev-guest driver encryption code uses the crypto API for SNP guest messaging with the AMD Security processor. In order to enable secure TSC, SEV-SNP guests need to send such a TSC_INFO message before the APs are booted. Details from the TSC_INFO response will then be used to program the VMSA before the APs are brought up. However, the crypto API is not available this early in the boot process. In preparation for moving the encryption code out of sev-guest to support secure TSC and to ease review, switch to using the AES GCM library implementation instead. Drop __enc_payload() and dec_payload() helpers as both are small and can be moved to the respective callers. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Acked-by: Borislav Petkov (AMD) Tested-by: Peter Gonda Link: https://lore.kernel.org/r/20241009092850.197575-2-nikunj@amd.com --- arch/x86/include/asm/sev.h | 3 + drivers/virt/coco/sev-guest/Kconfig | 4 +- drivers/virt/coco/sev-guest/sev-guest.c | 175 ++++++------------------ 3 files changed, 43 insertions(+), 139 deletions(-) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index ee34ab00a8d6..e7977f76d77e 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -120,6 +120,9 @@ struct snp_req_data { }; #define MAX_AUTHTAG_LEN 32 +#define AUTHTAG_LEN 16 +#define AAD_LEN 48 +#define MSG_HDR_VER 1 /* See SNP spec SNP_GUEST_REQUEST section for the structure */ enum msg_type { diff --git a/drivers/virt/coco/sev-guest/Kconfig b/drivers/virt/coco/sev-guest/Kconfig index 1cffc72c41cb..0b772bd921d8 100644 --- a/drivers/virt/coco/sev-guest/Kconfig +++ b/drivers/virt/coco/sev-guest/Kconfig @@ -2,9 +2,7 @@ config SEV_GUEST tristate "AMD SEV Guest driver" default m depends on AMD_MEM_ENCRYPT - select CRYPTO - select CRYPTO_AEAD2 - select CRYPTO_GCM + select CRYPTO_LIB_AESGCM select TSM_REPORTS help SEV-SNP firmware provides the guest a mechanism to communicate with diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index 89754b019be2..a33daff516ed 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -17,8 +17,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -31,26 +30,18 @@ #include #define DEVICE_NAME "sev-guest" -#define AAD_LEN 48 -#define MSG_HDR_VER 1 #define SNP_REQ_MAX_RETRY_DURATION (60*HZ) #define SNP_REQ_RETRY_DELAY (2*HZ) #define SVSM_MAX_RETRIES 3 -struct snp_guest_crypto { - struct crypto_aead *tfm; - u8 *iv, *authtag; - int iv_len, a_len; -}; - struct snp_guest_dev { struct device *dev; struct miscdevice misc; void *certs_data; - struct snp_guest_crypto *crypto; + struct aesgcm_ctx *ctx; /* request and response are in unencrypted memory */ struct snp_guest_msg *request, *response; @@ -169,132 +160,31 @@ static inline struct snp_guest_dev *to_snp_dev(struct file *file) return container_of(dev, struct snp_guest_dev, misc); } -static struct snp_guest_crypto *init_crypto(struct snp_guest_dev *snp_dev, u8 *key, size_t keylen) +static struct aesgcm_ctx *snp_init_crypto(u8 *key, size_t keylen) { - struct snp_guest_crypto *crypto; + struct aesgcm_ctx *ctx; - crypto = kzalloc(sizeof(*crypto), GFP_KERNEL_ACCOUNT); - if (!crypto) + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); + if (!ctx) return NULL; - crypto->tfm = crypto_alloc_aead("gcm(aes)", 0, 0); - if (IS_ERR(crypto->tfm)) - goto e_free; - - if (crypto_aead_setkey(crypto->tfm, key, keylen)) - goto e_free_crypto; - - crypto->iv_len = crypto_aead_ivsize(crypto->tfm); - crypto->iv = kmalloc(crypto->iv_len, GFP_KERNEL_ACCOUNT); - if (!crypto->iv) - goto e_free_crypto; - - if (crypto_aead_authsize(crypto->tfm) > MAX_AUTHTAG_LEN) { - if (crypto_aead_setauthsize(crypto->tfm, MAX_AUTHTAG_LEN)) { - dev_err(snp_dev->dev, "failed to set authsize to %d\n", MAX_AUTHTAG_LEN); - goto e_free_iv; - } + if (aesgcm_expandkey(ctx, key, keylen, AUTHTAG_LEN)) { + pr_err("Crypto context initialization failed\n"); + kfree(ctx); + return NULL; } - crypto->a_len = crypto_aead_authsize(crypto->tfm); - crypto->authtag = kmalloc(crypto->a_len, GFP_KERNEL_ACCOUNT); - if (!crypto->authtag) - goto e_free_iv; - - return crypto; - -e_free_iv: - kfree(crypto->iv); -e_free_crypto: - crypto_free_aead(crypto->tfm); -e_free: - kfree(crypto); - - return NULL; -} - -static void deinit_crypto(struct snp_guest_crypto *crypto) -{ - crypto_free_aead(crypto->tfm); - kfree(crypto->iv); - kfree(crypto->authtag); - kfree(crypto); -} - -static int enc_dec_message(struct snp_guest_crypto *crypto, struct snp_guest_msg *msg, - u8 *src_buf, u8 *dst_buf, size_t len, bool enc) -{ - struct snp_guest_msg_hdr *hdr = &msg->hdr; - struct scatterlist src[3], dst[3]; - DECLARE_CRYPTO_WAIT(wait); - struct aead_request *req; - int ret; - - req = aead_request_alloc(crypto->tfm, GFP_KERNEL); - if (!req) - return -ENOMEM; - - /* - * AEAD memory operations: - * +------ AAD -------+------- DATA -----+---- AUTHTAG----+ - * | msg header | plaintext | hdr->authtag | - * | bytes 30h - 5Fh | or | | - * | | cipher | | - * +------------------+------------------+----------------+ - */ - sg_init_table(src, 3); - sg_set_buf(&src[0], &hdr->algo, AAD_LEN); - sg_set_buf(&src[1], src_buf, hdr->msg_sz); - sg_set_buf(&src[2], hdr->authtag, crypto->a_len); - - sg_init_table(dst, 3); - sg_set_buf(&dst[0], &hdr->algo, AAD_LEN); - sg_set_buf(&dst[1], dst_buf, hdr->msg_sz); - sg_set_buf(&dst[2], hdr->authtag, crypto->a_len); - - aead_request_set_ad(req, AAD_LEN); - aead_request_set_tfm(req, crypto->tfm); - aead_request_set_callback(req, 0, crypto_req_done, &wait); - - aead_request_set_crypt(req, src, dst, len, crypto->iv); - ret = crypto_wait_req(enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req), &wait); - - aead_request_free(req); - return ret; -} - -static int __enc_payload(struct snp_guest_dev *snp_dev, struct snp_guest_msg *msg, - void *plaintext, size_t len) -{ - struct snp_guest_crypto *crypto = snp_dev->crypto; - struct snp_guest_msg_hdr *hdr = &msg->hdr; - - memset(crypto->iv, 0, crypto->iv_len); - memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); - - return enc_dec_message(crypto, msg, plaintext, msg->payload, len, true); -} - -static int dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_msg *msg, - void *plaintext, size_t len) -{ - struct snp_guest_crypto *crypto = snp_dev->crypto; - struct snp_guest_msg_hdr *hdr = &msg->hdr; - - /* Build IV with response buffer sequence number */ - memset(crypto->iv, 0, crypto->iv_len); - memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); - - return enc_dec_message(crypto, msg, msg->payload, plaintext, len, false); + return ctx; } static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, void *payload, u32 sz) { - struct snp_guest_crypto *crypto = snp_dev->crypto; struct snp_guest_msg *resp_msg = &snp_dev->secret_response; struct snp_guest_msg *req_msg = &snp_dev->secret_request; struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr; struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr; + struct aesgcm_ctx *ctx = snp_dev->ctx; + u8 iv[GCM_AES_IV_SIZE] = {}; pr_debug("response [seqno %lld type %d version %d sz %d]\n", resp_msg_hdr->msg_seqno, resp_msg_hdr->msg_type, resp_msg_hdr->msg_version, @@ -316,11 +206,16 @@ static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, void *payload, * If the message size is greater than our buffer length then return * an error. */ - if (unlikely((resp_msg_hdr->msg_sz + crypto->a_len) > sz)) + if (unlikely((resp_msg_hdr->msg_sz + ctx->authsize) > sz)) return -EBADMSG; /* Decrypt the payload */ - return dec_payload(snp_dev, resp_msg, payload, resp_msg_hdr->msg_sz + crypto->a_len); + memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno))); + if (!aesgcm_decrypt(ctx, payload, resp_msg->payload, resp_msg_hdr->msg_sz, + &resp_msg_hdr->algo, AAD_LEN, iv, resp_msg_hdr->authtag)) + return -EBADMSG; + + return 0; } static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 type, @@ -328,6 +223,8 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 { struct snp_guest_msg *msg = &snp_dev->secret_request; struct snp_guest_msg_hdr *hdr = &msg->hdr; + struct aesgcm_ctx *ctx = snp_dev->ctx; + u8 iv[GCM_AES_IV_SIZE] = {}; memset(msg, 0, sizeof(*msg)); @@ -347,7 +244,14 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 pr_debug("request [seqno %lld type %d version %d sz %d]\n", hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); - return __enc_payload(snp_dev, msg, payload, sz); + if (WARN_ON((sz + ctx->authsize) > sizeof(msg->payload))) + return -EBADMSG; + + memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno))); + aesgcm_encrypt(ctx, msg->payload, payload, sz, &hdr->algo, AAD_LEN, + iv, hdr->authtag); + + return 0; } static int __handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, @@ -495,7 +399,6 @@ struct snp_req_resp { static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg) { - struct snp_guest_crypto *crypto = snp_dev->crypto; struct snp_report_req *report_req = &snp_dev->req.report; struct snp_report_resp *report_resp; int rc, resp_len; @@ -513,7 +416,7 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + crypto->a_len; + resp_len = sizeof(report_resp->data) + snp_dev->ctx->authsize; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) return -ENOMEM; @@ -534,7 +437,6 @@ e_free: static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg) { struct snp_derived_key_req *derived_key_req = &snp_dev->req.derived_key; - struct snp_guest_crypto *crypto = snp_dev->crypto; struct snp_derived_key_resp derived_key_resp = {0}; int rc, resp_len; /* Response data is 64 bytes and max authsize for GCM is 16 bytes. */ @@ -550,7 +452,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(derived_key_resp.data) + crypto->a_len; + resp_len = sizeof(derived_key_resp.data) + snp_dev->ctx->authsize; if (sizeof(buf) < resp_len) return -ENOMEM; @@ -579,7 +481,6 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques { struct snp_ext_report_req *report_req = &snp_dev->req.ext_report; - struct snp_guest_crypto *crypto = snp_dev->crypto; struct snp_report_resp *report_resp; int ret, npages = 0, resp_len; sockptr_t certs_address; @@ -622,7 +523,7 @@ cmd: * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + crypto->a_len; + resp_len = sizeof(report_resp->data) + snp_dev->ctx->authsize; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) return -ENOMEM; @@ -1147,8 +1048,8 @@ static int __init sev_guest_probe(struct platform_device *pdev) goto e_free_response; ret = -EIO; - snp_dev->crypto = init_crypto(snp_dev, snp_dev->vmpck, VMPCK_KEY_LEN); - if (!snp_dev->crypto) + snp_dev->ctx = snp_init_crypto(snp_dev->vmpck, VMPCK_KEY_LEN); + if (!snp_dev->ctx) goto e_free_cert_data; misc = &snp_dev->misc; @@ -1174,11 +1075,13 @@ static int __init sev_guest_probe(struct platform_device *pdev) ret = misc_register(misc); if (ret) - goto e_free_cert_data; + goto e_free_ctx; dev_info(dev, "Initialized SEV guest driver (using VMPCK%d communication key)\n", vmpck_id); return 0; +e_free_ctx: + kfree(snp_dev->ctx); e_free_cert_data: free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); e_free_response: @@ -1197,7 +1100,7 @@ static void __exit sev_guest_remove(struct platform_device *pdev) free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg)); free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg)); - deinit_crypto(snp_dev->crypto); + kfree(snp_dev->ctx); misc_deregister(&snp_dev->misc); } From f75ff17fb48b1991d7a2822de5acc12bba240dc1 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:33 +0530 Subject: [PATCH 04/12] x86/sev: Handle failures from snp_init() Address the ignored failures from snp_init() in sme_enable(). Add error handling for scenarios where snp_init() fails to retrieve the SEV-SNP CC blob or encounters issues while parsing the CC blob. Ensure that SNP guests will error out early, preventing delayed error reporting or undefined behavior. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241009092850.197575-3-nikunj@amd.com --- arch/x86/mm/mem_encrypt_identity.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index ac33b2263a43..e6c7686f443a 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -495,10 +495,10 @@ void __head sme_enable(struct boot_params *bp) unsigned int eax, ebx, ecx, edx; unsigned long feature_mask; unsigned long me_mask; - bool snp; + bool snp_en; u64 msr; - snp = snp_init(bp); + snp_en = snp_init(bp); /* Check for the SME/SEV support leaf */ eax = 0x80000000; @@ -531,8 +531,11 @@ void __head sme_enable(struct boot_params *bp) RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV); feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT; - /* The SEV-SNP CC blob should never be present unless SEV-SNP is enabled. */ - if (snp && !(msr & MSR_AMD64_SEV_SNP_ENABLED)) + /* + * Any discrepancies between the presence of a CC blob and SNP + * enablement abort the guest. + */ + if (snp_en ^ !!(msr & MSR_AMD64_SEV_SNP_ENABLED)) snp_abort(); /* Check if memory encryption is enabled */ From 6068754a4fff67654e87b37cdecd5275a372110f Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:34 +0530 Subject: [PATCH 05/12] x86/sev: Cache the secrets page address Instead of calling get_secrets_page(), which parses the CC blob every time to get the secrets page physical address (secrets_pa), save the secrets page physical address during snp_init() from the CC blob. Since get_secrets_page() is no longer used, remove the function. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241009092850.197575-4-nikunj@amd.com --- arch/x86/coco/sev/core.c | 51 +++++++++------------------------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index ff19e805e7a1..af50a3846e53 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -92,6 +92,9 @@ static struct ghcb *boot_ghcb __section(".data"); /* Bitmap of SEV features supported by the hypervisor */ static u64 sev_hv_features __ro_after_init; +/* Secrets page physical address from the CC blob */ +static u64 secrets_pa __ro_after_init; + /* #VC handler runtime per-CPU data */ struct sev_es_runtime_data { struct ghcb ghcb_page; @@ -695,45 +698,13 @@ void noinstr __sev_es_nmi_complete(void) __sev_put_ghcb(&state); } -static u64 __init get_secrets_page(void) -{ - u64 pa_data = boot_params.cc_blob_address; - struct cc_blob_sev_info info; - void *map; - - /* - * The CC blob contains the address of the secrets page, check if the - * blob is present. - */ - if (!pa_data) - return 0; - - map = early_memremap(pa_data, sizeof(info)); - if (!map) { - pr_err("Unable to locate SNP secrets page: failed to map the Confidential Computing blob.\n"); - return 0; - } - memcpy(&info, map, sizeof(info)); - early_memunmap(map, sizeof(info)); - - /* smoke-test the secrets page passed */ - if (!info.secrets_phys || info.secrets_len != PAGE_SIZE) - return 0; - - return info.secrets_phys; -} - static u64 __init get_snp_jump_table_addr(void) { struct snp_secrets_page *secrets; void __iomem *mem; - u64 pa, addr; + u64 addr; - pa = get_secrets_page(); - if (!pa) - return 0; - - mem = ioremap_encrypted(pa, PAGE_SIZE); + mem = ioremap_encrypted(secrets_pa, PAGE_SIZE); if (!mem) { pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n"); return 0; @@ -2273,6 +2244,11 @@ bool __head snp_init(struct boot_params *bp) if (!cc_info) return false; + if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE) + secrets_pa = cc_info->secrets_phys; + else + return false; + setup_cpuid_table(cc_info); svsm_setup(cc_info); @@ -2469,16 +2445,11 @@ static struct platform_device sev_guest_device = { static int __init snp_init_platform_device(void) { struct sev_guest_platform_data data; - u64 gpa; if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) return -ENODEV; - gpa = get_secrets_page(); - if (!gpa) - return -ENODEV; - - data.secrets_gpa = gpa; + data.secrets_gpa = secrets_pa; if (platform_device_add_data(&sev_guest_device, &data, sizeof(data))) return -ENODEV; From 999d73686ba1c0700aba4ac0fe86e26f759468a9 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:35 +0530 Subject: [PATCH 06/12] virt: sev-guest: Consolidate SNP guest messaging parameters to a struct Add a snp_guest_req structure to eliminate the need to pass a long list of parameters. This structure will be used to call the SNP Guest message request API, simplifying the function arguments. Update the snp_issue_guest_request() prototype to include the new guest request structure. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241009092850.197575-5-nikunj@amd.com --- arch/x86/coco/sev/core.c | 9 +-- arch/x86/include/asm/sev.h | 19 +++++- drivers/virt/coco/sev-guest/sev-guest.c | 84 ++++++++++++++++--------- 3 files changed, 76 insertions(+), 36 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index af50a3846e53..c7b4270d0e18 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -2373,7 +2373,8 @@ int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, } EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req); -int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio) +int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, + struct snp_guest_request_ioctl *rio) { struct ghcb_state state; struct es_em_ctxt ctxt; @@ -2397,12 +2398,12 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct sn vc_ghcb_invalidate(ghcb); - if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { + if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { ghcb_set_rax(ghcb, input->data_gpa); ghcb_set_rbx(ghcb, input->data_npages); } - ret = sev_es_ghcb_hv_call(ghcb, &ctxt, exit_code, input->req_gpa, input->resp_gpa); + ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa); if (ret) goto e_put; @@ -2417,7 +2418,7 @@ int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct sn case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN): /* Number of expected pages are returned in RBX */ - if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { + if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { input->data_npages = ghcb_get_rbx(ghcb); ret = -ENOSPC; break; diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index e7977f76d77e..27fa1c9c3465 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -174,6 +174,19 @@ struct sev_guest_platform_data { u64 secrets_gpa; }; +struct snp_guest_req { + void *req_buf; + size_t req_sz; + + void *resp_buf; + size_t resp_sz; + + u64 exit_code; + unsigned int vmpck_id; + u8 msg_version; + u8 msg_type; +}; + /* * The secrets page contains 96-bytes of reserved field that can be used by * the guest OS. The guest OS uses the area to save the message sequence @@ -395,7 +408,8 @@ void snp_set_wakeup_secondary_cpu(void); bool snp_init(struct boot_params *bp); void __noreturn snp_abort(void); void snp_dmi_setup(void); -int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio); +int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, + struct snp_guest_request_ioctl *rio); int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input); void snp_accept_memory(phys_addr_t start, phys_addr_t end); u64 snp_get_unsupported_features(u64 status); @@ -425,7 +439,8 @@ static inline void snp_set_wakeup_secondary_cpu(void) { } static inline bool snp_init(struct boot_params *bp) { return false; } static inline void snp_abort(void) { } static inline void snp_dmi_setup(void) { } -static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio) +static inline int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, + struct snp_guest_request_ioctl *rio) { return -ENOTTY; } diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index a33daff516ed..2a1b542168b1 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -177,7 +177,7 @@ static struct aesgcm_ctx *snp_init_crypto(u8 *key, size_t keylen) return ctx; } -static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, void *payload, u32 sz) +static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_req *req) { struct snp_guest_msg *resp_msg = &snp_dev->secret_response; struct snp_guest_msg *req_msg = &snp_dev->secret_request; @@ -206,20 +206,19 @@ static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, void *payload, * If the message size is greater than our buffer length then return * an error. */ - if (unlikely((resp_msg_hdr->msg_sz + ctx->authsize) > sz)) + if (unlikely((resp_msg_hdr->msg_sz + ctx->authsize) > req->resp_sz)) return -EBADMSG; /* Decrypt the payload */ memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno))); - if (!aesgcm_decrypt(ctx, payload, resp_msg->payload, resp_msg_hdr->msg_sz, + if (!aesgcm_decrypt(ctx, req->resp_buf, resp_msg->payload, resp_msg_hdr->msg_sz, &resp_msg_hdr->algo, AAD_LEN, iv, resp_msg_hdr->authtag)) return -EBADMSG; return 0; } -static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 type, - void *payload, size_t sz) +static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, struct snp_guest_req *req) { struct snp_guest_msg *msg = &snp_dev->secret_request; struct snp_guest_msg_hdr *hdr = &msg->hdr; @@ -231,11 +230,11 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 hdr->algo = SNP_AEAD_AES_256_GCM; hdr->hdr_version = MSG_HDR_VER; hdr->hdr_sz = sizeof(*hdr); - hdr->msg_type = type; - hdr->msg_version = version; + hdr->msg_type = req->msg_type; + hdr->msg_version = req->msg_version; hdr->msg_seqno = seqno; - hdr->msg_vmpck = vmpck_id; - hdr->msg_sz = sz; + hdr->msg_vmpck = req->vmpck_id; + hdr->msg_sz = req->req_sz; /* Verify the sequence number is non-zero */ if (!hdr->msg_seqno) @@ -244,17 +243,17 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 pr_debug("request [seqno %lld type %d version %d sz %d]\n", hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); - if (WARN_ON((sz + ctx->authsize) > sizeof(msg->payload))) + if (WARN_ON((req->req_sz + ctx->authsize) > sizeof(msg->payload))) return -EBADMSG; memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno))); - aesgcm_encrypt(ctx, msg->payload, payload, sz, &hdr->algo, AAD_LEN, - iv, hdr->authtag); + aesgcm_encrypt(ctx, msg->payload, req->req_buf, req->req_sz, &hdr->algo, + AAD_LEN, iv, hdr->authtag); return 0; } -static int __handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, +static int __handle_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, struct snp_guest_request_ioctl *rio) { unsigned long req_start = jiffies; @@ -269,7 +268,7 @@ retry_request: * sequence number must be incremented or the VMPCK must be deleted to * prevent reuse of the IV. */ - rc = snp_issue_guest_request(exit_code, &snp_dev->input, rio); + rc = snp_issue_guest_request(req, &snp_dev->input, rio); switch (rc) { case -ENOSPC: /* @@ -280,7 +279,7 @@ retry_request: * IV reuse. */ override_npages = snp_dev->input.data_npages; - exit_code = SVM_VMGEXIT_GUEST_REQUEST; + req->exit_code = SVM_VMGEXIT_GUEST_REQUEST; /* * Override the error to inform callers the given extended @@ -340,10 +339,8 @@ retry_request: return rc; } -static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, - struct snp_guest_request_ioctl *rio, u8 type, - void *req_buf, size_t req_sz, void *resp_buf, - u32 resp_sz) +static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, + struct snp_guest_request_ioctl *rio) { u64 seqno; int rc; @@ -357,7 +354,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, memset(snp_dev->response, 0, sizeof(struct snp_guest_msg)); /* Encrypt the userspace provided payload in snp_dev->secret_request. */ - rc = enc_payload(snp_dev, seqno, rio->msg_version, type, req_buf, req_sz); + rc = enc_payload(snp_dev, seqno, req); if (rc) return rc; @@ -368,7 +365,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, memcpy(snp_dev->request, &snp_dev->secret_request, sizeof(snp_dev->secret_request)); - rc = __handle_guest_request(snp_dev, exit_code, rio); + rc = __handle_guest_request(snp_dev, req, rio); if (rc) { if (rc == -EIO && rio->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) @@ -382,7 +379,7 @@ static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, return rc; } - rc = verify_and_dec_payload(snp_dev, resp_buf, resp_sz); + rc = verify_and_dec_payload(snp_dev, req); if (rc) { dev_alert(snp_dev->dev, "Detected unexpected decode failure from ASP. rc: %d\n", rc); snp_disable_vmpck(snp_dev); @@ -401,6 +398,7 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io { struct snp_report_req *report_req = &snp_dev->req.report; struct snp_report_resp *report_resp; + struct snp_guest_req req = {}; int rc, resp_len; lockdep_assert_held(&snp_cmd_mutex); @@ -421,8 +419,16 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io if (!report_resp) return -ENOMEM; - rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg, SNP_MSG_REPORT_REQ, - report_req, sizeof(*report_req), report_resp->data, resp_len); + req.msg_version = arg->msg_version; + req.msg_type = SNP_MSG_REPORT_REQ; + req.vmpck_id = vmpck_id; + req.req_buf = report_req; + req.req_sz = sizeof(*report_req); + req.resp_buf = report_resp->data; + req.resp_sz = resp_len; + req.exit_code = SVM_VMGEXIT_GUEST_REQUEST; + + rc = snp_send_guest_request(snp_dev, &req, arg); if (rc) goto e_free; @@ -438,6 +444,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque { struct snp_derived_key_req *derived_key_req = &snp_dev->req.derived_key; struct snp_derived_key_resp derived_key_resp = {0}; + struct snp_guest_req req = {}; int rc, resp_len; /* Response data is 64 bytes and max authsize for GCM is 16 bytes. */ u8 buf[64 + 16]; @@ -460,8 +467,16 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque sizeof(*derived_key_req))) return -EFAULT; - rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg, SNP_MSG_KEY_REQ, - derived_key_req, sizeof(*derived_key_req), buf, resp_len); + req.msg_version = arg->msg_version; + req.msg_type = SNP_MSG_KEY_REQ; + req.vmpck_id = vmpck_id; + req.req_buf = derived_key_req; + req.req_sz = sizeof(*derived_key_req); + req.resp_buf = buf; + req.resp_sz = resp_len; + req.exit_code = SVM_VMGEXIT_GUEST_REQUEST; + + rc = snp_send_guest_request(snp_dev, &req, arg); if (rc) return rc; @@ -482,6 +497,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques { struct snp_ext_report_req *report_req = &snp_dev->req.ext_report; struct snp_report_resp *report_resp; + struct snp_guest_req req = {}; int ret, npages = 0, resp_len; sockptr_t certs_address; @@ -529,9 +545,17 @@ cmd: return -ENOMEM; snp_dev->input.data_npages = npages; - ret = handle_guest_request(snp_dev, SVM_VMGEXIT_EXT_GUEST_REQUEST, arg, SNP_MSG_REPORT_REQ, - &report_req->data, sizeof(report_req->data), - report_resp->data, resp_len); + + req.msg_version = arg->msg_version; + req.msg_type = SNP_MSG_REPORT_REQ; + req.vmpck_id = vmpck_id; + req.req_buf = &report_req->data; + req.req_sz = sizeof(report_req->data); + req.resp_buf = report_resp->data; + req.resp_sz = resp_len; + req.exit_code = SVM_VMGEXIT_EXT_GUEST_REQUEST; + + ret = snp_send_guest_request(snp_dev, &req, arg); /* If certs length is invalid then copy the returned length */ if (arg->vmm_error == SNP_GUEST_VMM_ERR_INVALID_LEN) { @@ -1057,7 +1081,7 @@ static int __init sev_guest_probe(struct platform_device *pdev) misc->name = DEVICE_NAME; misc->fops = &snp_guest_fops; - /* initial the input address for guest request */ + /* Initialize the input addresses for guest request */ snp_dev->input.req_gpa = __pa(snp_dev->request); snp_dev->input.resp_gpa = __pa(snp_dev->response); snp_dev->input.data_gpa = __pa(snp_dev->certs_data); From ae596615d93dedbdfffbe383f821bea5c5289576 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:36 +0530 Subject: [PATCH 07/12] virt: sev-guest: Reduce the scope of SNP command mutex The SNP command mutex is used to serialize access to the shared buffer, command handling, and message sequence number. All shared buffer, command handling, and message sequence updates are done within snp_send_guest_request(), so moving the mutex to this function is appropriate and maintains the critical section. Since the mutex is now taken at a later point in time, remove the lockdep checks that occur before taking the mutex. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241009092850.197575-6-nikunj@amd.com --- drivers/virt/coco/sev-guest/sev-guest.c | 35 ++++++------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index 2a1b542168b1..1bddef822446 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -345,6 +345,14 @@ static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_gues u64 seqno; int rc; + guard(mutex)(&snp_cmd_mutex); + + /* Check if the VMPCK is not empty */ + if (is_vmpck_empty(snp_dev)) { + dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); + return -ENOTTY; + } + /* Get message sequence and verify that its a non-zero */ seqno = snp_get_msg_seqno(snp_dev); if (!seqno) @@ -401,8 +409,6 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io struct snp_guest_req req = {}; int rc, resp_len; - lockdep_assert_held(&snp_cmd_mutex); - if (!arg->req_data || !arg->resp_data) return -EINVAL; @@ -449,8 +455,6 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque /* Response data is 64 bytes and max authsize for GCM is 16 bytes. */ u8 buf[64 + 16]; - lockdep_assert_held(&snp_cmd_mutex); - if (!arg->req_data || !arg->resp_data) return -EINVAL; @@ -501,8 +505,6 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques int ret, npages = 0, resp_len; sockptr_t certs_address; - lockdep_assert_held(&snp_cmd_mutex); - if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data)) return -EINVAL; @@ -598,15 +600,6 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long if (!input.msg_version) return -EINVAL; - mutex_lock(&snp_cmd_mutex); - - /* Check if the VMPCK is not empty */ - if (is_vmpck_empty(snp_dev)) { - dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); - mutex_unlock(&snp_cmd_mutex); - return -ENOTTY; - } - switch (ioctl) { case SNP_GET_REPORT: ret = get_report(snp_dev, &input); @@ -628,8 +621,6 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long break; } - mutex_unlock(&snp_cmd_mutex); - if (input.exitinfo2 && copy_to_user(argp, &input, sizeof(input))) return -EFAULT; @@ -744,8 +735,6 @@ static int sev_svsm_report_new(struct tsm_report *report, void *data) man_len = SZ_4K; certs_len = SEV_FW_BLOB_MAX_SIZE; - guard(mutex)(&snp_cmd_mutex); - if (guid_is_null(&desc->service_guid)) { call_id = SVSM_ATTEST_CALL(SVSM_ATTEST_SERVICES); } else { @@ -880,14 +869,6 @@ static int sev_report_new(struct tsm_report *report, void *data) if (!buf) return -ENOMEM; - guard(mutex)(&snp_cmd_mutex); - - /* Check if the VMPCK is not empty */ - if (is_vmpck_empty(snp_dev)) { - dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); - return -ENOTTY; - } - cert_table = buf + report_size; struct snp_ext_report_req ext_req = { .data = { .vmpl = desc->privlevel }, From 0a895c0d9b73d934de95aa0dd4e631c394bdd25d Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 9 Oct 2024 14:58:37 +0530 Subject: [PATCH 08/12] virt: sev-guest: Carve out SNP message context structure Currently, the sev-guest driver is the only user of SNP guest messaging. The snp_guest_dev structure holds all the allocated buffers, secrets page and VMPCK details. In preparation for adding messaging allocation and initialization APIs, decouple snp_guest_dev from messaging-related information by carving out the guest message context structure(snp_msg_desc). Incorporate this newly added context into snp_send_guest_request() and all related functions, replacing the use of the snp_guest_dev. No functional change. Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20241009092850.197575-7-nikunj@amd.com --- arch/x86/include/asm/sev.h | 21 +++ drivers/virt/coco/sev-guest/sev-guest.c | 178 ++++++++++++------------ 2 files changed, 108 insertions(+), 91 deletions(-) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 27fa1c9c3465..2e49c4a9e7fe 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -234,6 +234,27 @@ struct snp_secrets_page { u8 rsvd4[3744]; } __packed; +struct snp_msg_desc { + /* request and response are in unencrypted memory */ + struct snp_guest_msg *request, *response; + + /* + * Avoid information leakage by double-buffering shared messages + * in fields that are in regular encrypted memory. + */ + struct snp_guest_msg secret_request, secret_response; + + struct snp_secrets_page *secrets; + struct snp_req_data input; + + void *certs_data; + + struct aesgcm_ctx *ctx; + + u32 *os_area_msg_seqno; + u8 *vmpck; +}; + /* * The SVSM Calling Area (CA) related structures. */ diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index 1bddef822446..fca5c45ed5cd 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -40,26 +40,13 @@ struct snp_guest_dev { struct device *dev; struct miscdevice misc; - void *certs_data; - struct aesgcm_ctx *ctx; - /* request and response are in unencrypted memory */ - struct snp_guest_msg *request, *response; + struct snp_msg_desc *msg_desc; - /* - * Avoid information leakage by double-buffering shared messages - * in fields that are in regular encrypted memory. - */ - struct snp_guest_msg secret_request, secret_response; - - struct snp_secrets_page *secrets; - struct snp_req_data input; union { struct snp_report_req report; struct snp_derived_key_req derived_key; struct snp_ext_report_req ext_report; } req; - u32 *os_area_msg_seqno; - u8 *vmpck; }; /* @@ -76,12 +63,12 @@ MODULE_PARM_DESC(vmpck_id, "The VMPCK ID to use when communicating with the PSP. /* Mutex to serialize the shared buffer access and command handling. */ static DEFINE_MUTEX(snp_cmd_mutex); -static bool is_vmpck_empty(struct snp_guest_dev *snp_dev) +static bool is_vmpck_empty(struct snp_msg_desc *mdesc) { char zero_key[VMPCK_KEY_LEN] = {0}; - if (snp_dev->vmpck) - return !memcmp(snp_dev->vmpck, zero_key, VMPCK_KEY_LEN); + if (mdesc->vmpck) + return !memcmp(mdesc->vmpck, zero_key, VMPCK_KEY_LEN); return true; } @@ -103,30 +90,30 @@ static bool is_vmpck_empty(struct snp_guest_dev *snp_dev) * vulnerable. If the sequence number were incremented for a fresh IV the ASP * will reject the request. */ -static void snp_disable_vmpck(struct snp_guest_dev *snp_dev) +static void snp_disable_vmpck(struct snp_msg_desc *mdesc) { - dev_alert(snp_dev->dev, "Disabling VMPCK%d communication key to prevent IV reuse.\n", + pr_alert("Disabling VMPCK%d communication key to prevent IV reuse.\n", vmpck_id); - memzero_explicit(snp_dev->vmpck, VMPCK_KEY_LEN); - snp_dev->vmpck = NULL; + memzero_explicit(mdesc->vmpck, VMPCK_KEY_LEN); + mdesc->vmpck = NULL; } -static inline u64 __snp_get_msg_seqno(struct snp_guest_dev *snp_dev) +static inline u64 __snp_get_msg_seqno(struct snp_msg_desc *mdesc) { u64 count; lockdep_assert_held(&snp_cmd_mutex); /* Read the current message sequence counter from secrets pages */ - count = *snp_dev->os_area_msg_seqno; + count = *mdesc->os_area_msg_seqno; return count + 1; } /* Return a non-zero on success */ -static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev) +static u64 snp_get_msg_seqno(struct snp_msg_desc *mdesc) { - u64 count = __snp_get_msg_seqno(snp_dev); + u64 count = __snp_get_msg_seqno(mdesc); /* * The message sequence counter for the SNP guest request is a 64-bit @@ -137,20 +124,20 @@ static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev) * invalid number and will fail the message request. */ if (count >= UINT_MAX) { - dev_err(snp_dev->dev, "request message sequence counter overflow\n"); + pr_err("request message sequence counter overflow\n"); return 0; } return count; } -static void snp_inc_msg_seqno(struct snp_guest_dev *snp_dev) +static void snp_inc_msg_seqno(struct snp_msg_desc *mdesc) { /* * The counter is also incremented by the PSP, so increment it by 2 * and save in secrets page. */ - *snp_dev->os_area_msg_seqno += 2; + *mdesc->os_area_msg_seqno += 2; } static inline struct snp_guest_dev *to_snp_dev(struct file *file) @@ -177,13 +164,13 @@ static struct aesgcm_ctx *snp_init_crypto(u8 *key, size_t keylen) return ctx; } -static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_req *req) +static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_req *req) { - struct snp_guest_msg *resp_msg = &snp_dev->secret_response; - struct snp_guest_msg *req_msg = &snp_dev->secret_request; + struct snp_guest_msg *resp_msg = &mdesc->secret_response; + struct snp_guest_msg *req_msg = &mdesc->secret_request; struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr; struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr; - struct aesgcm_ctx *ctx = snp_dev->ctx; + struct aesgcm_ctx *ctx = mdesc->ctx; u8 iv[GCM_AES_IV_SIZE] = {}; pr_debug("response [seqno %lld type %d version %d sz %d]\n", @@ -191,7 +178,7 @@ static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_gues resp_msg_hdr->msg_sz); /* Copy response from shared memory to encrypted memory. */ - memcpy(resp_msg, snp_dev->response, sizeof(*resp_msg)); + memcpy(resp_msg, mdesc->response, sizeof(*resp_msg)); /* Verify that the sequence counter is incremented by 1 */ if (unlikely(resp_msg_hdr->msg_seqno != (req_msg_hdr->msg_seqno + 1))) @@ -218,11 +205,11 @@ static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_gues return 0; } -static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, struct snp_guest_req *req) +static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_req *req) { - struct snp_guest_msg *msg = &snp_dev->secret_request; + struct snp_guest_msg *msg = &mdesc->secret_request; struct snp_guest_msg_hdr *hdr = &msg->hdr; - struct aesgcm_ctx *ctx = snp_dev->ctx; + struct aesgcm_ctx *ctx = mdesc->ctx; u8 iv[GCM_AES_IV_SIZE] = {}; memset(msg, 0, sizeof(*msg)); @@ -253,7 +240,7 @@ static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, struct snp_gues return 0; } -static int __handle_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, +static int __handle_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req, struct snp_guest_request_ioctl *rio) { unsigned long req_start = jiffies; @@ -268,7 +255,7 @@ retry_request: * sequence number must be incremented or the VMPCK must be deleted to * prevent reuse of the IV. */ - rc = snp_issue_guest_request(req, &snp_dev->input, rio); + rc = snp_issue_guest_request(req, &mdesc->input, rio); switch (rc) { case -ENOSPC: /* @@ -278,7 +265,7 @@ retry_request: * order to increment the sequence number and thus avoid * IV reuse. */ - override_npages = snp_dev->input.data_npages; + override_npages = mdesc->input.data_npages; req->exit_code = SVM_VMGEXIT_GUEST_REQUEST; /* @@ -318,7 +305,7 @@ retry_request: * structure and any failure will wipe the VMPCK, preventing further * use anyway. */ - snp_inc_msg_seqno(snp_dev); + snp_inc_msg_seqno(mdesc); if (override_err) { rio->exitinfo2 = override_err; @@ -334,12 +321,12 @@ retry_request: } if (override_npages) - snp_dev->input.data_npages = override_npages; + mdesc->input.data_npages = override_npages; return rc; } -static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, +static int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req, struct snp_guest_request_ioctl *rio) { u64 seqno; @@ -348,21 +335,21 @@ static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_gues guard(mutex)(&snp_cmd_mutex); /* Check if the VMPCK is not empty */ - if (is_vmpck_empty(snp_dev)) { - dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); + if (is_vmpck_empty(mdesc)) { + pr_err_ratelimited("VMPCK is disabled\n"); return -ENOTTY; } /* Get message sequence and verify that its a non-zero */ - seqno = snp_get_msg_seqno(snp_dev); + seqno = snp_get_msg_seqno(mdesc); if (!seqno) return -EIO; /* Clear shared memory's response for the host to populate. */ - memset(snp_dev->response, 0, sizeof(struct snp_guest_msg)); + memset(mdesc->response, 0, sizeof(struct snp_guest_msg)); - /* Encrypt the userspace provided payload in snp_dev->secret_request. */ - rc = enc_payload(snp_dev, seqno, req); + /* Encrypt the userspace provided payload in mdesc->secret_request. */ + rc = enc_payload(mdesc, seqno, req); if (rc) return rc; @@ -370,27 +357,26 @@ static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_gues * Write the fully encrypted request to the shared unencrypted * request page. */ - memcpy(snp_dev->request, &snp_dev->secret_request, - sizeof(snp_dev->secret_request)); + memcpy(mdesc->request, &mdesc->secret_request, + sizeof(mdesc->secret_request)); - rc = __handle_guest_request(snp_dev, req, rio); + rc = __handle_guest_request(mdesc, req, rio); if (rc) { if (rc == -EIO && rio->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) return rc; - dev_alert(snp_dev->dev, - "Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n", - rc, rio->exitinfo2); + pr_alert("Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n", + rc, rio->exitinfo2); - snp_disable_vmpck(snp_dev); + snp_disable_vmpck(mdesc); return rc; } - rc = verify_and_dec_payload(snp_dev, req); + rc = verify_and_dec_payload(mdesc, req); if (rc) { - dev_alert(snp_dev->dev, "Detected unexpected decode failure from ASP. rc: %d\n", rc); - snp_disable_vmpck(snp_dev); + pr_alert("Detected unexpected decode failure from ASP. rc: %d\n", rc); + snp_disable_vmpck(mdesc); return rc; } @@ -405,6 +391,7 @@ struct snp_req_resp { static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg) { struct snp_report_req *report_req = &snp_dev->req.report; + struct snp_msg_desc *mdesc = snp_dev->msg_desc; struct snp_report_resp *report_resp; struct snp_guest_req req = {}; int rc, resp_len; @@ -420,7 +407,7 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + snp_dev->ctx->authsize; + resp_len = sizeof(report_resp->data) + mdesc->ctx->authsize; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) return -ENOMEM; @@ -434,7 +421,7 @@ static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_io req.resp_sz = resp_len; req.exit_code = SVM_VMGEXIT_GUEST_REQUEST; - rc = snp_send_guest_request(snp_dev, &req, arg); + rc = snp_send_guest_request(mdesc, &req, arg); if (rc) goto e_free; @@ -450,6 +437,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque { struct snp_derived_key_req *derived_key_req = &snp_dev->req.derived_key; struct snp_derived_key_resp derived_key_resp = {0}; + struct snp_msg_desc *mdesc = snp_dev->msg_desc; struct snp_guest_req req = {}; int rc, resp_len; /* Response data is 64 bytes and max authsize for GCM is 16 bytes. */ @@ -463,7 +451,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(derived_key_resp.data) + snp_dev->ctx->authsize; + resp_len = sizeof(derived_key_resp.data) + mdesc->ctx->authsize; if (sizeof(buf) < resp_len) return -ENOMEM; @@ -480,7 +468,7 @@ static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_reque req.resp_sz = resp_len; req.exit_code = SVM_VMGEXIT_GUEST_REQUEST; - rc = snp_send_guest_request(snp_dev, &req, arg); + rc = snp_send_guest_request(mdesc, &req, arg); if (rc) return rc; @@ -500,6 +488,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques { struct snp_ext_report_req *report_req = &snp_dev->req.ext_report; + struct snp_msg_desc *mdesc = snp_dev->msg_desc; struct snp_report_resp *report_resp; struct snp_guest_req req = {}; int ret, npages = 0, resp_len; @@ -533,7 +522,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques * the host. If host does not supply any certs in it, then copy * zeros to indicate that certificate data was not provided. */ - memset(snp_dev->certs_data, 0, report_req->certs_len); + memset(mdesc->certs_data, 0, report_req->certs_len); npages = report_req->certs_len >> PAGE_SHIFT; cmd: /* @@ -541,12 +530,12 @@ cmd: * response payload. Make sure that it has enough space to cover the * authtag. */ - resp_len = sizeof(report_resp->data) + snp_dev->ctx->authsize; + resp_len = sizeof(report_resp->data) + mdesc->ctx->authsize; report_resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT); if (!report_resp) return -ENOMEM; - snp_dev->input.data_npages = npages; + mdesc->input.data_npages = npages; req.msg_version = arg->msg_version; req.msg_type = SNP_MSG_REPORT_REQ; @@ -557,11 +546,11 @@ cmd: req.resp_sz = resp_len; req.exit_code = SVM_VMGEXIT_EXT_GUEST_REQUEST; - ret = snp_send_guest_request(snp_dev, &req, arg); + ret = snp_send_guest_request(mdesc, &req, arg); /* If certs length is invalid then copy the returned length */ if (arg->vmm_error == SNP_GUEST_VMM_ERR_INVALID_LEN) { - report_req->certs_len = snp_dev->input.data_npages << PAGE_SHIFT; + report_req->certs_len = mdesc->input.data_npages << PAGE_SHIFT; if (copy_to_sockptr(io->req_data, report_req, sizeof(*report_req))) ret = -EFAULT; @@ -570,7 +559,7 @@ cmd: if (ret) goto e_free; - if (npages && copy_to_sockptr(certs_address, snp_dev->certs_data, report_req->certs_len)) { + if (npages && copy_to_sockptr(certs_address, mdesc->certs_data, report_req->certs_len)) { ret = -EFAULT; goto e_free; } @@ -994,6 +983,7 @@ static int __init sev_guest_probe(struct platform_device *pdev) struct snp_secrets_page *secrets; struct device *dev = &pdev->dev; struct snp_guest_dev *snp_dev; + struct snp_msg_desc *mdesc; struct miscdevice *misc; void __iomem *mapping; int ret; @@ -1018,43 +1008,47 @@ static int __init sev_guest_probe(struct platform_device *pdev) if (!snp_dev) goto e_unmap; + mdesc = devm_kzalloc(&pdev->dev, sizeof(struct snp_msg_desc), GFP_KERNEL); + if (!mdesc) + goto e_unmap; + /* Adjust the default VMPCK key based on the executing VMPL level */ if (vmpck_id == -1) vmpck_id = snp_vmpl; ret = -EINVAL; - snp_dev->vmpck = get_vmpck(vmpck_id, secrets, &snp_dev->os_area_msg_seqno); - if (!snp_dev->vmpck) { + mdesc->vmpck = get_vmpck(vmpck_id, secrets, &mdesc->os_area_msg_seqno); + if (!mdesc->vmpck) { dev_err(dev, "Invalid VMPCK%d communication key\n", vmpck_id); goto e_unmap; } /* Verify that VMPCK is not zero. */ - if (is_vmpck_empty(snp_dev)) { + if (is_vmpck_empty(mdesc)) { dev_err(dev, "Empty VMPCK%d communication key\n", vmpck_id); goto e_unmap; } platform_set_drvdata(pdev, snp_dev); snp_dev->dev = dev; - snp_dev->secrets = secrets; + mdesc->secrets = secrets; /* Allocate the shared page used for the request and response message. */ - snp_dev->request = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); - if (!snp_dev->request) + mdesc->request = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); + if (!mdesc->request) goto e_unmap; - snp_dev->response = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); - if (!snp_dev->response) + mdesc->response = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); + if (!mdesc->response) goto e_free_request; - snp_dev->certs_data = alloc_shared_pages(dev, SEV_FW_BLOB_MAX_SIZE); - if (!snp_dev->certs_data) + mdesc->certs_data = alloc_shared_pages(dev, SEV_FW_BLOB_MAX_SIZE); + if (!mdesc->certs_data) goto e_free_response; ret = -EIO; - snp_dev->ctx = snp_init_crypto(snp_dev->vmpck, VMPCK_KEY_LEN); - if (!snp_dev->ctx) + mdesc->ctx = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN); + if (!mdesc->ctx) goto e_free_cert_data; misc = &snp_dev->misc; @@ -1063,9 +1057,9 @@ static int __init sev_guest_probe(struct platform_device *pdev) misc->fops = &snp_guest_fops; /* Initialize the input addresses for guest request */ - snp_dev->input.req_gpa = __pa(snp_dev->request); - snp_dev->input.resp_gpa = __pa(snp_dev->response); - snp_dev->input.data_gpa = __pa(snp_dev->certs_data); + mdesc->input.req_gpa = __pa(mdesc->request); + mdesc->input.resp_gpa = __pa(mdesc->response); + mdesc->input.data_gpa = __pa(mdesc->certs_data); /* Set the privlevel_floor attribute based on the vmpck_id */ sev_tsm_ops.privlevel_floor = vmpck_id; @@ -1082,17 +1076,18 @@ static int __init sev_guest_probe(struct platform_device *pdev) if (ret) goto e_free_ctx; + snp_dev->msg_desc = mdesc; dev_info(dev, "Initialized SEV guest driver (using VMPCK%d communication key)\n", vmpck_id); return 0; e_free_ctx: - kfree(snp_dev->ctx); + kfree(mdesc->ctx); e_free_cert_data: - free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); + free_shared_pages(mdesc->certs_data, SEV_FW_BLOB_MAX_SIZE); e_free_response: - free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg)); + free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg)); e_free_request: - free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg)); + free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); e_unmap: iounmap(mapping); return ret; @@ -1101,11 +1096,12 @@ e_unmap: static void __exit sev_guest_remove(struct platform_device *pdev) { struct snp_guest_dev *snp_dev = platform_get_drvdata(pdev); + struct snp_msg_desc *mdesc = snp_dev->msg_desc; - free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); - free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg)); - free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg)); - kfree(snp_dev->ctx); + free_shared_pages(mdesc->certs_data, SEV_FW_BLOB_MAX_SIZE); + free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg)); + free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); + kfree(mdesc->ctx); misc_deregister(&snp_dev->misc); } From f30470c190c2f4776e0baeba1f53fd8dd3820394 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Thu, 1 Aug 2024 19:14:17 +0000 Subject: [PATCH 09/12] x86/boot: Skip video memory access in the decompressor for SEV-ES/SNP Accessing guest video memory/RAM in the decompressor causes guest termination as the boot stage2 #VC handler for SEV-ES/SNP systems does not support MMIO handling. This issue is observed during a SEV-ES/SNP guest kexec as kexec -c adds screen_info to the boot parameters passed to the second kernel, which causes console output to be dumped to both video and serial. As the decompressor output gets cleared really fast, it is preferable to get the console output only on serial, hence, skip accessing the video RAM during decompressor stage to prevent guest termination. Serial console output during decompressor stage works as boot stage2 #VC handler already supports handling port I/O. [ bp: Massage. ] Suggested-by: Borislav Petkov (AMD) Suggested-by: Thomas Lendacky Signed-off-by: Ashish Kalra Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Kuppuswamy Sathyanarayanan Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/8a55ea86524c686e575d273311acbe57ce8cee23.1722520012.git.ashish.kalra@amd.com --- arch/x86/boot/compressed/misc.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 04a35b2c26e9..0d37420cad02 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -385,6 +385,19 @@ static void parse_mem_encrypt(struct setup_header *hdr) hdr->xloadflags |= XLF_MEM_ENCRYPTION; } +static void early_sev_detect(void) +{ + /* + * Accessing video memory causes guest termination because + * the boot stage2 #VC handler of SEV-ES/SNP guests does not + * support MMIO handling and kexec -c adds screen_info to the + * boot parameters passed to the kexec kernel, which causes + * console output to be dumped to both video and serial. + */ + if (sev_status & MSR_AMD64_SEV_ES_ENABLED) + lines = cols = 0; +} + /* * The compressed kernel image (ZO), has been moved so that its position * is against the end of the buffer used to hold the uncompressed kernel @@ -440,6 +453,8 @@ asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output) */ early_tdx_detect(); + early_sev_detect(); + console_init(); /* From 2a783066b6f5f5250b838d2acfc716561d2a66e0 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Thu, 1 Aug 2024 19:14:34 +0000 Subject: [PATCH 10/12] x86/mm: Refactor __set_clr_pte_enc() Refactor __set_clr_pte_enc() and add two new helper functions to set/clear PTE C-bit from early SEV/SNP initialization code and later during shutdown/kexec especially when all CPUs are stopped and interrupts are disabled and set_memory_xx() interfaces can't be used. Co-developed-by: Borislav Petkov (AMD) Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ashish Kalra Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/5df4aa450447f28294d1c5a890e27b63ed4ded36.1722520012.git.ashish.kalra@amd.com --- arch/x86/include/asm/sev.h | 20 ++++++++++ arch/x86/mm/mem_encrypt_amd.c | 75 +++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 2e49c4a9e7fe..5f598937f090 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -322,6 +322,22 @@ struct svsm_attest_call { u8 rsvd[4]; }; +/* PTE descriptor used for the prepare_pte_enc() operations. */ +struct pte_enc_desc { + pte_t *kpte; + int pte_level; + bool encrypt; + /* pfn of the kpte above */ + unsigned long pfn; + /* physical address of @pfn */ + unsigned long pa; + /* virtual address of @pfn */ + void *va; + /* memory covered by the pte */ + unsigned long size; + pgprot_t new_pgprot; +}; + /* * SVSM protocol structure */ @@ -437,6 +453,8 @@ u64 snp_get_unsupported_features(u64 status); u64 sev_get_status(void); void sev_show_status(void); void snp_update_svsm_ca(void); +int prepare_pte_enc(struct pte_enc_desc *d); +void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot); #else /* !CONFIG_AMD_MEM_ENCRYPT */ @@ -474,6 +492,8 @@ static inline u64 snp_get_unsupported_features(u64 status) { return 0; } static inline u64 sev_get_status(void) { return 0; } static inline void sev_show_status(void) { } static inline void snp_update_svsm_ca(void) { } +static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; } +static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { } #endif /* CONFIG_AMD_MEM_ENCRYPT */ diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index 86a476a426c2..f4be81db72ee 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -311,59 +311,82 @@ static int amd_enc_status_change_finish(unsigned long vaddr, int npages, bool en return 0; } -static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) +int prepare_pte_enc(struct pte_enc_desc *d) { - pgprot_t old_prot, new_prot; - unsigned long pfn, pa, size; - pte_t new_pte; + pgprot_t old_prot; - pfn = pg_level_to_pfn(level, kpte, &old_prot); - if (!pfn) - return; + d->pfn = pg_level_to_pfn(d->pte_level, d->kpte, &old_prot); + if (!d->pfn) + return 1; - new_prot = old_prot; - if (enc) - pgprot_val(new_prot) |= _PAGE_ENC; + d->new_pgprot = old_prot; + if (d->encrypt) + pgprot_val(d->new_pgprot) |= _PAGE_ENC; else - pgprot_val(new_prot) &= ~_PAGE_ENC; + pgprot_val(d->new_pgprot) &= ~_PAGE_ENC; /* If prot is same then do nothing. */ - if (pgprot_val(old_prot) == pgprot_val(new_prot)) - return; + if (pgprot_val(old_prot) == pgprot_val(d->new_pgprot)) + return 1; - pa = pfn << PAGE_SHIFT; - size = page_level_size(level); + d->pa = d->pfn << PAGE_SHIFT; + d->size = page_level_size(d->pte_level); /* - * We are going to perform in-place en-/decryption and change the - * physical page attribute from C=1 to C=0 or vice versa. Flush the - * caches to ensure that data gets accessed with the correct C-bit. + * In-place en-/decryption and physical page attribute change + * from C=1 to C=0 or vice versa will be performed. Flush the + * caches to ensure that data gets accessed with the correct + * C-bit. */ - clflush_cache_range(__va(pa), size); + if (d->va) + clflush_cache_range(d->va, d->size); + else + clflush_cache_range(__va(d->pa), d->size); + + return 0; +} + +void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) +{ + pte_t new_pte; + + /* Change the page encryption mask. */ + new_pte = pfn_pte(pfn, new_prot); + set_pte_atomic(kpte, new_pte); +} + +static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) +{ + struct pte_enc_desc d = { + .kpte = kpte, + .pte_level = level, + .encrypt = enc + }; + + if (prepare_pte_enc(&d)) + return; /* Encrypt/decrypt the contents in-place */ if (enc) { - sme_early_encrypt(pa, size); + sme_early_encrypt(d.pa, d.size); } else { - sme_early_decrypt(pa, size); + sme_early_decrypt(d.pa, d.size); /* * ON SNP, the page state in the RMP table must happen * before the page table updates. */ - early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1); + early_snp_set_memory_shared((unsigned long)__va(d.pa), d.pa, 1); } - /* Change the page encryption mask. */ - new_pte = pfn_pte(pfn, new_prot); - set_pte_atomic(kpte, new_pte); + set_pte_enc_mask(kpte, d.pfn, d.new_pgprot); /* * If page is set encrypted in the page table, then update the RMP table to * add this page as private. */ if (enc) - early_snp_set_memory_private((unsigned long)__va(pa), pa, 1); + early_snp_set_memory_private((unsigned long)__va(d.pa), d.pa, 1); } static int __init early_set_memory_enc_dec(unsigned long vaddr, From 3074152e56c9b0f9b9c67edfbc08b371db050b6d Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Thu, 1 Aug 2024 19:14:50 +0000 Subject: [PATCH 11/12] x86/sev: Convert shared memory back to private on kexec SNP guests allocate shared buffers to perform I/O. It is done by allocating pages normally from the buddy allocator and converting them to shared with set_memory_decrypted(). The second, kexec-ed, kernel has no idea what memory is converted this way. It only sees E820_TYPE_RAM. Accessing shared memory via private mapping will cause unrecoverable RMP page-faults. On kexec, walk direct mapping and convert all shared memory back to private. It makes all RAM private again and second kernel may use it normally. Additionally, for SNP guests, convert all bss decrypted section pages back to private. The conversion occurs in two steps: stopping new conversions and unsharing all memory. In the case of normal kexec, the stopping of conversions takes place while scheduling is still functioning. This allows for waiting until any ongoing conversions are finished. The second step is carried out when all CPUs except one are inactive and interrupts are disabled. This prevents any conflicts with code that may access shared memory. Co-developed-by: Borislav Petkov (AMD) Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ashish Kalra Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/05a8c15fb665dbb062b04a8cb3d592a63f235937.1722520012.git.ashish.kalra@amd.com --- arch/x86/coco/sev/core.c | 131 ++++++++++++++++++++++++++++++++++ arch/x86/include/asm/sev.h | 4 ++ arch/x86/mm/mem_encrypt_amd.c | 2 + 3 files changed, 137 insertions(+) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index c7b4270d0e18..97f445f3366a 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -954,6 +954,137 @@ void snp_accept_memory(phys_addr_t start, phys_addr_t end) set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); } +static void set_pte_enc(pte_t *kpte, int level, void *va) +{ + struct pte_enc_desc d = { + .kpte = kpte, + .pte_level = level, + .va = va, + .encrypt = true + }; + + prepare_pte_enc(&d); + set_pte_enc_mask(kpte, d.pfn, d.new_pgprot); +} + +static void unshare_all_memory(void) +{ + unsigned long addr, end, size, ghcb; + struct sev_es_runtime_data *data; + unsigned int npages, level; + bool skipped_addr; + pte_t *pte; + int cpu; + + /* Unshare the direct mapping. */ + addr = PAGE_OFFSET; + end = PAGE_OFFSET + get_max_mapped(); + + while (addr < end) { + pte = lookup_address(addr, &level); + size = page_level_size(level); + npages = size / PAGE_SIZE; + skipped_addr = false; + + if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) { + addr += size; + continue; + } + + /* + * Ensure that all the per-CPU GHCBs are made private at the + * end of the unsharing loop so that the switch to the slower + * MSR protocol happens last. + */ + for_each_possible_cpu(cpu) { + data = per_cpu(runtime_data, cpu); + ghcb = (unsigned long)&data->ghcb_page; + + if (addr <= ghcb && ghcb <= addr + size) { + skipped_addr = true; + break; + } + } + + if (!skipped_addr) { + set_pte_enc(pte, level, (void *)addr); + snp_set_memory_private(addr, npages); + } + addr += size; + } + + /* Unshare all bss decrypted memory. */ + addr = (unsigned long)__start_bss_decrypted; + end = (unsigned long)__start_bss_decrypted_unused; + npages = (end - addr) >> PAGE_SHIFT; + + for (; addr < end; addr += PAGE_SIZE) { + pte = lookup_address(addr, &level); + if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) + continue; + + set_pte_enc(pte, level, (void *)addr); + } + addr = (unsigned long)__start_bss_decrypted; + snp_set_memory_private(addr, npages); + + __flush_tlb_all(); +} + +/* Stop new private<->shared conversions */ +void snp_kexec_begin(void) +{ + if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) + return; + + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return; + + /* + * Crash kernel ends up here with interrupts disabled: can't wait for + * conversions to finish. + * + * If race happened, just report and proceed. + */ + if (!set_memory_enc_stop_conversion()) + pr_warn("Failed to stop shared<->private conversions\n"); +} + +void snp_kexec_finish(void) +{ + struct sev_es_runtime_data *data; + unsigned int level, cpu; + unsigned long size; + struct ghcb *ghcb; + pte_t *pte; + + if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) + return; + + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + return; + + unshare_all_memory(); + + /* + * Switch to using the MSR protocol to change per-CPU GHCBs to + * private. All the per-CPU GHCBs have been switched back to private, + * so can't do any more GHCB calls to the hypervisor beyond this point + * until the kexec'ed kernel starts running. + */ + boot_ghcb = NULL; + sev_cfg.ghcbs_initialized = false; + + for_each_possible_cpu(cpu) { + data = per_cpu(runtime_data, cpu); + ghcb = &data->ghcb_page; + pte = lookup_address((unsigned long)ghcb, &level); + size = page_level_size(level); + set_pte_enc(pte, level, (void *)ghcb); + snp_set_memory_private((unsigned long)ghcb, (size / PAGE_SIZE)); + } +} + static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa) { int ret; diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 5f598937f090..91f08af31078 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -455,6 +455,8 @@ void sev_show_status(void); void snp_update_svsm_ca(void); int prepare_pte_enc(struct pte_enc_desc *d); void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot); +void snp_kexec_finish(void); +void snp_kexec_begin(void); #else /* !CONFIG_AMD_MEM_ENCRYPT */ @@ -494,6 +496,8 @@ static inline void sev_show_status(void) { } static inline void snp_update_svsm_ca(void) { } static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; } static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { } +static inline void snp_kexec_finish(void) { } +static inline void snp_kexec_begin(void) { } #endif /* CONFIG_AMD_MEM_ENCRYPT */ diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index f4be81db72ee..774f9677458f 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -490,6 +490,8 @@ void __init sme_early_init(void) x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish; x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required; x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required; + x86_platform.guest.enc_kexec_begin = snp_kexec_begin; + x86_platform.guest.enc_kexec_finish = snp_kexec_finish; /* * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the From 8bca85cc1eb72e21a3544ab32e546a819d8674ca Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Wed, 6 Nov 2024 18:21:58 +0100 Subject: [PATCH 12/12] x86/sev: Cleanup vc_handle_msr() Carve out the MSR_SVSM_CAA into a helper with the suggestion that upcoming future users should do the same. Rename that silly exit_info_1 into what it actually means in this function - whether the MSR access is a read or a write. No functional changes. Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Reviewed-by: Pankaj Gupta Link: https://lore.kernel.org/r/20241106172647.GAZyum1zngPDwyD2IJ@fat_crate.local --- arch/x86/coco/sev/core.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 97f445f3366a..c5b0148b8c0a 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -1406,35 +1406,39 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd) return 0; } +/* Writes to the SVSM CAA MSR are ignored */ +static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) +{ + if (write) + return ES_OK; + + regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); + regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); + + return ES_OK; +} + static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct pt_regs *regs = ctxt->regs; enum es_result ret; - u64 exit_info_1; + bool write; /* Is it a WRMSR? */ - exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0; + write = ctxt->insn.opcode.bytes[1] == 0x30; - if (regs->cx == MSR_SVSM_CAA) { - /* Writes to the SVSM CAA msr are ignored */ - if (exit_info_1) - return ES_OK; - - regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); - regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); - - return ES_OK; - } + if (regs->cx == MSR_SVSM_CAA) + return __vc_handle_msr_caa(regs, write); ghcb_set_rcx(ghcb, regs->cx); - if (exit_info_1) { + if (write) { ghcb_set_rax(ghcb, regs->ax); ghcb_set_rdx(ghcb, regs->dx); } - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); - if ((ret == ES_OK) && (!exit_info_1)) { + if ((ret == ES_OK) && !write) { regs->ax = ghcb->save.rax; regs->dx = ghcb->save.rdx; }