diff --git a/Documentation/powerpc/index.rst b/Documentation/powerpc/index.rst index 0d45f0fc8e57..afe2d5e54db6 100644 --- a/Documentation/powerpc/index.rst +++ b/Documentation/powerpc/index.rst @@ -30,6 +30,7 @@ powerpc syscall64-abi transactional_memory ultravisor + vas-api .. only:: subproject and html diff --git a/Documentation/powerpc/vas-api.rst b/Documentation/powerpc/vas-api.rst new file mode 100644 index 000000000000..1217c2f1595e --- /dev/null +++ b/Documentation/powerpc/vas-api.rst @@ -0,0 +1,292 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. _VAS-API: + +=================================================== +Virtual Accelerator Switchboard (VAS) userspace API +=================================================== + +Introduction +============ + +Power9 processor introduced Virtual Accelerator Switchboard (VAS) which +allows both userspace and kernel communicate to co-processor +(hardware accelerator) referred to as the Nest Accelerator (NX). The NX +unit comprises of one or more hardware engines or co-processor types +such as 842 compression, GZIP compression and encryption. On power9, +userspace applications will have access to only GZIP Compression engine +which supports ZLIB and GZIP compression algorithms in the hardware. + +To communicate with NX, kernel has to establish a channel or window and +then requests can be submitted directly without kernel involvement. +Requests to the GZIP engine must be formatted as a co-processor Request +Block (CRB) and these CRBs must be submitted to the NX using COPY/PASTE +instructions to paste the CRB to hardware address that is associated with +the engine's request queue. + +The GZIP engine provides two priority levels of requests: Normal and +High. Only Normal requests are supported from userspace right now. + +This document explains userspace API that is used to interact with +kernel to setup channel / window which can be used to send compression +requests directly to NX accelerator. + + +Overview +======== + +Application access to the GZIP engine is provided through +/dev/crypto/nx-gzip device node implemented by the VAS/NX device driver. +An application must open the /dev/crypto/nx-gzip device to obtain a file +descriptor (fd). Then should issue VAS_TX_WIN_OPEN ioctl with this fd to +establish connection to the engine. It means send window is opened on GZIP +engine for this process. Once a connection is established, the application +should use the mmap() system call to map the hardware address of engine's +request queue into the application's virtual address space. + +The application can then submit one or more requests to the the engine by +using copy/paste instructions and pasting the CRBs to the virtual address +(aka paste_address) returned by mmap(). User space can close the +established connection or send window by closing the file descriptior +(close(fd)) or upon the process exit. + +Note that applications can send several requests with the same window or +can establish multiple windows, but one window for each file descriptor. + +Following sections provide additional details and references about the +individual steps. + +NX-GZIP Device Node +=================== + +There is one /dev/crypto/nx-gzip node in the system and it provides +access to all GZIP engines in the system. The only valid operations on +/dev/crypto/nx-gzip are: + + * open() the device for read and write. + * issue VAS_TX_WIN_OPEN ioctl + * mmap() the engine's request queue into application's virtual + address space (i.e. get a paste_address for the co-processor + engine). + * close the device node. + +Other file operations on this device node are undefined. + +Note that the copy and paste operations go directly to the hardware and +do not go through this device. Refer COPY/PASTE document for more +details. + +Although a system may have several instances of the NX co-processor +engines (typically, one per P9 chip) there is just one +/dev/crypto/nx-gzip device node in the system. When the nx-gzip device +node is opened, Kernel opens send window on a suitable instance of NX +accelerator. It finds CPU on which the user process is executing and +determine the NX instance for the corresponding chip on which this CPU +belongs. + +Applications may chose a specific instance of the NX co-processor using +the vas_id field in the VAS_TX_WIN_OPEN ioctl as detailed below. + +A userspace library libnxz is available here but still in development: + https://github.com/abalib/power-gzip + +Applications that use inflate / deflate calls can link with libnxz +instead of libz and use NX GZIP compression without any modification. + +Open /dev/crypto/nx-gzip +======================== + +The nx-gzip device should be opened for read and write. No special +privileges are needed to open the device. Each window corresponds to one +file descriptor. So if the userspace process needs multiple windows, +several open calls have to be issued. + +See open(2) system call man pages for other details such as return values, +error codes and restrictions. + +VAS_TX_WIN_OPEN ioctl +===================== + +Applications should use the VAS_TX_WIN_OPEN ioctl as follows to establish +a connection with NX co-processor engine: + + :: + struct vas_tx_win_open_attr { + __u32 version; + __s16 vas_id; /* specific instance of vas or -1 + for default */ + __u16 reserved1; + __u64 flags; /* For future use */ + __u64 reserved2[6]; + }; + + version: The version field must be currently set to 1. + vas_id: If '-1' is passed, kernel will make a best-effort attempt + to assign an optimal instance of NX for the process. To + select the specific VAS instance, refer + "Discovery of available VAS engines" section below. + + flags, reserved1 and reserved2[6] fields are for future extension + and must be set to 0. + + The attributes attr for the VAS_TX_WIN_OPEN ioctl are defined as + follows: + #define VAS_MAGIC 'v' + #define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 1, + struct vas_tx_win_open_attr) + + struct vas_tx_win_open_attr attr; + rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr); + + The VAS_TX_WIN_OPEN ioctl returns 0 on success. On errors, it + returns -1 and sets the errno variable to indicate the error. + + Error conditions: + EINVAL fd does not refer to a valid VAS device. + EINVAL Invalid vas ID + EINVAL version is not set with proper value + EEXIST Window is already opened for the given fd + ENOMEM Memory is not available to allocate window + ENOSPC System has too many active windows (connections) + opened + EINVAL reserved fields are not set to 0. + + See the ioctl(2) man page for more details, error codes and + restrictions. + +mmap() NX-GZIP device +===================== + +The mmap() system call for a NX-GZIP device fd returns a paste_address +that the application can use to copy/paste its CRB to the hardware engines. + :: + + paste_addr = mmap(addr, size, prot, flags, fd, offset); + + Only restrictions on mmap for a NX-GZIP device fd are: + * size should be PAGE_SIZE + * offset parameter should be 0ULL + + Refer to mmap(2) man page for additional details/restrictions. + In addition to the error conditions listed on the mmap(2) man + page, can also fail with one of the following error codes: + + EINVAL fd is not associated with an open window + (i.e mmap() does not follow a successful call + to the VAS_TX_WIN_OPEN ioctl). + EINVAL offset field is not 0ULL. + +Discovery of available VAS engines +================================== + +Each available VAS instance in the system will have a device tree node +like /proc/device-tree/vas@* or /proc/device-tree/xscom@*/vas@*. +Determine the chip or VAS instance and use the corresponding ibm,vas-id +property value in this node to select specific VAS instance. + +Copy/Paste operations +===================== + +Applications should use the copy and paste instructions to send CRB to NX. +Refer section 4.4 in PowerISA for Copy/Paste instructions: +https://openpowerfoundation.org/?resource_lib=power-isa-version-3-0 + +CRB Specification and use NX +============================ + +Applications should format requests to the co-processor using the +co-processor Request Block (CRBs). Refer NX-GZIP user's manual for the format +of CRB and use NX from userspace such as sending requests and checking +request status. + +NX Fault handling +================= + +Applications send requests to NX and wait for the status by polling on +co-processor Status Block (CSB) flags. NX updates status in CSB after each +request is processed. Refer NX-GZIP user's manual for the format of CSB and +status flags. + +In case if NX encounters translation error (called NX page fault) on CSB +address or any request buffer, raises an interrupt on the CPU to handle the +fault. Page fault can happen if an application passes invalid addresses or +request buffers are not in memory. The operating system handles the fault by +updating CSB with the following data: + + csb.flags = CSB_V; + csb.cc = CSB_CC_TRANSLATION; + csb.ce = CSB_CE_TERMINATION; + csb.address = fault_address; + +When an application receives translation error, it can touch or access +the page that has a fault address so that this page will be in memory. Then +the application can resend this request to NX. + +If the OS can not update CSB due to invalid CSB address, sends SEGV signal +to the process who opened the send window on which the original request was +issued. This signal returns with the following siginfo struct: + + siginfo.si_signo = SIGSEGV; + siginfo.si_errno = EFAULT; + siginfo.si_code = SEGV_MAPERR; + siginfo.si_addr = CSB adress; + +In the case of multi-thread applications, NX send windows can be shared +across all threads. For example, a child thread can open a send window, +but other threads can send requests to NX using this window. These +requests will be successful even in the case of OS handling faults as long +as CSB address is valid. If the NX request contains an invalid CSB address, +the signal will be sent to the child thread that opened the window. But if +the thread is exited without closing the window and the request is issued +using this window. the signal will be issued to the thread group leader +(tgid). It is up to the application whether to ignore or handle these +signals. + +NX-GZIP User's Manual: +https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf + +Simple example +============== + + :: + int use_nx_gzip() + { + int rc, fd; + void *addr; + struct vas_setup_attr txattr; + + fd = open("/dev/crypto/nx-gzip", O_RDWR); + if (fd < 0) { + fprintf(stderr, "open nx-gzip failed\n"); + return -1; + } + memset(&txattr, 0, sizeof(txattr)); + txattr.version = 1; + txattr.vas_id = -1 + rc = ioctl(fd, VAS_TX_WIN_OPEN, + (unsigned long)&txattr); + if (rc < 0) { + fprintf(stderr, "ioctl() n %d, error %d\n", + rc, errno); + return rc; + } + addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, + MAP_SHARED, fd, 0ULL); + if (addr == MAP_FAILED) { + fprintf(stderr, "mmap() failed, errno %d\n", + errno); + return -errno; + } + do { + //Format CRB request with compression or + //uncompression + // Refer tests for vas_copy/vas_paste + vas_copy((&crb, 0, 1); + vas_paste(addr, 0, 1); + // Poll on csb.flags with timeout + // csb address is listed in CRB + } while (true) + close(fd) or window can be closed upon process exit + } + + Refer https://github.com/abalib/power-gzip for tests or more + use cases. diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index f759edafd938..f18accb1bcd8 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -286,6 +286,7 @@ Code Seq# Include File Comments 'v' 00-1F linux/fs.h conflict! 'v' 00-0F linux/sonypi.h conflict! 'v' 00-0F media/v4l2-subdev.h conflict! +'v' 20-27 arch/powerpc/include/uapi/asm/vas-api.h VAS API 'v' C0-FF linux/meye.h conflict! 'w' all CERN SCI driver 'y' 00-1F packet based user level communications diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h index f93e6b0f5c84..e33f80b0ea81 100644 --- a/arch/powerpc/include/asm/vas.h +++ b/arch/powerpc/include/asm/vas.h @@ -86,7 +86,6 @@ struct vas_tx_win_attr { int wcreds_max; int lpid; int pidr; /* hardware PID (from SPRN_PID) */ - int pid; /* linux process id */ int pswid; int rsvd_txbuf_count; int tc_mode; @@ -163,4 +162,16 @@ int vas_copy_crb(void *crb, int offset); */ int vas_paste_crb(struct vas_window *win, int offset, bool re); +/* + * Register / unregister coprocessor type to VAS API which will be exported + * to user space. Applications can use this API to open / close window + * which can be used to send / receive requests directly to cooprcessor. + * + * Only NX GZIP coprocessor type is supported now, but this API can be + * used for others in future. + */ +int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, + const char *name); +void vas_unregister_coproc_api(void); + #endif /* __ASM_POWERPC_VAS_H */ diff --git a/arch/powerpc/include/uapi/asm/vas-api.h b/arch/powerpc/include/uapi/asm/vas-api.h new file mode 100644 index 000000000000..fe95d67e3bab --- /dev/null +++ b/arch/powerpc/include/uapi/asm/vas-api.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * Copyright 2019 IBM Corp. + */ + +#ifndef _UAPI_MISC_VAS_H +#define _UAPI_MISC_VAS_H + +#include + +#define VAS_MAGIC 'v' +#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 0x20, struct vas_tx_win_open_attr) + +struct vas_tx_win_open_attr { + __u32 version; + __s16 vas_id; /* specific instance of vas or -1 for default */ + __u16 reserved1; + __u64 flags; /* Future use */ + __u64 reserved2[6]; +}; + +#endif /* _UAPI_MISC_VAS_H */ diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile index 395789ffc482..fe3f0fb5aeca 100644 --- a/arch/powerpc/platforms/powernv/Makefile +++ b/arch/powerpc/platforms/powernv/Makefile @@ -17,7 +17,7 @@ obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o obj-$(CONFIG_OPAL_PRD) += opal-prd.o obj-$(CONFIG_PERF_EVENTS) += opal-imc.o obj-$(CONFIG_PPC_MEMTRACE) += memtrace.o -obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o +obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o vas-api.o obj-$(CONFIG_OCXL_BASE) += ocxl.o obj-$(CONFIG_SCOM_DEBUGFS) += opal-xscom.o obj-$(CONFIG_PPC_SECURE_BOOT) += opal-secvar.o diff --git a/arch/powerpc/platforms/powernv/vas-api.c b/arch/powerpc/platforms/powernv/vas-api.c new file mode 100644 index 000000000000..98ed5d8c5441 --- /dev/null +++ b/arch/powerpc/platforms/powernv/vas-api.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * VAS user space API for its accelerators (Only NX-GZIP is supported now) + * Copyright (C) 2019 Haren Myneni, IBM Corp + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "vas.h" + +/* + * The driver creates the device node that can be used as follows: + * For NX-GZIP + * + * fd = open("/dev/crypto/nx-gzip", O_RDWR); + * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr); + * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL). + * vas_copy(&crb, 0, 1); + * vas_paste(paste_addr, 0, 1); + * close(fd) or exit process to close window. + * + * where "vas_copy" and "vas_paste" are defined in copy-paste.h. + * copy/paste returns to the user space directly. So refer NX hardware + * documententation for exact copy/paste usage and completion / error + * conditions. + */ + +/* + * Wrapper object for the nx-gzip device - there is just one instance of + * this node for the whole system. + */ +static struct coproc_dev { + struct cdev cdev; + struct device *device; + char *name; + dev_t devt; + struct class *class; + enum vas_cop_type cop_type; +} coproc_device; + +struct coproc_instance { + struct coproc_dev *coproc; + struct vas_window *txwin; +}; + +static char *coproc_devnode(struct device *dev, umode_t *mode) +{ + return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev)); +} + +static int coproc_open(struct inode *inode, struct file *fp) +{ + struct coproc_instance *cp_inst; + + cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL); + if (!cp_inst) + return -ENOMEM; + + cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev, + cdev); + fp->private_data = cp_inst; + + return 0; +} + +static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) +{ + void __user *uptr = (void __user *)arg; + struct vas_tx_win_attr txattr = {}; + struct vas_tx_win_open_attr uattr; + struct coproc_instance *cp_inst; + struct vas_window *txwin; + int rc, vasid; + + cp_inst = fp->private_data; + + /* + * One window for file descriptor + */ + if (cp_inst->txwin) + return -EEXIST; + + rc = copy_from_user(&uattr, uptr, sizeof(uattr)); + if (rc) { + pr_err("%s(): copy_from_user() returns %d\n", __func__, rc); + return -EFAULT; + } + + if (uattr.version != 1) { + pr_err("Invalid version\n"); + return -EINVAL; + } + + vasid = uattr.vas_id; + + vas_init_tx_win_attr(&txattr, cp_inst->coproc->cop_type); + + txattr.lpid = mfspr(SPRN_LPID); + txattr.pidr = mfspr(SPRN_PID); + txattr.user_win = true; + txattr.rsvd_txbuf_count = false; + txattr.pswid = false; + + pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr, + mfspr(SPRN_PID)); + + txwin = vas_tx_win_open(vasid, cp_inst->coproc->cop_type, &txattr); + if (IS_ERR(txwin)) { + pr_err("%s() vas_tx_win_open() failed, %ld\n", __func__, + PTR_ERR(txwin)); + return PTR_ERR(txwin); + } + + cp_inst->txwin = txwin; + + return 0; +} + +static int coproc_release(struct inode *inode, struct file *fp) +{ + struct coproc_instance *cp_inst = fp->private_data; + + if (cp_inst->txwin) { + vas_win_close(cp_inst->txwin); + cp_inst->txwin = NULL; + } + + kfree(cp_inst); + fp->private_data = NULL; + + /* + * We don't know here if user has other receive windows + * open, so we can't really call clear_thread_tidr(). + * So, once the process calls set_thread_tidr(), the + * TIDR value sticks around until process exits, resulting + * in an extra copy in restore_sprs(). + */ + + return 0; +} + +static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) +{ + struct coproc_instance *cp_inst = fp->private_data; + struct vas_window *txwin; + unsigned long pfn; + u64 paste_addr; + pgprot_t prot; + int rc; + + txwin = cp_inst->txwin; + + if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { + pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__, + (vma->vm_end - vma->vm_start), PAGE_SIZE); + return -EINVAL; + } + + /* Ensure instance has an open send window */ + if (!txwin) { + pr_err("%s(): No send window open?\n", __func__); + return -EINVAL; + } + + vas_win_paste_addr(txwin, &paste_addr, NULL); + pfn = paste_addr >> PAGE_SHIFT; + + /* flags, page_prot from cxl_mmap(), except we want cachable */ + vma->vm_flags |= VM_IO | VM_PFNMAP; + vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); + + prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY); + + rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + vma->vm_end - vma->vm_start, prot); + + pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__, + paste_addr, vma->vm_start, rc); + + return rc; +} + +static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case VAS_TX_WIN_OPEN: + return coproc_ioc_tx_win_open(fp, arg); + default: + return -EINVAL; + } +} + +static struct file_operations coproc_fops = { + .open = coproc_open, + .release = coproc_release, + .mmap = coproc_mmap, + .unlocked_ioctl = coproc_ioctl, +}; + +/* + * Supporting only nx-gzip coprocessor type now, but this API code + * extended to other coprocessor types later. + */ +int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, + const char *name) +{ + int rc = -EINVAL; + dev_t devno; + + rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name); + if (rc) { + pr_err("Unable to allocate coproc major number: %i\n", rc); + return rc; + } + + pr_devel("%s device allocated, dev [%i,%i]\n", name, + MAJOR(coproc_device.devt), MINOR(coproc_device.devt)); + + coproc_device.class = class_create(mod, name); + if (IS_ERR(coproc_device.class)) { + rc = PTR_ERR(coproc_device.class); + pr_err("Unable to create %s class %d\n", name, rc); + goto err_class; + } + coproc_device.class->devnode = coproc_devnode; + coproc_device.cop_type = cop_type; + + coproc_fops.owner = mod; + cdev_init(&coproc_device.cdev, &coproc_fops); + + devno = MKDEV(MAJOR(coproc_device.devt), 0); + rc = cdev_add(&coproc_device.cdev, devno, 1); + if (rc) { + pr_err("cdev_add() failed %d\n", rc); + goto err_cdev; + } + + coproc_device.device = device_create(coproc_device.class, NULL, + devno, NULL, name, MINOR(devno)); + if (IS_ERR(coproc_device.device)) { + rc = PTR_ERR(coproc_device.device); + pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc); + goto err; + } + + pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno), + MINOR(devno)); + + return 0; + +err: + cdev_del(&coproc_device.cdev); +err_cdev: + class_destroy(coproc_device.class); +err_class: + unregister_chrdev_region(coproc_device.devt, 1); + return rc; +} +EXPORT_SYMBOL_GPL(vas_register_coproc_api); + +void vas_unregister_coproc_api(void) +{ + dev_t devno; + + cdev_del(&coproc_device.cdev); + devno = MKDEV(MAJOR(coproc_device.devt), 0); + device_destroy(coproc_device.class, devno); + + class_destroy(coproc_device.class); + unregister_chrdev_region(coproc_device.devt, 1); +} +EXPORT_SYMBOL_GPL(vas_unregister_coproc_api); diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c index d62787f502c9..6434f9cb5aed 100644 --- a/arch/powerpc/platforms/powernv/vas-window.c +++ b/arch/powerpc/platforms/powernv/vas-window.c @@ -26,7 +26,7 @@ * Compute the paste address region for the window @window using the * ->paste_base_addr and ->paste_win_id_shift we got from device tree. */ -static void compute_paste_address(struct vas_window *window, u64 *addr, int *len) +void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len) { int winid; u64 base, shift; @@ -80,7 +80,7 @@ static void *map_paste_region(struct vas_window *txwin) goto free_name; txwin->paste_addr_name = name; - compute_paste_address(txwin, &start, &len); + vas_win_paste_addr(txwin, &start, &len); if (!request_mem_region(start, len, name)) { pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", @@ -138,7 +138,7 @@ static void unmap_paste_region(struct vas_window *window) u64 busaddr_start; if (window->paste_kaddr) { - compute_paste_address(window, &busaddr_start, &len); + vas_win_paste_addr(window, &busaddr_start, &len); unmap_region(window->paste_kaddr, busaddr_start, len); window->paste_kaddr = NULL; kfree(window->paste_addr_name); @@ -817,7 +817,8 @@ void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop) { memset(rxattr, 0, sizeof(*rxattr)); - if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { + if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI || + cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) { rxattr->pin_win = true; rxattr->nx_win = true; rxattr->fault_win = false; @@ -892,7 +893,8 @@ void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop) { memset(txattr, 0, sizeof(*txattr)); - if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { + if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI || + cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) { txattr->rej_no_credit = false; txattr->rx_wcred_mode = true; txattr->tx_wcred_mode = true; @@ -976,9 +978,14 @@ static bool tx_win_args_valid(enum vas_cop_type cop, if (attr->wcreds_max > VAS_TX_WCREDS_MAX) return false; - if (attr->user_win && - (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count)) - return false; + if (attr->user_win) { + if (attr->rsvd_txbuf_count) + return false; + + if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP && + cop != VAS_COP_TYPE_GZIP_HIPRI) + return false; + } return true; } diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h index a7143b16232f..70f793e8f6cc 100644 --- a/arch/powerpc/platforms/powernv/vas.h +++ b/arch/powerpc/platforms/powernv/vas.h @@ -437,6 +437,8 @@ extern irqreturn_t vas_fault_handler(int irq, void *dev_id); extern void vas_return_credit(struct vas_window *window, bool tx); extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst, uint32_t pswid); +extern void vas_win_paste_addr(struct vas_window *window, u64 *addr, + int *len); static inline int vas_window_pid(struct vas_window *window) { diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile index 015155da59c2..bc89a20e5d9d 100644 --- a/drivers/crypto/nx/Makefile +++ b/drivers/crypto/nx/Makefile @@ -15,4 +15,4 @@ obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compres obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o nx-compress-objs := nx-842.o nx-compress-pseries-objs := nx-842-pseries.o -nx-compress-powernv-objs := nx-842-powernv.o +nx-compress-powernv-objs := nx-common-powernv.o diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-common-powernv.c similarity index 87% rename from drivers/crypto/nx/nx-842-powernv.c rename to drivers/crypto/nx/nx-common-powernv.c index c037a2403b82..13c65deda8e9 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Driver for IBM PowerNV 842 compression accelerator + * Driver for IBM PowerNV compression accelerator * * Copyright (C) 2015 Dan Streetman, IBM Corp */ @@ -20,7 +20,7 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman "); -MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors"); +MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors"); MODULE_ALIAS_CRYPTO("842"); MODULE_ALIAS_CRYPTO("842-nx"); @@ -40,9 +40,9 @@ struct nx842_workmem { char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */ } __packed __aligned(WORKMEM_ALIGN); -struct nx842_coproc { +struct nx_coproc { unsigned int chip_id; - unsigned int ct; + unsigned int ct; /* Can be 842 or GZIP high/normal*/ unsigned int ci; /* Coprocessor instance, used with icswx */ struct { struct vas_window *rxwin; @@ -58,9 +58,16 @@ struct nx842_coproc { static DEFINE_PER_CPU(struct vas_window *, cpu_txwin); /* no cpu hotplug on powernv, so this list never changes after init */ -static LIST_HEAD(nx842_coprocs); +static LIST_HEAD(nx_coprocs); static unsigned int nx842_ct; /* used in icswx function */ +/* + * Using same values as in skiboot or coprocessor type representing + * in NX workbook. + */ +#define NX_CT_GZIP (2) /* on P9 and later */ +#define NX_CT_842 (3) + static int (*nx842_powernv_exec)(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *workmem, int fc); @@ -666,15 +673,15 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, wmem, CCW_FC_842_DECOMP_CRC); } -static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc, +static inline void nx_add_coprocs_list(struct nx_coproc *coproc, int chipid) { coproc->chip_id = chipid; INIT_LIST_HEAD(&coproc->list); - list_add(&coproc->list, &nx842_coprocs); + list_add(&coproc->list, &nx_coprocs); } -static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc) +static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc) { struct vas_window *txwin = NULL; struct vas_tx_win_attr txattr; @@ -685,7 +692,6 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc) */ vas_init_tx_win_attr(&txattr, coproc->ct); txattr.lpid = 0; /* lpid is 0 for kernel requests */ - txattr.pid = 0; /* pid is 0 for kernel requests */ /* * Open a VAS send window which is used to send request to NX. @@ -704,9 +710,9 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc) * cpu_txwin is used in copy/paste operation for each compression / * decompression request. */ -static int nx842_open_percpu_txwins(void) +static int nx_open_percpu_txwins(void) { - struct nx842_coproc *coproc, *n; + struct nx_coproc *coproc, *n; unsigned int i, chip_id; for_each_possible_cpu(i) { @@ -714,17 +720,18 @@ static int nx842_open_percpu_txwins(void) chip_id = cpu_to_chip_id(i); - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + list_for_each_entry_safe(coproc, n, &nx_coprocs, list) { /* * Kernel requests use only high priority FIFOs. So * open send windows for these FIFOs. + * GZIP is not supported in kernel right now. */ if (coproc->ct != VAS_COP_TYPE_842_HIPRI) continue; if (coproc->chip_id == chip_id) { - txwin = nx842_alloc_txwin(coproc); + txwin = nx_alloc_txwin(coproc); if (IS_ERR(txwin)) return PTR_ERR(txwin); @@ -743,13 +750,28 @@ static int nx842_open_percpu_txwins(void) return 0; } +static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority, + int high, int normal) +{ + if (!strcmp(priority, "High")) + coproc->ct = high; + else if (!strcmp(priority, "Normal")) + coproc->ct = normal; + else { + pr_err("Invalid RxFIFO priority value\n"); + return -EINVAL; + } + + return 0; +} + static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, - int vasid, int *ct) + int vasid, int type, int *ct) { struct vas_window *rxwin = NULL; struct vas_rx_win_attr rxattr; - struct nx842_coproc *coproc; u32 lpid, pid, tid, fifo_size; + struct nx_coproc *coproc; u64 rx_fifo; const char *priority; int ret; @@ -794,15 +816,15 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, if (!coproc) return -ENOMEM; - if (!strcmp(priority, "High")) - coproc->ct = VAS_COP_TYPE_842_HIPRI; - else if (!strcmp(priority, "Normal")) - coproc->ct = VAS_COP_TYPE_842; - else { - pr_err("Invalid RxFIFO priority value\n"); - ret = -EINVAL; + if (type == NX_CT_842) + ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI, + VAS_COP_TYPE_842); + else if (type == NX_CT_GZIP) + ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI, + VAS_COP_TYPE_GZIP); + + if (ret) goto err_out; - } vas_init_rx_win_attr(&rxattr, coproc->ct); rxattr.rx_fifo = (void *)rx_fifo; @@ -830,7 +852,7 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, coproc->vas.rxwin = rxwin; coproc->vas.id = vasid; - nx842_add_coprocs_list(coproc, chip_id); + nx_add_coprocs_list(coproc, chip_id); /* * (lpid, pid, tid) combination has to be unique for each @@ -848,13 +870,47 @@ err_out: return ret; } - -static int __init nx842_powernv_probe_vas(struct device_node *pn) +static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip) +{ + int ret = 0; + + if (opal_check_token(OPAL_NX_COPROC_INIT)) { + ret = opal_nx_coproc_init(chip_id, ct_842); + + if (!ret) + ret = opal_nx_coproc_init(chip_id, ct_gzip); + + if (ret) { + ret = opal_error_code(ret); + pr_err("Failed to initialize NX for chip(%d): %d\n", + chip_id, ret); + } + } else + pr_warn("Firmware doesn't support NX initialization\n"); + + return ret; +} + +static int __init find_nx_device_tree(struct device_node *dn, int chip_id, + int vasid, int type, char *devname, + int *ct) +{ + int ret = 0; + + if (of_device_is_compatible(dn, devname)) { + ret = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct); + if (ret) + of_node_put(dn); + } + + return ret; +} + +static int __init nx_powernv_probe_vas(struct device_node *pn) { - struct device_node *dn; int chip_id, vasid, ret = 0; - int nx_fifo_found = 0; - int uninitialized_var(ct); + int ct_842 = 0, ct_gzip = 0; + struct device_node *dn; chip_id = of_get_ibm_chip_id(pn); if (chip_id < 0) { @@ -869,40 +925,33 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn) } for_each_child_of_node(pn, dn) { - if (of_device_is_compatible(dn, "ibm,p9-nx-842")) { - ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct); - if (ret) { - of_node_put(dn); - return ret; - } - nx_fifo_found++; - } + ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842, + "ibm,p9-nx-842", &ct_842); + + if (!ret) + ret = find_nx_device_tree(dn, chip_id, vasid, + NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip); + + if (ret) + return ret; } - if (!nx_fifo_found) { - pr_err("NX842 FIFO nodes are missing\n"); + if (!ct_842 || !ct_gzip) { + pr_err("NX FIFO nodes are missing\n"); return -EINVAL; } /* * Initialize NX instance for both high and normal priority FIFOs. */ - if (opal_check_token(OPAL_NX_COPROC_INIT)) { - ret = opal_nx_coproc_init(chip_id, ct); - if (ret) { - pr_err("Failed to initialize NX for chip(%d): %d\n", - chip_id, ret); - ret = opal_error_code(ret); - } - } else - pr_warn("Firmware doesn't support NX initialization\n"); + ret = nx_coproc_init(chip_id, ct_842, ct_gzip); return ret; } static int __init nx842_powernv_probe(struct device_node *dn) { - struct nx842_coproc *coproc; + struct nx_coproc *coproc; unsigned int ct, ci; int chip_id; @@ -922,13 +971,13 @@ static int __init nx842_powernv_probe(struct device_node *dn) return -EINVAL; } - coproc = kmalloc(sizeof(*coproc), GFP_KERNEL); + coproc = kzalloc(sizeof(*coproc), GFP_KERNEL); if (!coproc) return -ENOMEM; coproc->ct = ct; coproc->ci = ci; - nx842_add_coprocs_list(coproc, chip_id); + nx_add_coprocs_list(coproc, chip_id); pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci); @@ -941,9 +990,9 @@ static int __init nx842_powernv_probe(struct device_node *dn) return 0; } -static void nx842_delete_coprocs(void) +static void nx_delete_coprocs(void) { - struct nx842_coproc *coproc, *n; + struct nx_coproc *coproc, *n; struct vas_window *txwin; int i; @@ -955,10 +1004,10 @@ static void nx842_delete_coprocs(void) if (txwin) vas_win_close(txwin); - per_cpu(cpu_txwin, i) = 0; + per_cpu(cpu_txwin, i) = NULL; } - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + list_for_each_entry_safe(coproc, n, &nx_coprocs, list) { if (coproc->vas.rxwin) vas_win_close(coproc->vas.rxwin); @@ -1002,7 +1051,7 @@ static struct crypto_alg nx842_powernv_alg = { .coa_decompress = nx842_crypto_decompress } } }; -static __init int nx842_powernv_init(void) +static __init int nx_compress_powernv_init(void) { struct device_node *dn; int ret; @@ -1017,15 +1066,15 @@ static __init int nx842_powernv_init(void) BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); for_each_compatible_node(dn, NULL, "ibm,power9-nx") { - ret = nx842_powernv_probe_vas(dn); + ret = nx_powernv_probe_vas(dn); if (ret) { - nx842_delete_coprocs(); + nx_delete_coprocs(); of_node_put(dn); return ret; } } - if (list_empty(&nx842_coprocs)) { + if (list_empty(&nx_coprocs)) { for_each_compatible_node(dn, NULL, "ibm,power-nx") nx842_powernv_probe(dn); @@ -1034,9 +1083,25 @@ static __init int nx842_powernv_init(void) nx842_powernv_exec = nx842_exec_icswx; } else { - ret = nx842_open_percpu_txwins(); + /* + * Register VAS user space API for NX GZIP so + * that user space can use GZIP engine. + * Using high FIFO priority for kernel requests and + * normal FIFO priority is assigned for userspace. + * 842 compression is supported only in kernel. + */ + ret = vas_register_coproc_api(THIS_MODULE, VAS_COP_TYPE_GZIP, + "nx-gzip"); + + /* + * GZIP is not supported in kernel right now. + * So open tx windows only for 842. + */ + if (!ret) + ret = nx_open_percpu_txwins(); + if (ret) { - nx842_delete_coprocs(); + nx_delete_coprocs(); return ret; } @@ -1045,18 +1110,27 @@ static __init int nx842_powernv_init(void) ret = crypto_register_alg(&nx842_powernv_alg); if (ret) { - nx842_delete_coprocs(); + nx_delete_coprocs(); return ret; } return 0; } -module_init(nx842_powernv_init); +module_init(nx_compress_powernv_init); -static void __exit nx842_powernv_exit(void) +static void __exit nx_compress_powernv_exit(void) { + /* + * GZIP engine is supported only in power9 or later and nx842_ct + * is used on power8 (icswx). + * VAS API for NX GZIP is registered during init for user space + * use. So delete this API use for GZIP engine. + */ + if (!nx842_ct) + vas_unregister_coproc_api(); + crypto_unregister_alg(&nx842_powernv_alg); - nx842_delete_coprocs(); + nx_delete_coprocs(); } -module_exit(nx842_powernv_exit); +module_exit(nx_compress_powernv_exit);