mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-25 12:04:46 +08:00
6e7f3ee97b
The original architecture of /sys/bus/dsa invented a scheme whereby a single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled all device types and internally routed them to different different drivers. Those internal drivers were invisible to userspace. With the idxd driver transitioned to a proper bus device-driver model, the legacy behavior needs to be preserved due to it being exposed to user space via sysfs. Create a compat driver to provide the legacy behavior for /sys/bus/dsa/drivers/dsa. This should satisfy user tool accel-config v3.2 or ealier where this behavior is expected. If the distro has a newer accel-config then the legacy mode does not need to be enabled. When the compat driver binds the device (i.e. dsa0) to the dsa driver, it will be bound to the new idxd_drv. The wq device (i.e. wq0.0) will be bound to either the dmaengine_drv or the user_drv. The dsa_drv becomes a routing mechansim for the new drivers. It will not support additional external drivers that are implemented later. Reviewed-by: Dan Williams <dan.j.williams@intel.com> Signed-off-by: Dave Jiang <dave.jiang@intel.com> Link: https://lore.kernel.org/r/162637468705.744545.4399080971745974435.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul <vkoul@kernel.org>
385 lines
8.2 KiB
C
385 lines
8.2 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
|
|
#include <linux/init.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/device.h>
|
|
#include <linux/sched/task.h>
|
|
#include <linux/intel-svm.h>
|
|
#include <linux/io-64-nonatomic-lo-hi.h>
|
|
#include <linux/cdev.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/poll.h>
|
|
#include <linux/iommu.h>
|
|
#include <uapi/linux/idxd.h>
|
|
#include "registers.h"
|
|
#include "idxd.h"
|
|
|
|
struct idxd_cdev_context {
|
|
const char *name;
|
|
dev_t devt;
|
|
struct ida minor_ida;
|
|
};
|
|
|
|
/*
|
|
* ictx is an array based off of accelerator types. enum idxd_type
|
|
* is used as index
|
|
*/
|
|
static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
|
|
{ .name = "dsa" },
|
|
{ .name = "iax" }
|
|
};
|
|
|
|
struct idxd_user_context {
|
|
struct idxd_wq *wq;
|
|
struct task_struct *task;
|
|
unsigned int pasid;
|
|
unsigned int flags;
|
|
struct iommu_sva *sva;
|
|
};
|
|
|
|
static void idxd_cdev_dev_release(struct device *dev)
|
|
{
|
|
struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
|
|
struct idxd_cdev_context *cdev_ctx;
|
|
struct idxd_wq *wq = idxd_cdev->wq;
|
|
|
|
cdev_ctx = &ictx[wq->idxd->data->type];
|
|
ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
|
|
kfree(idxd_cdev);
|
|
}
|
|
|
|
static struct device_type idxd_cdev_device_type = {
|
|
.name = "idxd_cdev",
|
|
.release = idxd_cdev_dev_release,
|
|
};
|
|
|
|
static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
|
|
{
|
|
struct cdev *cdev = inode->i_cdev;
|
|
|
|
return container_of(cdev, struct idxd_cdev, cdev);
|
|
}
|
|
|
|
static inline struct idxd_wq *inode_wq(struct inode *inode)
|
|
{
|
|
struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
|
|
|
|
return idxd_cdev->wq;
|
|
}
|
|
|
|
static int idxd_cdev_open(struct inode *inode, struct file *filp)
|
|
{
|
|
struct idxd_user_context *ctx;
|
|
struct idxd_device *idxd;
|
|
struct idxd_wq *wq;
|
|
struct device *dev;
|
|
int rc = 0;
|
|
struct iommu_sva *sva;
|
|
unsigned int pasid;
|
|
|
|
wq = inode_wq(inode);
|
|
idxd = wq->idxd;
|
|
dev = &idxd->pdev->dev;
|
|
|
|
dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
|
|
|
|
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
|
|
if (!ctx)
|
|
return -ENOMEM;
|
|
|
|
mutex_lock(&wq->wq_lock);
|
|
|
|
if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
|
|
rc = -EBUSY;
|
|
goto failed;
|
|
}
|
|
|
|
ctx->wq = wq;
|
|
filp->private_data = ctx;
|
|
|
|
if (device_pasid_enabled(idxd)) {
|
|
sva = iommu_sva_bind_device(dev, current->mm, NULL);
|
|
if (IS_ERR(sva)) {
|
|
rc = PTR_ERR(sva);
|
|
dev_err(dev, "pasid allocation failed: %d\n", rc);
|
|
goto failed;
|
|
}
|
|
|
|
pasid = iommu_sva_get_pasid(sva);
|
|
if (pasid == IOMMU_PASID_INVALID) {
|
|
iommu_sva_unbind_device(sva);
|
|
rc = -EINVAL;
|
|
goto failed;
|
|
}
|
|
|
|
ctx->sva = sva;
|
|
ctx->pasid = pasid;
|
|
|
|
if (wq_dedicated(wq)) {
|
|
rc = idxd_wq_set_pasid(wq, pasid);
|
|
if (rc < 0) {
|
|
iommu_sva_unbind_device(sva);
|
|
dev_err(dev, "wq set pasid failed: %d\n", rc);
|
|
goto failed;
|
|
}
|
|
}
|
|
}
|
|
|
|
idxd_wq_get(wq);
|
|
mutex_unlock(&wq->wq_lock);
|
|
return 0;
|
|
|
|
failed:
|
|
mutex_unlock(&wq->wq_lock);
|
|
kfree(ctx);
|
|
return rc;
|
|
}
|
|
|
|
static int idxd_cdev_release(struct inode *node, struct file *filep)
|
|
{
|
|
struct idxd_user_context *ctx = filep->private_data;
|
|
struct idxd_wq *wq = ctx->wq;
|
|
struct idxd_device *idxd = wq->idxd;
|
|
struct device *dev = &idxd->pdev->dev;
|
|
int rc;
|
|
|
|
dev_dbg(dev, "%s called\n", __func__);
|
|
filep->private_data = NULL;
|
|
|
|
/* Wait for in-flight operations to complete. */
|
|
if (wq_shared(wq)) {
|
|
idxd_device_drain_pasid(idxd, ctx->pasid);
|
|
} else {
|
|
if (device_pasid_enabled(idxd)) {
|
|
/* The wq disable in the disable pasid function will drain the wq */
|
|
rc = idxd_wq_disable_pasid(wq);
|
|
if (rc < 0)
|
|
dev_err(dev, "wq disable pasid failed.\n");
|
|
} else {
|
|
idxd_wq_drain(wq);
|
|
}
|
|
}
|
|
|
|
if (ctx->sva)
|
|
iommu_sva_unbind_device(ctx->sva);
|
|
kfree(ctx);
|
|
mutex_lock(&wq->wq_lock);
|
|
idxd_wq_put(wq);
|
|
mutex_unlock(&wq->wq_lock);
|
|
return 0;
|
|
}
|
|
|
|
static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
|
|
const char *func)
|
|
{
|
|
struct device *dev = &wq->idxd->pdev->dev;
|
|
|
|
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
|
|
dev_info_ratelimited(dev,
|
|
"%s: %s: mapping too large: %lu\n",
|
|
current->comm, func,
|
|
vma->vm_end - vma->vm_start);
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
|
|
{
|
|
struct idxd_user_context *ctx = filp->private_data;
|
|
struct idxd_wq *wq = ctx->wq;
|
|
struct idxd_device *idxd = wq->idxd;
|
|
struct pci_dev *pdev = idxd->pdev;
|
|
phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
|
|
unsigned long pfn;
|
|
int rc;
|
|
|
|
dev_dbg(&pdev->dev, "%s called\n", __func__);
|
|
rc = check_vma(wq, vma, __func__);
|
|
if (rc < 0)
|
|
return rc;
|
|
|
|
vma->vm_flags |= VM_DONTCOPY;
|
|
pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
|
|
IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
|
|
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
|
|
vma->vm_private_data = ctx;
|
|
|
|
return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
|
|
vma->vm_page_prot);
|
|
}
|
|
|
|
static __poll_t idxd_cdev_poll(struct file *filp,
|
|
struct poll_table_struct *wait)
|
|
{
|
|
struct idxd_user_context *ctx = filp->private_data;
|
|
struct idxd_wq *wq = ctx->wq;
|
|
struct idxd_device *idxd = wq->idxd;
|
|
unsigned long flags;
|
|
__poll_t out = 0;
|
|
|
|
poll_wait(filp, &wq->err_queue, wait);
|
|
spin_lock_irqsave(&idxd->dev_lock, flags);
|
|
if (idxd->sw_err.valid)
|
|
out = EPOLLIN | EPOLLRDNORM;
|
|
spin_unlock_irqrestore(&idxd->dev_lock, flags);
|
|
|
|
return out;
|
|
}
|
|
|
|
static const struct file_operations idxd_cdev_fops = {
|
|
.owner = THIS_MODULE,
|
|
.open = idxd_cdev_open,
|
|
.release = idxd_cdev_release,
|
|
.mmap = idxd_cdev_mmap,
|
|
.poll = idxd_cdev_poll,
|
|
};
|
|
|
|
int idxd_cdev_get_major(struct idxd_device *idxd)
|
|
{
|
|
return MAJOR(ictx[idxd->data->type].devt);
|
|
}
|
|
|
|
int idxd_wq_add_cdev(struct idxd_wq *wq)
|
|
{
|
|
struct idxd_device *idxd = wq->idxd;
|
|
struct idxd_cdev *idxd_cdev;
|
|
struct cdev *cdev;
|
|
struct device *dev;
|
|
struct idxd_cdev_context *cdev_ctx;
|
|
int rc, minor;
|
|
|
|
idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
|
|
if (!idxd_cdev)
|
|
return -ENOMEM;
|
|
|
|
idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
|
|
idxd_cdev->wq = wq;
|
|
cdev = &idxd_cdev->cdev;
|
|
dev = cdev_dev(idxd_cdev);
|
|
cdev_ctx = &ictx[wq->idxd->data->type];
|
|
minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
|
|
if (minor < 0) {
|
|
kfree(idxd_cdev);
|
|
return minor;
|
|
}
|
|
idxd_cdev->minor = minor;
|
|
|
|
device_initialize(dev);
|
|
dev->parent = wq_confdev(wq);
|
|
dev->bus = &dsa_bus_type;
|
|
dev->type = &idxd_cdev_device_type;
|
|
dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
|
|
|
|
rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
|
|
if (rc < 0)
|
|
goto err;
|
|
|
|
wq->idxd_cdev = idxd_cdev;
|
|
cdev_init(cdev, &idxd_cdev_fops);
|
|
rc = cdev_device_add(cdev, dev);
|
|
if (rc) {
|
|
dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
|
|
goto err;
|
|
}
|
|
|
|
return 0;
|
|
|
|
err:
|
|
put_device(dev);
|
|
wq->idxd_cdev = NULL;
|
|
return rc;
|
|
}
|
|
|
|
void idxd_wq_del_cdev(struct idxd_wq *wq)
|
|
{
|
|
struct idxd_cdev *idxd_cdev;
|
|
|
|
idxd_cdev = wq->idxd_cdev;
|
|
wq->idxd_cdev = NULL;
|
|
cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
|
|
put_device(cdev_dev(idxd_cdev));
|
|
}
|
|
|
|
static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
|
|
{
|
|
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
|
|
struct idxd_device *idxd = wq->idxd;
|
|
int rc;
|
|
|
|
if (idxd->state != IDXD_DEV_ENABLED)
|
|
return -ENXIO;
|
|
|
|
mutex_lock(&wq->wq_lock);
|
|
wq->type = IDXD_WQT_USER;
|
|
rc = __drv_enable_wq(wq);
|
|
if (rc < 0)
|
|
goto err;
|
|
|
|
rc = idxd_wq_add_cdev(wq);
|
|
if (rc < 0)
|
|
goto err_cdev;
|
|
|
|
mutex_unlock(&wq->wq_lock);
|
|
return 0;
|
|
|
|
err_cdev:
|
|
__drv_disable_wq(wq);
|
|
err:
|
|
wq->type = IDXD_WQT_NONE;
|
|
mutex_unlock(&wq->wq_lock);
|
|
return rc;
|
|
}
|
|
|
|
static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
|
|
{
|
|
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
|
|
|
|
mutex_lock(&wq->wq_lock);
|
|
idxd_wq_del_cdev(wq);
|
|
__drv_disable_wq(wq);
|
|
wq->type = IDXD_WQT_NONE;
|
|
mutex_unlock(&wq->wq_lock);
|
|
}
|
|
|
|
static enum idxd_dev_type dev_types[] = {
|
|
IDXD_DEV_WQ,
|
|
IDXD_DEV_NONE,
|
|
};
|
|
|
|
struct idxd_device_driver idxd_user_drv = {
|
|
.probe = idxd_user_drv_probe,
|
|
.remove = idxd_user_drv_remove,
|
|
.name = "user",
|
|
.type = dev_types,
|
|
};
|
|
EXPORT_SYMBOL_GPL(idxd_user_drv);
|
|
|
|
int idxd_cdev_register(void)
|
|
{
|
|
int rc, i;
|
|
|
|
for (i = 0; i < IDXD_TYPE_MAX; i++) {
|
|
ida_init(&ictx[i].minor_ida);
|
|
rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
|
|
ictx[i].name);
|
|
if (rc)
|
|
return rc;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void idxd_cdev_remove(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < IDXD_TYPE_MAX; i++) {
|
|
unregister_chrdev_region(ictx[i].devt, MINORMASK);
|
|
ida_destroy(&ictx[i].minor_ida);
|
|
}
|
|
}
|