linux/drivers/cxl/core.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include "cxl.h"

/**
 * DOC: cxl core
 *
 * The CXL core provides a sysfs hierarchy for control devices and a rendezvous
 * point for cross-device interleave coordination through cxl ports.
 */

static DEFINE_IDA(cxl_port_ida);

static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	return sysfs_emit(buf, "%s\n", dev->type->name);
}
static DEVICE_ATTR_RO(devtype);

static struct attribute *cxl_base_attributes[] = {
	&dev_attr_devtype.attr,
	NULL,
};

static struct attribute_group cxl_base_attribute_group = {
	.attrs = cxl_base_attributes,
};

static void cxl_dport_release(struct cxl_dport *dport)
{
	list_del(&dport->list);
	put_device(dport->dport);
	kfree(dport);
}

static void cxl_port_release(struct device *dev)
{
	struct cxl_port *port = to_cxl_port(dev);
	struct cxl_dport *dport, *_d;

	device_lock(dev);
	list_for_each_entry_safe(dport, _d, &port->dports, list)
		cxl_dport_release(dport);
	device_unlock(dev);
	ida_free(&cxl_port_ida, port->id);
	kfree(port);
}

static const struct attribute_group *cxl_port_attribute_groups[] = {
	&cxl_base_attribute_group,
	NULL,
};

static const struct device_type cxl_port_type = {
	.name = "cxl_port",
	.release = cxl_port_release,
	.groups = cxl_port_attribute_groups,
};

struct cxl_port *to_cxl_port(struct device *dev)
{
	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
			  "not a cxl_port device\n"))
		return NULL;
	return container_of(dev, struct cxl_port, dev);
}

static void unregister_port(void *_port)
{
	struct cxl_port *port = _port;
	struct cxl_dport *dport;

	device_lock(&port->dev);
	list_for_each_entry(dport, &port->dports, list) {
		char link_name[CXL_TARGET_STRLEN];

		if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d",
			     dport->port_id) >= CXL_TARGET_STRLEN)
			continue;
		sysfs_remove_link(&port->dev.kobj, link_name);
	}
	device_unlock(&port->dev);
	device_unregister(&port->dev);
}

static void cxl_unlink_uport(void *_port)
{
	struct cxl_port *port = _port;

	sysfs_remove_link(&port->dev.kobj, "uport");
}

static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
{
	int rc;

	rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
	if (rc)
		return rc;
	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
}

static struct cxl_port *cxl_port_alloc(struct device *uport,
				       resource_size_t component_reg_phys,
				       struct cxl_port *parent_port)
{
	struct cxl_port *port;
	struct device *dev;
	int rc;

	port = kzalloc(sizeof(*port), GFP_KERNEL);
	if (!port)
		return ERR_PTR(-ENOMEM);

	rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
	if (rc < 0)
		goto err;
	port->id = rc;

	/*
	 * The top-level cxl_port "cxl_root" does not have a cxl_port as
	 * its parent and it does not have any corresponding component
	 * registers as its decode is described by a fixed platform
	 * description.
	 */
	dev = &port->dev;
	if (parent_port)
		dev->parent = &parent_port->dev;
	else
		dev->parent = uport;

	port->uport = uport;
	port->component_reg_phys = component_reg_phys;
	INIT_LIST_HEAD(&port->dports);

	device_initialize(dev);
	device_set_pm_not_required(dev);
	dev->bus = &cxl_bus_type;
	dev->type = &cxl_port_type;

	return port;

err:
	kfree(port);
	return ERR_PTR(rc);
}

/**
 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
 * @host: host device for devm operations
 * @uport: "physical" device implementing this upstream port
 * @component_reg_phys: (optional) for configurable cxl_port instances
 * @parent_port: next hop up in the CXL memory decode hierarchy
 */
struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
				   resource_size_t component_reg_phys,
				   struct cxl_port *parent_port)
{
	struct cxl_port *port;
	struct device *dev;
	int rc;

	port = cxl_port_alloc(uport, component_reg_phys, parent_port);
	if (IS_ERR(port))
		return port;

	dev = &port->dev;
	if (parent_port)
		rc = dev_set_name(dev, "port%d", port->id);
	else
		rc = dev_set_name(dev, "root%d", port->id);
	if (rc)
		goto err;

	rc = device_add(dev);
	if (rc)
		goto err;

	rc = devm_add_action_or_reset(host, unregister_port, port);
	if (rc)
		return ERR_PTR(rc);

	rc = devm_cxl_link_uport(host, port);
	if (rc)
		return ERR_PTR(rc);

	return port;

err:
	put_device(dev);
	return ERR_PTR(rc);
}
EXPORT_SYMBOL_GPL(devm_cxl_add_port);

static struct cxl_dport *find_dport(struct cxl_port *port, int id)
{
	struct cxl_dport *dport;

	device_lock_assert(&port->dev);
	list_for_each_entry (dport, &port->dports, list)
		if (dport->port_id == id)
			return dport;
	return NULL;
}

static int add_dport(struct cxl_port *port, struct cxl_dport *new)
{
	struct cxl_dport *dup;

	device_lock(&port->dev);
	dup = find_dport(port, new->port_id);
	if (dup)
		dev_err(&port->dev,
			"unable to add dport%d-%s non-unique port id (%s)\n",
			new->port_id, dev_name(new->dport),
			dev_name(dup->dport));
	else
		list_add_tail(&new->list, &port->dports);
	device_unlock(&port->dev);

	return dup ? -EEXIST : 0;
}

/**
 * cxl_add_dport - append downstream port data to a cxl_port
 * @port: the cxl_port that references this dport
 * @dport_dev: firmware or PCI device representing the dport
 * @port_id: identifier for this dport in a decoder's target list
 * @component_reg_phys: optional location of CXL component registers
 *
 * Note that all allocations and links are undone by cxl_port deletion
 * and release.
 */
int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
		  resource_size_t component_reg_phys)
{
	char link_name[CXL_TARGET_STRLEN];
	struct cxl_dport *dport;
	int rc;

	if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
	    CXL_TARGET_STRLEN)
		return -EINVAL;

	dport = kzalloc(sizeof(*dport), GFP_KERNEL);
	if (!dport)
		return -ENOMEM;

	INIT_LIST_HEAD(&dport->list);
	dport->dport = get_device(dport_dev);
	dport->port_id = port_id;
	dport->component_reg_phys = component_reg_phys;
	dport->port = port;

	rc = add_dport(port, dport);
	if (rc)
		goto err;

	rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
	if (rc)
		goto err;

	return 0;
err:
	cxl_dport_release(dport);
	return rc;
}
EXPORT_SYMBOL_GPL(cxl_add_dport);

/**
 * cxl_probe_component_regs() - Detect CXL Component register blocks
 * @dev: Host device of the @base mapping
 * @base: Mapping containing the HDM Decoder Capability Header
 * @map: Map object describing the register block information found
 *
 * See CXL 2.0 8.2.4 Component Register Layout and Definition
 * See CXL 2.0 8.2.5.5 CXL Device Register Interface
 *
 * Probe for component register information and return it in map object.
 */
void cxl_probe_component_regs(struct device *dev, void __iomem *base,
			      struct cxl_component_reg_map *map)
{
	int cap, cap_count;
	u64 cap_array;

	*map = (struct cxl_component_reg_map) { 0 };

	/*
	 * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in
	 * CXL 2.0 8.2.4 Table 141.
	 */
	base += CXL_CM_OFFSET;

	cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET);

	if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
		dev_err(dev,
			"Couldn't locate the CXL.cache and CXL.mem capability array header./n");
		return;
	}

	/* It's assumed that future versions will be backward compatible */
	cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array);

	for (cap = 1; cap <= cap_count; cap++) {
		void __iomem *register_block;
		u32 hdr;
		int decoder_cnt;
		u16 cap_id, offset;
		u32 length;

		hdr = readl(base + cap * 0x4);

		cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr);
		offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr);
		register_block = base + offset;

		switch (cap_id) {
		case CXL_CM_CAP_CAP_ID_HDM:
			dev_dbg(dev, "found HDM decoder capability (0x%x)\n",
				offset);

			hdr = readl(register_block);

			decoder_cnt = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, hdr);
			length = 0x20 * decoder_cnt + 0x10;

			map->hdm_decoder.valid = true;
			map->hdm_decoder.offset = offset;
			map->hdm_decoder.size = length;
			break;
		default:
			dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
				offset);
			break;
		}
	}
}
EXPORT_SYMBOL_GPL(cxl_probe_component_regs);

/**
 * cxl_probe_device_regs() - Detect CXL Device register blocks
 * @dev: Host device of the @base mapping
 * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface
 * @map: Map object describing the register block information found
 *
 * Probe for device register information and return it in map object.
 */
void cxl_probe_device_regs(struct device *dev, void __iomem *base,
			   struct cxl_device_reg_map *map)
{
	int cap, cap_count;
	u64 cap_array;

	*map = (struct cxl_device_reg_map){ 0 };

	cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET);
	if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) !=
	    CXLDEV_CAP_ARRAY_CAP_ID)
		return;

	cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array);

	for (cap = 1; cap <= cap_count; cap++) {
		u32 offset, length;
		u16 cap_id;

		cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK,
				   readl(base + cap * 0x10));
		offset = readl(base + cap * 0x10 + 0x4);
		length = readl(base + cap * 0x10 + 0x8);

		switch (cap_id) {
		case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
			dev_dbg(dev, "found Status capability (0x%x)\n", offset);

			map->status.valid = true;
			map->status.offset = offset;
			map->status.size = length;
			break;
		case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
			dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
			map->mbox.valid = true;
			map->mbox.offset = offset;
			map->mbox.size = length;
			break;
		case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
			dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
			break;
		case CXLDEV_CAP_CAP_ID_MEMDEV:
			dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
			map->memdev.valid = true;
			map->memdev.offset = offset;
			map->memdev.size = length;
			break;
		default:
			if (cap_id >= 0x8000)
				dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset);
			else
				dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset);
			break;
		}
	}
}
EXPORT_SYMBOL_GPL(cxl_probe_device_regs);

static void __iomem *devm_cxl_iomap_block(struct device *dev,
					  resource_size_t addr,
					  resource_size_t length)
{
	void __iomem *ret_val;
	struct resource *res;

	res = devm_request_mem_region(dev, addr, length, dev_name(dev));
	if (!res) {
		resource_size_t end = addr + length - 1;

		dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end);
		return NULL;
	}

	ret_val = devm_ioremap(dev, addr, length);
	if (!ret_val)
		dev_err(dev, "Failed to map region %pr\n", res);

	return ret_val;
}

int cxl_map_component_regs(struct pci_dev *pdev,
			   struct cxl_component_regs *regs,
			   struct cxl_register_map *map)
{
	struct device *dev = &pdev->dev;
	resource_size_t phys_addr;
	resource_size_t length;

	phys_addr = pci_resource_start(pdev, map->barno);
	phys_addr += map->block_offset;

	phys_addr += map->component_map.hdm_decoder.offset;
	length = map->component_map.hdm_decoder.size;
	regs->hdm_decoder = devm_cxl_iomap_block(dev, phys_addr, length);
	if (!regs->hdm_decoder)
		return -ENOMEM;

	return 0;
}
EXPORT_SYMBOL_GPL(cxl_map_component_regs);

int cxl_map_device_regs(struct pci_dev *pdev,
			struct cxl_device_regs *regs,
			struct cxl_register_map *map)
{
	struct device *dev = &pdev->dev;
	resource_size_t phys_addr;

	phys_addr = pci_resource_start(pdev, map->barno);
	phys_addr += map->block_offset;

	if (map->device_map.status.valid) {
		resource_size_t addr;
		resource_size_t length;

		addr = phys_addr + map->device_map.status.offset;
		length = map->device_map.status.size;
		regs->status = devm_cxl_iomap_block(dev, addr, length);
		if (!regs->status)
			return -ENOMEM;
	}

	if (map->device_map.mbox.valid) {
		resource_size_t addr;
		resource_size_t length;

		addr = phys_addr + map->device_map.mbox.offset;
		length = map->device_map.mbox.size;
		regs->mbox = devm_cxl_iomap_block(dev, addr, length);
		if (!regs->mbox)
			return -ENOMEM;
	}

	if (map->device_map.memdev.valid) {
		resource_size_t addr;
		resource_size_t length;

		addr = phys_addr + map->device_map.memdev.offset;
		length = map->device_map.memdev.size;
		regs->memdev = devm_cxl_iomap_block(dev, addr, length);
		if (!regs->memdev)
			return -ENOMEM;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(cxl_map_device_regs);

struct bus_type cxl_bus_type = {
	.name = "cxl",
};
EXPORT_SYMBOL_GPL(cxl_bus_type);

static __init int cxl_core_init(void)
{
	return bus_register(&cxl_bus_type);
}

static void cxl_core_exit(void)
{
	bus_unregister(&cxl_bus_type);
}

module_init(cxl_core_init);
module_exit(cxl_core_exit);
MODULE_LICENSE("GPL v2");