2021-02-17 12:09:50 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
|
|
|
|
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
|
2021-09-09 13:12:32 +08:00
|
|
|
#include <linux/io-64-nonatomic-lo-hi.h>
|
2021-02-17 12:09:50 +08:00
|
|
|
#include <linux/module.h>
|
2021-04-17 08:43:30 +08:00
|
|
|
#include <linux/sizes.h>
|
2021-02-17 12:09:52 +08:00
|
|
|
#include <linux/mutex.h>
|
2021-06-04 08:50:36 +08:00
|
|
|
#include <linux/list.h>
|
2021-02-17 12:09:50 +08:00
|
|
|
#include <linux/pci.h>
|
|
|
|
|
#include <linux/io.h>
|
2021-08-03 01:29:38 +08:00
|
|
|
#include "cxlmem.h"
|
2021-02-17 12:09:50 +08:00
|
|
|
#include "pci.h"
|
2021-02-17 12:09:51 +08:00
|
|
|
#include "cxl.h"
|
|
|
|
|
|
|
|
|
|
/**
|
cxl: Rename mem to pci
As the driver has undergone development, it's become clear that the
majority [entirety?] of the current functionality in mem.c is actually a
layer encapsulating functionality exposed through PCI based
interactions. This layer can be used either in isolation or to provide
functionality for higher level functionality.
CXL capabilities exist in a parallel domain to PCIe. CXL devices are
enumerable and controllable via "legacy" PCIe mechanisms; however, their
CXL capabilities are a superset of PCIe. For example, a CXL device may
be connected to a non-CXL capable PCIe root port, and therefore will not
be able to participate in CXL.mem or CXL.cache operations, but can still
be accessed through PCIe mechanisms for CXL.io operations.
To properly represent the PCI nature of this driver, and in preparation for
introducing a new driver for the CXL.mem / HDM decoder (Host-managed Device
Memory) capabilities of a CXL memory expander, rename mem.c to pci.c so that
mem.c is available for this new driver.
The result of the change is that there is a clear layering distinction
in the driver, and a systems administrator may load only the cxl_pci
module and gain access to such operations as, firmware update, offline
provisioning of devices, and error collection. In addition to freeing up
the file name for another purpose, there are two primary reasons this is
useful,
1. Acting upon devices which don't have full CXL capabilities. This
may happen for instance if the CXL device is connected in a CXL
unaware part of the platform topology.
2. Userspace-first provisioning for devices without kernel driver
interference. This may be useful when provisioning a new device
in a specific manner that might otherwise be blocked or prevented
by the real CXL mem driver.
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
Link: https://lore.kernel.org/r/20210526174413.802913-1-ben.widawsky@intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-27 01:44:13 +08:00
|
|
|
* DOC: cxl pci
|
2021-02-17 12:09:51 +08:00
|
|
|
*
|
cxl: Rename mem to pci
As the driver has undergone development, it's become clear that the
majority [entirety?] of the current functionality in mem.c is actually a
layer encapsulating functionality exposed through PCI based
interactions. This layer can be used either in isolation or to provide
functionality for higher level functionality.
CXL capabilities exist in a parallel domain to PCIe. CXL devices are
enumerable and controllable via "legacy" PCIe mechanisms; however, their
CXL capabilities are a superset of PCIe. For example, a CXL device may
be connected to a non-CXL capable PCIe root port, and therefore will not
be able to participate in CXL.mem or CXL.cache operations, but can still
be accessed through PCIe mechanisms for CXL.io operations.
To properly represent the PCI nature of this driver, and in preparation for
introducing a new driver for the CXL.mem / HDM decoder (Host-managed Device
Memory) capabilities of a CXL memory expander, rename mem.c to pci.c so that
mem.c is available for this new driver.
The result of the change is that there is a clear layering distinction
in the driver, and a systems administrator may load only the cxl_pci
module and gain access to such operations as, firmware update, offline
provisioning of devices, and error collection. In addition to freeing up
the file name for another purpose, there are two primary reasons this is
useful,
1. Acting upon devices which don't have full CXL capabilities. This
may happen for instance if the CXL device is connected in a CXL
unaware part of the platform topology.
2. Userspace-first provisioning for devices without kernel driver
interference. This may be useful when provisioning a new device
in a specific manner that might otherwise be blocked or prevented
by the real CXL mem driver.
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
Link: https://lore.kernel.org/r/20210526174413.802913-1-ben.widawsky@intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-27 01:44:13 +08:00
|
|
|
* This implements the PCI exclusive functionality for a CXL device as it is
|
|
|
|
|
* defined by the Compute Express Link specification. CXL devices may surface
|
2021-09-14 00:33:24 +08:00
|
|
|
* certain functionality even if it isn't CXL enabled. While this driver is
|
|
|
|
|
* focused around the PCI specific aspects of a CXL device, it binds to the
|
|
|
|
|
* specific CXL memory device class code, and therefore the implementation of
|
|
|
|
|
* cxl_pci is focused around CXL memory devices.
|
2021-02-17 12:09:51 +08:00
|
|
|
*
|
|
|
|
|
* The driver has several responsibilities, mainly:
|
|
|
|
|
* - Create the memX device and register on the CXL bus.
|
|
|
|
|
* - Enumerate device's register interface and map them.
|
2021-09-14 00:33:24 +08:00
|
|
|
* - Registers nvdimm bridge device with cxl_core.
|
|
|
|
|
* - Registers a CXL mailbox with cxl_core.
|
2021-02-17 12:09:51 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define cxl_doorbell_busy(cxlm) \
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
(readl((cxlm)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \
|
2021-02-17 12:09:51 +08:00
|
|
|
CXLDEV_MBOX_CTRL_DOORBELL)
|
|
|
|
|
|
|
|
|
|
/* CXL 2.0 - 8.2.8.4 */
|
|
|
|
|
#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_mbox_wait_for_doorbell(struct cxl_mem *cxlm)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
|
|
|
|
const unsigned long start = jiffies;
|
|
|
|
|
unsigned long end = start;
|
|
|
|
|
|
|
|
|
|
while (cxl_doorbell_busy(cxlm)) {
|
|
|
|
|
end = jiffies;
|
|
|
|
|
|
|
|
|
|
if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
|
|
|
|
|
/* Check again in case preempted before timeout test */
|
|
|
|
|
if (!cxl_doorbell_busy(cxlm))
|
|
|
|
|
break;
|
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
|
}
|
|
|
|
|
cpu_relax();
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_dbg(cxlm->dev, "Doorbell wait took %dms",
|
2021-02-17 12:09:51 +08:00
|
|
|
jiffies_to_msecs(end) - jiffies_to_msecs(start));
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static void cxl_pci_mbox_timeout(struct cxl_mem *cxlm,
|
2021-09-09 13:12:21 +08:00
|
|
|
struct cxl_mbox_cmd *mbox_cmd)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
2021-09-09 13:12:09 +08:00
|
|
|
struct device *dev = cxlm->dev;
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
|
dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n",
|
|
|
|
|
mbox_cmd->opcode, mbox_cmd->size_in);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2021-09-14 00:33:24 +08:00
|
|
|
* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
|
2021-02-17 12:09:51 +08:00
|
|
|
* @cxlm: The CXL memory device to communicate with.
|
|
|
|
|
* @mbox_cmd: Command to send to the memory device.
|
|
|
|
|
*
|
|
|
|
|
* Context: Any context. Expects mbox_mutex to be held.
|
|
|
|
|
* Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
|
|
|
|
|
* Caller should check the return code in @mbox_cmd to make sure it
|
|
|
|
|
* succeeded.
|
|
|
|
|
*
|
|
|
|
|
* This is a generic form of the CXL mailbox send command thus only using the
|
|
|
|
|
* registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
|
|
|
|
|
* devices, and perhaps other types of CXL devices may have further information
|
|
|
|
|
* available upon error conditions. Driver facilities wishing to send mailbox
|
|
|
|
|
* commands should use the wrapper command.
|
|
|
|
|
*
|
|
|
|
|
* The CXL spec allows for up to two mailboxes. The intention is for the primary
|
|
|
|
|
* mailbox to be OS controlled and the secondary mailbox to be used by system
|
|
|
|
|
* firmware. This allows the OS and firmware to communicate with the device and
|
|
|
|
|
* not need to coordinate with each other. The driver only uses the primary
|
|
|
|
|
* mailbox.
|
|
|
|
|
*/
|
2021-09-14 00:33:24 +08:00
|
|
|
static int __cxl_pci_mbox_send_cmd(struct cxl_mem *cxlm,
|
2021-09-09 13:12:21 +08:00
|
|
|
struct cxl_mbox_cmd *mbox_cmd)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
|
2021-09-09 13:12:09 +08:00
|
|
|
struct device *dev = cxlm->dev;
|
2021-02-17 12:09:51 +08:00
|
|
|
u64 cmd_reg, status_reg;
|
|
|
|
|
size_t out_len;
|
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
|
|
lockdep_assert_held(&cxlm->mbox_mutex);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
|
|
|
|
|
* 1. Caller reads MB Control Register to verify doorbell is clear
|
|
|
|
|
* 2. Caller writes Command Register
|
|
|
|
|
* 3. Caller writes Command Payload Registers if input payload is non-empty
|
|
|
|
|
* 4. Caller writes MB Control Register to set doorbell
|
|
|
|
|
* 5. Caller either polls for doorbell to be clear or waits for interrupt if configured
|
|
|
|
|
* 6. Caller reads MB Status Register to fetch Return code
|
|
|
|
|
* 7. If command successful, Caller reads Command Register to get Payload Length
|
|
|
|
|
* 8. If output payload is non-empty, host reads Command Payload Registers
|
|
|
|
|
*
|
|
|
|
|
* Hardware is free to do whatever it wants before the doorbell is rung,
|
|
|
|
|
* and isn't allowed to change anything after it clears the doorbell. As
|
|
|
|
|
* such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can
|
|
|
|
|
* also happen in any order (though some orders might not make sense).
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/* #1 */
|
|
|
|
|
if (cxl_doorbell_busy(cxlm)) {
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_err_ratelimited(dev, "Mailbox re-busy after acquiring\n");
|
2021-02-17 12:09:51 +08:00
|
|
|
return -EBUSY;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
|
|
|
|
|
mbox_cmd->opcode);
|
|
|
|
|
if (mbox_cmd->size_in) {
|
|
|
|
|
if (WARN_ON(!mbox_cmd->payload_in))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK,
|
|
|
|
|
mbox_cmd->size_in);
|
|
|
|
|
memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* #2, #3 */
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
|
/* #4 */
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_dbg(dev, "Sending command\n");
|
2021-02-17 12:09:51 +08:00
|
|
|
writel(CXLDEV_MBOX_CTRL_DOORBELL,
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
|
|
|
|
/* #5 */
|
2021-09-14 00:33:24 +08:00
|
|
|
rc = cxl_pci_mbox_wait_for_doorbell(cxlm);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc == -ETIMEDOUT) {
|
2021-09-14 00:33:24 +08:00
|
|
|
cxl_pci_mbox_timeout(cxlm, mbox_cmd);
|
2021-02-17 12:09:51 +08:00
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* #6 */
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
status_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
mbox_cmd->return_code =
|
|
|
|
|
FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
|
|
|
|
|
|
|
|
|
|
if (mbox_cmd->return_code != 0) {
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_dbg(dev, "Mailbox operation had an error\n");
|
2021-02-17 12:09:51 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* #7 */
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
cmd_reg = readq(cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);
|
|
|
|
|
|
|
|
|
|
/* #8 */
|
|
|
|
|
if (out_len && mbox_cmd->payload_out) {
|
|
|
|
|
/*
|
|
|
|
|
* Sanitize the copy. If hardware misbehaves, out_len per the
|
|
|
|
|
* spec can actually be greater than the max allowed size (21
|
|
|
|
|
* bits available but spec defined 1M max). The caller also may
|
|
|
|
|
* have requested less data than the hardware supplied even
|
|
|
|
|
* within spec.
|
|
|
|
|
*/
|
|
|
|
|
size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len);
|
|
|
|
|
|
|
|
|
|
memcpy_fromio(mbox_cmd->payload_out, payload, n);
|
|
|
|
|
mbox_cmd->size_out = n;
|
|
|
|
|
} else {
|
|
|
|
|
mbox_cmd->size_out = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2021-09-14 00:33:24 +08:00
|
|
|
* cxl_pci_mbox_get() - Acquire exclusive access to the mailbox.
|
2021-02-17 12:09:51 +08:00
|
|
|
* @cxlm: The memory device to gain access to.
|
|
|
|
|
*
|
|
|
|
|
* Context: Any context. Takes the mbox_mutex.
|
|
|
|
|
* Return: 0 if exclusive access was acquired.
|
|
|
|
|
*/
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_mbox_get(struct cxl_mem *cxlm)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
2021-09-09 13:12:09 +08:00
|
|
|
struct device *dev = cxlm->dev;
|
2021-02-17 12:09:51 +08:00
|
|
|
u64 md_status;
|
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
|
|
mutex_lock_io(&cxlm->mbox_mutex);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* XXX: There is some amount of ambiguity in the 2.0 version of the spec
|
|
|
|
|
* around the mailbox interface ready (8.2.8.5.1.1). The purpose of the
|
|
|
|
|
* bit is to allow firmware running on the device to notify the driver
|
|
|
|
|
* that it's ready to receive commands. It is unclear if the bit needs
|
|
|
|
|
* to be read for each transaction mailbox, ie. the firmware can switch
|
|
|
|
|
* it on and off as needed. Second, there is no defined timeout for
|
|
|
|
|
* mailbox ready, like there is for the doorbell interface.
|
|
|
|
|
*
|
|
|
|
|
* Assumptions:
|
|
|
|
|
* 1. The firmware might toggle the Mailbox Interface Ready bit, check
|
|
|
|
|
* it for every command.
|
|
|
|
|
*
|
|
|
|
|
* 2. If the doorbell is clear, the firmware should have first set the
|
|
|
|
|
* Mailbox Interface Ready bit. Therefore, waiting for the doorbell
|
|
|
|
|
* to be ready is sufficient.
|
|
|
|
|
*/
|
2021-09-14 00:33:24 +08:00
|
|
|
rc = cxl_pci_mbox_wait_for_doorbell(cxlm);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc) {
|
|
|
|
|
dev_warn(dev, "Mailbox interface not ready\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
md_status = readq(cxlm->regs.memdev + CXLMDEV_STATUS_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) {
|
|
|
|
|
dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n");
|
|
|
|
|
rc = -EBUSY;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Hardware shouldn't allow a ready status but also have failure bits
|
|
|
|
|
* set. Spit out an error, this should be a bug report
|
|
|
|
|
*/
|
|
|
|
|
rc = -EFAULT;
|
|
|
|
|
if (md_status & CXLMDEV_DEV_FATAL) {
|
|
|
|
|
dev_err(dev, "mbox: reported ready, but fatal\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
if (md_status & CXLMDEV_FW_HALT) {
|
|
|
|
|
dev_err(dev, "mbox: reported ready, but halted\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
if (CXLMDEV_RESET_NEEDED(md_status)) {
|
|
|
|
|
dev_err(dev, "mbox: reported ready, but reset needed\n");
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* with lock held */
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
mutex_unlock(&cxlm->mbox_mutex);
|
|
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2021-09-14 00:33:24 +08:00
|
|
|
* cxl_pci_mbox_put() - Release exclusive access to the mailbox.
|
2021-02-17 12:09:51 +08:00
|
|
|
* @cxlm: The CXL memory device to communicate with.
|
|
|
|
|
*
|
|
|
|
|
* Context: Any context. Expects mbox_mutex to be held.
|
|
|
|
|
*/
|
2021-09-14 00:33:24 +08:00
|
|
|
static void cxl_pci_mbox_put(struct cxl_mem *cxlm)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
|
|
|
|
mutex_unlock(&cxlm->mbox_mutex);
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-09 13:12:21 +08:00
|
|
|
static int cxl_pci_mbox_send(struct cxl_mem *cxlm, struct cxl_mbox_cmd *cmd)
|
|
|
|
|
{
|
|
|
|
|
int rc;
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
rc = cxl_pci_mbox_get(cxlm);
|
2021-09-09 13:12:21 +08:00
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
rc = __cxl_pci_mbox_send_cmd(cxlm, cmd);
|
|
|
|
|
cxl_pci_mbox_put(cxlm);
|
2021-09-09 13:12:21 +08:00
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_setup_mailbox(struct cxl_mem *cxlm)
|
2021-02-17 12:09:51 +08:00
|
|
|
{
|
cxl/mem: Introduce 'struct cxl_regs' for "composable" CXL devices
CXL MMIO register blocks are organized by device type and capabilities.
There are Component registers, Device registers (yes, an ambiguous
name), and Memory Device registers (a specific extension of Device
registers).
It is possible for a given device instance (endpoint or port) to
implement register sets from multiple of the above categories.
The driver code that enumerates and maps the registers is type specific
so it is useful to have a dedicated type and helpers for each block
type.
At the same time, once the registers are mapped the origin type does not
matter. It is overly pedantic to reference the register block type in
code that is using the registers.
In preparation for the endpoint driver to incorporate Component registers
into its MMIO operations reorganize the registers to allow typed
enumeration + mapping, but anonymous usage. With the end state of
'struct cxl_regs' to be:
struct cxl_regs {
union {
struct {
CXL_DEVICE_REGS();
};
struct cxl_device_regs device_regs;
};
union {
struct {
CXL_COMPONENT_REGS();
};
struct cxl_component_regs component_regs;
};
};
With this arrangement the driver can share component init code with
ports, but when using the registers it can directly reference the
component register block type by name without the 'component_regs'
prefix.
So, map + enumerate can be shared across drivers of different CXL
classes e.g.:
void cxl_setup_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_regs *regs);
void cxl_setup_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_regs *regs);
...while inline usage in the driver need not indicate where the
registers came from:
readl(cxlm->regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.hdm + HDM_OFFSET);
...instead of:
readl(cxlm->regs.device_regs.mbox + MBOX_OFFSET);
readl(cxlm->regs.component_regs.hdm + HDM_OFFSET);
This complexity of the definition in .h yields improvement in code
readability in .c while maintaining type-safety for organization of
setup code. It prepares the implementation to maintain organization in
the face of CXL devices that compose register interfaces consisting of
multiple types.
Given that this new container is named 'regs' rename the common register
base pointer @base, and fixup the kernel-doc for the missing @cxlmd
description.
Reviewed-by: Ben Widawsky <ben.widawsky@intel.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/162096971451.1865304.13540251513463515153.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2021-05-14 13:21:54 +08:00
|
|
|
const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
|
2021-02-17 12:09:51 +08:00
|
|
|
|
2021-09-09 13:12:21 +08:00
|
|
|
cxlm->mbox_send = cxl_pci_mbox_send;
|
2021-02-17 12:09:51 +08:00
|
|
|
cxlm->payload_size =
|
|
|
|
|
1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register
|
|
|
|
|
*
|
|
|
|
|
* If the size is too small, mandatory commands will not work and so
|
|
|
|
|
* there's no point in going forward. If the size is too large, there's
|
|
|
|
|
* no harm is soft limiting it.
|
|
|
|
|
*/
|
|
|
|
|
cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M);
|
|
|
|
|
if (cxlm->payload_size < 256) {
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_err(cxlm->dev, "Mailbox is too small (%zub)",
|
2021-02-17 12:09:51 +08:00
|
|
|
cxlm->payload_size);
|
|
|
|
|
return -ENXIO;
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-09 13:12:09 +08:00
|
|
|
dev_dbg(cxlm->dev, "Mailbox payload sized %zu",
|
2021-02-17 12:09:51 +08:00
|
|
|
cxlm->payload_size);
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map)
|
2021-04-08 06:26:20 +08:00
|
|
|
{
|
2021-05-28 08:49:19 +08:00
|
|
|
void __iomem *addr;
|
2021-10-14 07:53:29 +08:00
|
|
|
int bar = map->barno;
|
|
|
|
|
struct device *dev = &pdev->dev;
|
|
|
|
|
resource_size_t offset = map->block_offset;
|
2021-04-08 06:26:20 +08:00
|
|
|
|
2021-02-17 12:09:51 +08:00
|
|
|
/* Basic sanity check that BAR is big enough */
|
|
|
|
|
if (pci_resource_len(pdev, bar) < offset) {
|
2021-10-14 07:53:29 +08:00
|
|
|
dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar,
|
|
|
|
|
&pdev->resource[bar], &offset);
|
2021-10-16 05:57:27 +08:00
|
|
|
return -ENXIO;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
|
|
|
|
|
2021-06-04 08:50:36 +08:00
|
|
|
addr = pci_iomap(pdev, bar, 0);
|
2021-05-28 08:49:19 +08:00
|
|
|
if (!addr) {
|
2021-02-17 12:09:51 +08:00
|
|
|
dev_err(dev, "failed to map registers\n");
|
2021-10-16 05:57:27 +08:00
|
|
|
return -ENOMEM;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
|
|
|
|
|
2021-10-14 07:53:29 +08:00
|
|
|
dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n",
|
|
|
|
|
bar, &offset);
|
2021-05-21 05:29:53 +08:00
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
map->base = addr + map->block_offset;
|
|
|
|
|
return 0;
|
2021-06-04 08:50:36 +08:00
|
|
|
}
|
|
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static void cxl_unmap_regblock(struct pci_dev *pdev,
|
|
|
|
|
struct cxl_register_map *map)
|
2021-06-04 08:50:36 +08:00
|
|
|
{
|
2021-10-16 05:57:27 +08:00
|
|
|
pci_iounmap(pdev, map->base - map->block_offset);
|
|
|
|
|
map->base = NULL;
|
2021-02-17 12:09:51 +08:00
|
|
|
}
|
2021-02-17 12:09:50 +08:00
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_dvsec(struct pci_dev *pdev, int dvsec)
|
2021-02-17 12:09:50 +08:00
|
|
|
{
|
|
|
|
|
int pos;
|
|
|
|
|
|
|
|
|
|
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC);
|
|
|
|
|
if (!pos)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
while (pos) {
|
|
|
|
|
u16 vendor, id;
|
|
|
|
|
|
|
|
|
|
pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor);
|
|
|
|
|
pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id);
|
|
|
|
|
if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id)
|
|
|
|
|
return pos;
|
|
|
|
|
|
|
|
|
|
pos = pci_find_next_ext_capability(pdev, pos,
|
|
|
|
|
PCI_EXT_CAP_ID_DVSEC);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-16 05:57:27 +08:00
|
|
|
static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map)
|
2021-06-04 08:50:36 +08:00
|
|
|
{
|
2021-05-28 08:49:22 +08:00
|
|
|
struct cxl_component_reg_map *comp_map;
|
2021-06-04 08:50:36 +08:00
|
|
|
struct cxl_device_reg_map *dev_map;
|
2021-10-14 07:53:29 +08:00
|
|
|
struct device *dev = &pdev->dev;
|
2021-10-16 05:57:27 +08:00
|
|
|
void __iomem *base = map->base;
|
2021-06-04 08:50:36 +08:00
|
|
|
|
|
|
|
|
switch (map->reg_type) {
|
2021-05-28 08:49:22 +08:00
|
|
|
case CXL_REGLOC_RBI_COMPONENT:
|
|
|
|
|
comp_map = &map->component_map;
|
|
|
|
|
cxl_probe_component_regs(dev, base, comp_map);
|
|
|
|
|
if (!comp_map->hdm_decoder.valid) {
|
|
|
|
|
dev_err(dev, "HDM decoder registers not found\n");
|
|
|
|
|
return -ENXIO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dev_dbg(dev, "Set up component registers\n");
|
|
|
|
|
break;
|
2021-06-04 08:50:36 +08:00
|
|
|
case CXL_REGLOC_RBI_MEMDEV:
|
|
|
|
|
dev_map = &map->device_map;
|
|
|
|
|
cxl_probe_device_regs(dev, base, dev_map);
|
|
|
|
|
if (!dev_map->status.valid || !dev_map->mbox.valid ||
|
|
|
|
|
!dev_map->memdev.valid) {
|
|
|
|
|
dev_err(dev, "registers not found: %s%s%s\n",
|
|
|
|
|
!dev_map->status.valid ? "status " : "",
|
2021-09-04 10:20:50 +08:00
|
|
|
!dev_map->mbox.valid ? "mbox " : "",
|
|
|
|
|
!dev_map->memdev.valid ? "memdev " : "");
|
2021-06-04 08:50:36 +08:00
|
|
|
return -ENXIO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dev_dbg(dev, "Probing device registers...\n");
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int cxl_map_regs(struct cxl_mem *cxlm, struct cxl_register_map *map)
|
|
|
|
|
{
|
2021-09-09 13:12:09 +08:00
|
|
|
struct device *dev = cxlm->dev;
|
|
|
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
2021-06-04 08:50:36 +08:00
|
|
|
|
|
|
|
|
switch (map->reg_type) {
|
2021-05-28 08:49:22 +08:00
|
|
|
case CXL_REGLOC_RBI_COMPONENT:
|
|
|
|
|
cxl_map_component_regs(pdev, &cxlm->regs.component, map);
|
|
|
|
|
dev_dbg(dev, "Mapping component registers...\n");
|
|
|
|
|
break;
|
2021-06-04 08:50:36 +08:00
|
|
|
case CXL_REGLOC_RBI_MEMDEV:
|
|
|
|
|
cxl_map_device_regs(pdev, &cxlm->regs.device_regs, map);
|
|
|
|
|
dev_dbg(dev, "Probing device registers...\n");
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-14 07:53:29 +08:00
|
|
|
static void cxl_decode_regblock(u32 reg_lo, u32 reg_hi,
|
|
|
|
|
struct cxl_register_map *map)
|
2021-05-28 08:49:18 +08:00
|
|
|
{
|
2021-10-14 07:53:29 +08:00
|
|
|
map->block_offset =
|
|
|
|
|
((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK);
|
|
|
|
|
map->barno = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo);
|
|
|
|
|
map->reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo);
|
2021-05-28 08:49:18 +08:00
|
|
|
}
|
|
|
|
|
|
2021-04-08 06:26:21 +08:00
|
|
|
/**
|
2021-10-16 07:30:42 +08:00
|
|
|
* cxl_find_regblock() - Locate register blocks by type
|
|
|
|
|
* @pdev: The CXL PCI device to enumerate.
|
|
|
|
|
* @type: Register Block Indicator id
|
|
|
|
|
* @map: Enumeration output, clobbered on error
|
2021-04-08 06:26:21 +08:00
|
|
|
*
|
2021-10-16 07:30:42 +08:00
|
|
|
* Return: 0 if register block enumerated, negative error code otherwise
|
2021-04-08 06:26:21 +08:00
|
|
|
*
|
2021-10-16 07:30:42 +08:00
|
|
|
* A CXL DVSEC may point to one or more register blocks, search for them
|
|
|
|
|
* by @type.
|
2021-04-08 06:26:21 +08:00
|
|
|
*/
|
2021-10-16 07:30:42 +08:00
|
|
|
static int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
|
|
|
|
|
struct cxl_register_map *map)
|
2021-04-08 06:26:21 +08:00
|
|
|
{
|
2021-09-09 13:12:09 +08:00
|
|
|
u32 regloc_size, regblocks;
|
2021-10-16 07:30:42 +08:00
|
|
|
int regloc, i;
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
regloc = cxl_pci_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID);
|
2021-10-16 07:30:42 +08:00
|
|
|
if (!regloc)
|
2021-04-08 06:26:21 +08:00
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
|
|
pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size);
|
|
|
|
|
regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
|
|
|
|
|
|
|
|
|
|
regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET;
|
|
|
|
|
regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8;
|
|
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
for (i = 0; i < regblocks; i++, regloc += 8) {
|
2021-04-08 06:26:21 +08:00
|
|
|
u32 reg_lo, reg_hi;
|
|
|
|
|
|
|
|
|
|
pci_read_config_dword(pdev, regloc, ®_lo);
|
|
|
|
|
pci_read_config_dword(pdev, regloc + 4, ®_hi);
|
|
|
|
|
|
2021-10-14 07:53:29 +08:00
|
|
|
cxl_decode_regblock(reg_lo, reg_hi, map);
|
2021-05-28 08:49:18 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
if (map->reg_type == type)
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2021-07-17 07:15:46 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
return -ENODEV;
|
|
|
|
|
}
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
|
|
|
|
|
struct cxl_register_map *map)
|
|
|
|
|
{
|
|
|
|
|
int rc;
|
2021-07-17 07:15:47 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_find_regblock(pdev, type, map);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_map_regblock(pdev, map);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
|
|
rc = cxl_probe_regs(pdev, map);
|
|
|
|
|
cxl_unmap_regblock(pdev, map);
|
2021-04-08 06:26:21 +08:00
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
return rc;
|
2021-04-08 06:26:21 +08:00
|
|
|
}
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
2021-02-17 12:09:50 +08:00
|
|
|
{
|
2021-10-16 07:30:42 +08:00
|
|
|
struct cxl_register_map map;
|
2021-06-16 07:36:31 +08:00
|
|
|
struct cxl_memdev *cxlmd;
|
2021-04-08 06:26:20 +08:00
|
|
|
struct cxl_mem *cxlm;
|
2021-04-08 06:26:21 +08:00
|
|
|
int rc;
|
2021-02-17 12:09:51 +08:00
|
|
|
|
2021-09-09 13:12:38 +08:00
|
|
|
/*
|
|
|
|
|
* Double check the anonymous union trickery in struct cxl_regs
|
|
|
|
|
* FIXME switch to struct_group()
|
|
|
|
|
*/
|
|
|
|
|
BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
|
|
|
|
|
offsetof(struct cxl_regs, device_regs.memdev));
|
|
|
|
|
|
2021-02-17 12:09:51 +08:00
|
|
|
rc = pcim_enable_device(pdev);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
2021-02-17 12:09:50 +08:00
|
|
|
|
2021-09-09 13:12:09 +08:00
|
|
|
cxlm = cxl_mem_create(&pdev->dev);
|
2021-04-08 06:26:20 +08:00
|
|
|
if (IS_ERR(cxlm))
|
|
|
|
|
return PTR_ERR(cxlm);
|
|
|
|
|
|
2021-10-16 07:30:42 +08:00
|
|
|
rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
|
|
rc = cxl_map_regs(cxlm, &map);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
rc = cxl_pci_setup_mailbox(cxlm);
|
2021-02-17 12:09:51 +08:00
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-02-17 12:09:55 +08:00
|
|
|
rc = cxl_mem_enumerate_cmds(cxlm);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-02-17 12:09:52 +08:00
|
|
|
rc = cxl_mem_identify(cxlm);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-08-11 02:57:59 +08:00
|
|
|
rc = cxl_mem_create_range_info(cxlm);
|
|
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2021-09-09 13:12:32 +08:00
|
|
|
cxlmd = devm_cxl_add_memdev(cxlm);
|
2021-06-16 07:36:31 +08:00
|
|
|
if (IS_ERR(cxlmd))
|
|
|
|
|
return PTR_ERR(cxlmd);
|
|
|
|
|
|
|
|
|
|
if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
|
|
|
|
|
rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);
|
|
|
|
|
|
|
|
|
|
return rc;
|
2021-02-17 12:09:50 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const struct pci_device_id cxl_mem_pci_tbl[] = {
|
|
|
|
|
/* PCI class code for CXL.mem Type-3 Devices */
|
|
|
|
|
{ PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)},
|
|
|
|
|
{ /* terminate list */ },
|
|
|
|
|
};
|
|
|
|
|
MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
|
|
|
|
|
|
2021-09-14 00:33:24 +08:00
|
|
|
static struct pci_driver cxl_pci_driver = {
|
2021-02-17 12:09:50 +08:00
|
|
|
.name = KBUILD_MODNAME,
|
|
|
|
|
.id_table = cxl_mem_pci_tbl,
|
2021-09-14 00:33:24 +08:00
|
|
|
.probe = cxl_pci_probe,
|
2021-02-17 12:09:50 +08:00
|
|
|
.driver = {
|
|
|
|
|
.probe_type = PROBE_PREFER_ASYNCHRONOUS,
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
MODULE_LICENSE("GPL v2");
|
2021-09-14 00:33:24 +08:00
|
|
|
module_pci_driver(cxl_pci_driver);
|
2021-02-17 12:09:52 +08:00
|
|
|
MODULE_IMPORT_NS(CXL);
|
