korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-09-22 12:44:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

dmaengine updates for v5.6-rc1

Browse Source 
- Core:
    - Support for dynamic channels
    - Removal of various slave wrappers
    - Make few slave request APIs as private to dmaengine
    - Symlinks between channels and slaves
    - Support for hotplug of controllers
    - Support for metadata_ops for dma_async_tx_descriptor
    - Reporting DMA cached data amount
    - Virtual dma channel locking updates
 
  - New drivers/device/feature support support:
    - Driver for Intel data accelerators
    - Driver for TI K3 UDMA
    - Driver for PLX DMA engine
    - Driver for hisilicon Kunpeng DMA engine
    - Support for eDMA support for QorIQ LS1028A in fsl edma driver
    - Support for cyclic dma in sun4i driver
    - Support for X1830 in JZ4780 driver
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+vs47OPLdNbVcHzyfBQHDyUjg0cFAl4u+QkACgkQfBQHDyUj
 g0cCcg//awBruofTHIrBOwHmCX1a09mw5WmkFG48N7tYp4fvaI1aOgs3hH9PZiBG
 fFZUktodwYpEKg6JJOfm1RnLBuKm0+3zmaKGPdK1RcbaDURh8G9qhW65f4mfImvB
 GXlgw59WKtgPAM9zWW9UxjugAk4DBte5xVKYJUsI0t4P7k9TM4i0Fv0VmMUhhDuo
 buPD1cM/GWFHbE7OYJ51aGRtrOHV1nPgQaHBkWaT7EotzGsZ3gtWYzteI3BRXRV/
 IkSgxOefMkIgu1j3KIxFZ1CJDHCZSnx2B+AEMCcp63osyeHBOYoL7KQxo6tBjaRV
 fbCasbkTkvvJUjyZdtOdU2wqf7ZqoDkD+n5nkpENf4G1M8J5RiHmrFq96m3HRonE
 V1bmMslXhsJlvtoT6ec2iJFchiq0nx1XHyST6faUOK+0cd1lzbogWwztydQH4fwd
 TxfEd+eYlFFu3lGDfRp14Tz7fAcFNPZ2bJQhZkF6RpwUW3y3L0cJc3Y0AcWmNkvJ
 oStvTlbbUvgRgO7rvEyAmdPb31lE6PLaA0WCahcvf4zQxxNMyYyaWP73MegvqJGO
 pfJXBOWBTTKwu0fDR5UHJd3tEDABvcZnwBaCSYrpI5f9bJ4NRI3f4DIMwLBnw9IK
 aH6pzwo4gTAMuvxzq8KeTp3hU7kszyUN8q8hiTZlgVozMLKXhQY=
 =mv1v
 -----END PGP SIGNATURE-----

Merge tag 'dmaengine-5.6-rc1' of git://git.infradead.org/users/vkoul/slave-dma

Pull dmaengine updates from Vinod Koul:
 "This time we have a bunch of core changes to support dynamic channels,
  hotplug of controllers, new apis for metadata ops etc along with new
  drivers for Intel data accelerators, TI K3 UDMA, PLX DMA engine and
  hisilicon Kunpeng DMA engine. Also usual assorted updates to drivers.

  Core:
   - Support for dynamic channels
   - Removal of various slave wrappers
   - Make few slave request APIs as private to dmaengine
   - Symlinks between channels and slaves
   - Support for hotplug of controllers
   - Support for metadata_ops for dma_async_tx_descriptor
   - Reporting DMA cached data amount
   - Virtual dma channel locking updates

  New drivers/device/feature support support:
   - Driver for Intel data accelerators
   - Driver for TI K3 UDMA
   - Driver for PLX DMA engine
   - Driver for hisilicon Kunpeng DMA engine
   - Support for eDMA support for QorIQ LS1028A in fsl edma driver
   - Support for cyclic dma in sun4i driver
   - Support for X1830 in JZ4780 driver"

* tag 'dmaengine-5.6-rc1' of git://git.infradead.org/users/vkoul/slave-dma: (62 commits)
  dmaengine: Create symlinks between DMA channels and slaves
  dmaengine: hisilicon: Add Kunpeng DMA engine support
  dmaengine: idxd: add char driver to expose submission portal to userland
  dmaengine: idxd: connect idxd to dmaengine subsystem
  dmaengine: idxd: add descriptor manipulation routines
  dmaengine: idxd: add sysfs ABI for idxd driver
  dmaengine: idxd: add configuration component of driver
  dmaengine: idxd: Init and probe for Intel data accelerators
  dmaengine: add support to dynamic register/unregister of channels
  dmaengine: break out channel registration
  x86/asm: add iosubmit_cmds512() based on MOVDIR64B CPU instruction
  dmaengine: ti: k3-udma: fix spelling mistake "limted" -> "limited"
  dmaengine: s3c24xx-dma: fix spelling mistake "to" -> "too"
  dmaengine: Move dma_get_{,any_}slave_channel() to private dmaengine.h
  dmaengine: Remove dma_request_slave_channel_compat() wrapper
  dmaengine: Remove dma_device_satisfies_mask() wrapper
  dt-bindings: fsl-imx-sdma: Add i.MX8MM/i.MX8MN/i.MX8MP compatible string
  dmaengine: zynqmp_dma: fix burst length configuration
  dmaengine: sun4i: Add support for cyclic requests with dedicated DMA
  dmaengine: fsl-qdma: fix duplicated argument to &&
  ...
This commit is contained in:
Linus Torvalds 2020-01-27 10:55:50 -08:00
commit a5b871c91d
68 changed files with 15355 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
Documentation/ABI/stable/sysfs-driver-dma-idxd Normal file
View File

@ -0,0 +1,171 @@
What: sys/bus/dsa/devices/dsa<m>/cdev_major
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The major number that the character device driver assigned to
this device.
What: sys/bus/dsa/devices/dsa<m>/errors
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The error information for this device.
What: sys/bus/dsa/devices/dsa<m>/max_batch_size
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The largest number of work descriptors in a batch.
What: sys/bus/dsa/devices/dsa<m>/max_work_queues_size
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The maximum work queue size supported by this device.
What: sys/bus/dsa/devices/dsa<m>/max_engines
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The maximum number of engines supported by this device.
What: sys/bus/dsa/devices/dsa<m>/max_groups
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The maximum number of groups can be created under this device.
What: sys/bus/dsa/devices/dsa<m>/max_tokens
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The total number of bandwidth tokens supported by this device.
The bandwidth tokens represent resources within the DSA
implementation, and these resources are allocated by engines to
support operations.
What: sys/bus/dsa/devices/dsa<m>/max_transfer_size
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The number of bytes to be read from the source address to
perform the operation. The maximum transfer size is dependent on
the workqueue the descriptor was submitted to.
What: sys/bus/dsa/devices/dsa<m>/max_work_queues
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The maximum work queue number that this device supports.
What: sys/bus/dsa/devices/dsa<m>/numa_node
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The numa node number for this device.
What: sys/bus/dsa/devices/dsa<m>/op_cap
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The operation capability bit mask specify the operation types
supported by the this device.
What: sys/bus/dsa/devices/dsa<m>/state
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The state information of this device. It can be either enabled
or disabled.
What: sys/bus/dsa/devices/dsa<m>/group<m>.<n>
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The assigned group under this device.
What: sys/bus/dsa/devices/dsa<m>/engine<m>.<n>
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The assigned engine under this device.
What: sys/bus/dsa/devices/dsa<m>/wq<m>.<n>
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The assigned work queue under this device.
What: sys/bus/dsa/devices/dsa<m>/configurable
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: To indicate if this device is configurable or not.
What: sys/bus/dsa/devices/dsa<m>/token_limit
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The maximum number of bandwidth tokens that may be in use at
one time by operations that access low bandwidth memory in the
device.
What: sys/bus/dsa/devices/wq<m>.<n>/group_id
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The group id that this work queue belongs to.
What: sys/bus/dsa/devices/wq<m>.<n>/size
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The work queue size for this work queue.
What: sys/bus/dsa/devices/wq<m>.<n>/type
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The type of this work queue, it can be "kernel" type for work
queue usages in the kernel space or "user" type for work queue
usages by applications in user space.
What: sys/bus/dsa/devices/wq<m>.<n>/cdev_minor
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The minor number assigned to this work queue by the character
device driver.
What: sys/bus/dsa/devices/wq<m>.<n>/mode
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The work queue mode type for this work queue.
What: sys/bus/dsa/devices/wq<m>.<n>/priority
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The priority value of this work queue, it is a vlue relative to
other work queue in the same group to control quality of service
for dispatching work from multiple workqueues in the same group.
What: sys/bus/dsa/devices/wq<m>.<n>/state
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The current state of the work queue.
What: sys/bus/dsa/devices/wq<m>.<n>/threshold
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The number of entries in this work queue that may be filled
via a limited portal.
What: sys/bus/dsa/devices/engine<m>.<n>/group_id
Date: Oct 25, 2019
KernelVersion: 5.6.0
Contact: dmaengine@vger.kernel.org
Description: The group that this engine belongs to.

1
Documentation/devicetree/bindings/dma/fsl-edma.txt
View File

@ -10,6 +10,7 @@ Required properties:
- compatible :
- "fsl,vf610-edma" for eDMA used similar to that on Vybrid vf610 SoC
- "fsl,imx7ulp-edma" for eDMA2 used similar to that on i.mx7ulp
- "fsl,fsl,ls1028a-edma" for eDMA used similar to that on Vybrid vf610 SoC
- reg : Specifies base physical address(s) and size of the eDMA registers.
The 1st region is eDMA control register's address and size.
The 2nd and the 3rd regions are programmable channel multiplexing

3
Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt
View File

@ -10,6 +10,9 @@ Required properties:
"fsl,imx6q-sdma"
"fsl,imx7d-sdma"
"fsl,imx8mq-sdma"
"fsl,imx8mm-sdma"
"fsl,imx8mn-sdma"
"fsl,imx8mp-sdma"
The -to variants should be preferred since they allow to determine the
correct ROM script addresses needed for the driver to work without additional
firmware.

6
Documentation/devicetree/bindings/dma/jz4780-dma.txt
View File

@ -1,4 +1,4 @@
* Ingenic JZ4780 DMA Controller
* Ingenic XBurst DMA Controller
Required properties:
@ -8,10 +8,12 @@ Required properties:
* ingenic,jz4770-dma
* ingenic,jz4780-dma
* ingenic,x1000-dma
* ingenic,x1830-dma
- reg: Should contain the DMA channel registers location and length, followed
by the DMA controller registers location and length.
- interrupts: Should contain the interrupt specifier of the DMA controller.
- clocks: Should contain a clock specifier for the JZ4780/X1000 PDMA clock.
- clocks: Should contain a clock specifier for the JZ4780/X1000/X1830 PDMA
clock.
- #dma-cells: Must be <2>. Number of integer cells in the dmas property of
DMA clients (see below).

1
Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
View File

@ -30,6 +30,7 @@ Required Properties:
- "renesas,dmac-r8a7794" (R-Car E2)
- "renesas,dmac-r8a7795" (R-Car H3)
- "renesas,dmac-r8a7796" (R-Car M3-W)
- "renesas,dmac-r8a77961" (R-Car M3-W+)
- "renesas,dmac-r8a77965" (R-Car M3-N)
- "renesas,dmac-r8a77970" (R-Car V3M)
- "renesas,dmac-r8a77980" (R-Car V3H)

184
Documentation/devicetree/bindings/dma/ti/k3-udma.yaml Normal file
View File

@ -0,0 +1,184 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/dma/ti/k3-udma.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Texas Instruments K3 NAVSS Unified DMA Device Tree Bindings
maintainers:
- Peter Ujfalusi <peter.ujfalusi@ti.com>
description: |
The UDMA-P is intended to perform similar (but significantly upgraded)
functions as the packet-oriented DMA used on previous SoC devices. The UDMA-P
module supports the transmission and reception of various packet types.
The UDMA-P architecture facilitates the segmentation and reassembly of SoC DMA
data structure compliant packets to/from smaller data blocks that are natively
compatible with the specific requirements of each connected peripheral.
Multiple Tx and Rx channels are provided within the DMA which allow multiple
segmentation or reassembly operations to be ongoing. The DMA controller
maintains state information for each of the channels which allows packet
segmentation and reassembly operations to be time division multiplexed between
channels in order to share the underlying DMA hardware. An external DMA
scheduler is used to control the ordering and rate at which this multiplexing
occurs for Transmit operations. The ordering and rate of Receive operations
is indirectly controlled by the order in which blocks are pushed into the DMA
on the Rx PSI-L interface.
The UDMA-P also supports acting as both a UTC and UDMA-C for its internal
channels. Channels in the UDMA-P can be configured to be either Packet-Based
or Third-Party channels on a channel by channel basis.
All transfers within NAVSS is done between PSI-L source and destination
threads.
The peripherals serviced by UDMA can be PSI-L native (sa2ul, cpsw, etc) or
legacy, non PSI-L native peripherals. In the later case a special, small PDMA
is tasked to act as a bridge between the PSI-L fabric and the legacy
peripheral.
PDMAs can be configured via UDMAP peer registers to match with the
configuration of the legacy peripheral.
allOf:
- $ref: "../dma-controller.yaml#"
properties:
"#dma-cells":
const: 1
description: |
The cell is the PSI-L thread ID of the remote (to UDMAP) end.
Valid ranges for thread ID depends on the data movement direction:
for source thread IDs (rx): 0 - 0x7fff
for destination thread IDs (tx): 0x8000 - 0xffff
Please refer to the device documentation for the PSI-L thread map and also
the PSI-L peripheral chapter for the correct thread ID.
compatible:
enum:
- ti,am654-navss-main-udmap
- ti,am654-navss-mcu-udmap
- ti,j721e-navss-main-udmap
- ti,j721e-navss-mcu-udmap
reg:
maxItems: 3
reg-names:
items:
- const: gcfg
- const: rchanrt
- const: tchanrt
msi-parent: true
ti,sci:
description: phandle to TI-SCI compatible System controller node
allOf:
- $ref: /schemas/types.yaml#/definitions/phandle
ti,sci-dev-id:
description: TI-SCI device id of UDMAP
allOf:
- $ref: /schemas/types.yaml#/definitions/uint32
ti,ringacc:
description: phandle to the ring accelerator node
allOf:
- $ref: /schemas/types.yaml#/definitions/phandle
ti,sci-rm-range-tchan:
description: |
Array of UDMA tchan resource subtypes for resource allocation for this
host
allOf:
- $ref: /schemas/types.yaml#/definitions/uint32-array
minItems: 1
# Should be enough
maxItems: 255
ti,sci-rm-range-rchan:
description: |
Array of UDMA rchan resource subtypes for resource allocation for this
host
allOf:
- $ref: /schemas/types.yaml#/definitions/uint32-array
minItems: 1
# Should be enough
maxItems: 255
ti,sci-rm-range-rflow:
description: |
Array of UDMA rflow resource subtypes for resource allocation for this
host
allOf:
- $ref: /schemas/types.yaml#/definitions/uint32-array
minItems: 1
# Should be enough
maxItems: 255
required:
- compatible
- "#dma-cells"
- reg
- reg-names
- msi-parent
- ti,sci
- ti,sci-dev-id
- ti,ringacc
- ti,sci-rm-range-tchan
- ti,sci-rm-range-rchan
- ti,sci-rm-range-rflow
examples:
- |+
cbass_main {
#address-cells = <2>;
#size-cells = <2>;
cbass_main_navss: navss@30800000 {
compatible = "simple-mfd";
#address-cells = <2>;
#size-cells = <2>;
dma-coherent;
dma-ranges;
ranges;
ti,sci-dev-id = <118>;
main_udmap: dma-controller@31150000 {
compatible = "ti,am654-navss-main-udmap";
reg = <0x0 0x31150000 0x0 0x100>,
<0x0 0x34000000 0x0 0x100000>,
<0x0 0x35000000 0x0 0x100000>;
reg-names = "gcfg", "rchanrt", "tchanrt";
#dma-cells = <1>;
ti,ringacc = <&ringacc>;
msi-parent = <&inta_main_udmass>;
ti,sci = <&dmsc>;
ti,sci-dev-id = <188>;
ti,sci-rm-range-tchan = <0x1>, /* TX_HCHAN */
<0x2>; /* TX_CHAN */
ti,sci-rm-range-rchan = <0x4>, /* RX_HCHAN */
<0x5>; /* RX_CHAN */
ti,sci-rm-range-rflow = <0x6>; /* GP RFLOW */
};
};
mcasp0: mcasp@02B00000 {
dmas = <&main_udmap 0xc400>, <&main_udmap 0x4400>;
dma-names = "tx", "rx";
};
crypto: crypto@4E00000 {
compatible = "ti,sa2ul-crypto";
dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>, <&main_udmap 0x4001>;
dma-names = "tx", "rx1", "rx2";
};
};

59
Documentation/devicetree/bindings/soc/ti/k3-ringacc.txt Normal file
View File

@ -0,0 +1,59 @@
* Texas Instruments K3 NavigatorSS Ring Accelerator
The Ring Accelerator (RA) is a machine which converts read/write accesses
from/to a constant address into corresponding read/write accesses from/to a
circular data structure in memory. The RA eliminates the need for each DMA
controller which needs to access ring elements from having to know the current
state of the ring (base address, current offset). The DMA controller
performs a read or write access to a specific address range (which maps to the
source interface on the RA) and the RA replaces the address for the transaction
with a new address which corresponds to the head or tail element of the ring
(head for reads, tail for writes).
The Ring Accelerator is a hardware module that is responsible for accelerating
management of the packet queues. The K3 SoCs can have more than one RA instances
Required properties:
- compatible : Must be "ti,am654-navss-ringacc";
- reg : Should contain register location and length of the following
named register regions.
- reg-names : should be
"rt" - The RA Ring Real-time Control/Status Registers
"fifos" - The RA Queues Registers
"proxy_gcfg" - The RA Proxy Global Config Registers
"proxy_target" - The RA Proxy Datapath Registers
- ti,num-rings : Number of rings supported by RA
- ti,sci-rm-range-gp-rings : TI-SCI RM subtype for GP ring range
- ti,sci : phandle on TI-SCI compatible System controller node
- ti,sci-dev-id : TI-SCI device id of the ring accelerator
- msi-parent : phandle for "ti,sci-inta" interrupt controller
Optional properties:
-- ti,dma-ring-reset-quirk : enable ringacc / udma ring state interoperability
issue software w/a
Example:
ringacc: ringacc@3c000000 {
compatible = "ti,am654-navss-ringacc";
reg = <0x0 0x3c000000 0x0 0x400000>,
<0x0 0x38000000 0x0 0x400000>,
<0x0 0x31120000 0x0 0x100>,
<0x0 0x33000000 0x0 0x40000>;
reg-names = "rt", "fifos",
"proxy_gcfg", "proxy_target";
ti,num-rings = <818>;
ti,sci-rm-range-gp-rings = <0x2>; /* GP ring range */
ti,dma-ring-reset-quirk;
ti,sci = <&dmsc>;
ti,sci-dev-id = <187>;
msi-parent = <&inta_main_udmass>;
};
client:
dma_ipx: dma_ipx@<addr> {
...
ti,ringacc = <&ringacc>;
...
}

87
Documentation/driver-api/dmaengine/client.rst
View File

@ -151,6 +151,93 @@ The details of these operations are:
Note that callbacks will always be invoked from the DMA
engines tasklet, never from interrupt context.
Optional: per descriptor metadata
---------------------------------
DMAengine provides two ways for metadata support.
DESC_METADATA_CLIENT
The metadata buffer is allocated/provided by the client driver and it is
attached to the descriptor.
.. code-block:: c
int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
void *data, size_t len);
DESC_METADATA_ENGINE
The metadata buffer is allocated/managed by the DMA driver. The client
driver can ask for the pointer, maximum size and the currently used size of
the metadata and can directly update or read it.
Becasue the DMA driver manages the memory area containing the metadata,
clients must make sure that they do not try to access or get the pointer
after their transfer completion callback has run for the descriptor.
If no completion callback has been defined for the transfer, then the
metadata must not be accessed after issue_pending.
In other words: if the aim is to read back metadata after the transfer is
completed, then the client must use completion callback.
.. code-block:: c
void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
size_t *payload_len, size_t *max_len);
int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
size_t payload_len);
Client drivers can query if a given mode is supported with:
.. code-block:: c
bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan,
enum dma_desc_metadata_mode mode);
Depending on the used mode client drivers must follow different flow.
DESC_METADATA_CLIENT
- DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
1. prepare the descriptor (dmaengine_prep_*)
construct the metadata in the client's buffer
2. use dmaengine_desc_attach_metadata() to attach the buffer to the
descriptor
3. submit the transfer
- DMA_DEV_TO_MEM:
1. prepare the descriptor (dmaengine_prep_*)
2. use dmaengine_desc_attach_metadata() to attach the buffer to the
descriptor
3. submit the transfer
4. when the transfer is completed, the metadata should be available in the
attached buffer
DESC_METADATA_ENGINE
- DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
1. prepare the descriptor (dmaengine_prep_*)
2. use dmaengine_desc_get_metadata_ptr() to get the pointer to the
engine's metadata area
3. update the metadata at the pointer
4. use dmaengine_desc_set_metadata_len() to tell the DMA engine the
amount of data the client has placed into the metadata buffer
5. submit the transfer
- DMA_DEV_TO_MEM:
1. prepare the descriptor (dmaengine_prep_*)
2. submit the transfer
3. on transfer completion, use dmaengine_desc_get_metadata_ptr() to get
the pointer to the engine's metadata area
4. read out the metadata from the pointer
.. note::
When DESC_METADATA_ENGINE mode is used the metadata area for the descriptor
is no longer valid after the transfer has been completed (valid up to the
point when the completion callback returns if used).
Mixed use of DESC_METADATA_CLIENT / DESC_METADATA_ENGINE is not allowed,
client drivers must use either of the modes per descriptor.
4. Submit the transaction
Once the descriptor has been prepared and the callback information

48
Documentation/driver-api/dmaengine/provider.rst
View File

@ -247,6 +247,54 @@ after each transfer. In case of a ring buffer, they may loop
(DMA_CYCLIC). Addresses pointing to a device's register (e.g. a FIFO)
are typically fixed.
Per descriptor metadata support
-------------------------------
Some data movement architecture (DMA controller and peripherals) uses metadata
associated with a transaction. The DMA controller role is to transfer the
payload and the metadata alongside.
The metadata itself is not used by the DMA engine itself, but it contains
parameters, keys, vectors, etc for peripheral or from the peripheral.
The DMAengine framework provides a generic ways to facilitate the metadata for
descriptors. Depending on the architecture the DMA driver can implement either
or both of the methods and it is up to the client driver to choose which one
to use.
- DESC_METADATA_CLIENT
The metadata buffer is allocated/provided by the client driver and it is
attached (via the dmaengine_desc_attach_metadata() helper to the descriptor.
From the DMA driver the following is expected for this mode:
- DMA_MEM_TO_DEV / DEV_MEM_TO_MEM
The data from the provided metadata buffer should be prepared for the DMA
controller to be sent alongside of the payload data. Either by copying to a
hardware descriptor, or highly coupled packet.
- DMA_DEV_TO_MEM
On transfer completion the DMA driver must copy the metadata to the client
provided metadata buffer before notifying the client about the completion.
After the transfer completion, DMA drivers must not touch the metadata
buffer provided by the client.
- DESC_METADATA_ENGINE
The metadata buffer is allocated/managed by the DMA driver. The client driver
can ask for the pointer, maximum size and the currently used size of the
metadata and can directly update or read it. dmaengine_desc_get_metadata_ptr()
and dmaengine_desc_set_metadata_len() is provided as helper functions.
From the DMA driver the following is expected for this mode:
- get_metadata_ptr
Should return a pointer for the metadata buffer, the maximum size of the
metadata buffer and the currently used / valid (if any) bytes in the buffer.
- set_metadata_len
It is called by the clients after it have placed the metadata to the buffer
to let the DMA driver know the number of valid bytes provided.
Note: since the client will ask for the metadata pointer in the completion
callback (in DMA_DEV_TO_MEM case) the DMA driver must ensure that the
descriptor is not freed up prior the callback is called.
Device operations
-----------------

13
MAINTAINERS
View File

@ -8392,6 +8392,14 @@ Q: https://patchwork.kernel.org/project/linux-dmaengine/list/
S: Supported
F: drivers/dma/ioat*
INTEL IADX DRIVER
M: Dave Jiang <dave.jiang@intel.com>
L: dmaengine@vger.kernel.org
S: Supported
F: drivers/dma/idxd/*
F: include/uapi/linux/idxd.h
F: include/linux/idxd.h
INTEL IDLE DRIVER
M: Jacob Pan <jacob.jun.pan@linux.intel.com>
M: Len Brown <lenb@kernel.org>
@ -13162,6 +13170,11 @@ S: Maintained
F: drivers/iio/chemical/pms7003.c
F: Documentation/devicetree/bindings/iio/chemical/plantower,pms7003.yaml
PLX DMA DRIVER
M: Logan Gunthorpe <logang@deltatee.com>
S: Maintained
F: drivers/dma/plx_dma.c
PMBUS HARDWARE MONITORING DRIVERS
M: Guenter Roeck <linux@roeck-us.net>
L: linux-hwmon@vger.kernel.org

36
arch/x86/include/asm/io.h
View File

@ -399,4 +399,40 @@ extern bool arch_memremap_can_ram_remap(resource_size_t offset,
extern bool phys_mem_access_encrypted(unsigned long phys_addr,
unsigned long size);
/**
* iosubmit_cmds512 - copy data to single MMIO location, in 512-bit units
* @__dst: destination, in MMIO space (must be 512-bit aligned)
* @src: source
* @count: number of 512 bits quantities to submit
*
* Submit data from kernel space to MMIO space, in units of 512 bits at a
* time. Order of access is not guaranteed, nor is a memory barrier
* performed afterwards.
*
* Warning: Do not use this helper unless your driver has checked that the CPU
* instruction is supported on the platform.
*/
static inline void iosubmit_cmds512(void __iomem *__dst, const void *src,
size_t count)
{
/*
* Note that this isn't an "on-stack copy", just definition of "dst"
* as a pointer to 64-bytes of stuff that is going to be overwritten.
* In the MOVDIR64B case that may be needed as you can use the
* MOVDIR64B instruction to copy arbitrary memory around. This trick
* lets the compiler know how much gets clobbered.
*/
volatile struct { char _[64]; } *dst = __dst;
const u8 *from = src;
const u8 *end = from + count * 64;
while (from < end) {
/* MOVDIR64B [rdx], rax */
asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
: "=m" (dst)
: "d" (from), "a" (dst));
from += 64;
}
}
#endif /* _ASM_X86_IO_H */

30
drivers/dma/Kconfig
View File

@ -239,6 +239,14 @@ config FSL_RAID
the capability to offload memcpy, xor and pq computation
for raid5/6.
config HISI_DMA
tristate "HiSilicon DMA Engine support"
depends on ARM64 || (COMPILE_TEST && PCI_MSI)
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
Support HiSilicon Kunpeng DMA engine.
config IMG_MDC_DMA
tristate "IMG MDC support"
depends on MIPS || COMPILE_TEST
@ -273,6 +281,19 @@ config INTEL_IDMA64
Enable DMA support for Intel Low Power Subsystem such as found on
Intel Skylake PCH.
config INTEL_IDXD
tristate "Intel Data Accelerators support"
depends on PCI && X86_64
select DMA_ENGINE
select SBITMAP
help
Enable support for the Intel(R) data accelerators present
in Intel Xeon CPU.
Say Y if you have such a platform.
If unsure, say N.
config INTEL_IOATDMA
tristate "Intel I/OAT DMA support"
depends on PCI && X86_64
@ -497,6 +518,15 @@ config PXA_DMA
16 to 32 channels for peripheral to memory or memory to memory
transfers.
config PLX_DMA
tristate "PLX ExpressLane PEX Switch DMA Engine Support"
depends on PCI
select DMA_ENGINE
help
Some PLX ExpressLane PCI Switches support additional DMA engines.
These are exposed via extra functions on the switch's
upstream port. Each function exposes one DMA channel.
config SIRF_DMA
tristate "CSR SiRFprimaII/SiRFmarco DMA support"
depends on ARCH_SIRF

3
drivers/dma/Makefile
View File

@ -35,12 +35,14 @@ obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o
obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o
obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o
obj-$(CONFIG_FSL_RAID) += fsl_raid.o
obj-$(CONFIG_HISI_DMA) += hisi_dma.o
obj-$(CONFIG_HSU_DMA) += hsu/
obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
obj-$(CONFIG_IMX_DMA) += imx-dma.o
obj-$(CONFIG_IMX_SDMA) += imx-sdma.o
obj-$(CONFIG_INTEL_IDMA64) += idma64.o
obj-$(CONFIG_INTEL_IOATDMA) += ioat/
obj-$(CONFIG_INTEL_IDXD) += idxd/
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o
obj-$(CONFIG_K3_DMA) += k3dma.o
@ -59,6 +61,7 @@ obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o
obj-$(CONFIG_OWL_DMA) += owl-dma.o
obj-$(CONFIG_PCH_DMA) += pch_dma.o
obj-$(CONFIG_PL330_DMA) += pl330.o
obj-$(CONFIG_PLX_DMA) += plx_dma.o
obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/
obj-$(CONFIG_PXA_DMA) += pxa_dma.o
obj-$(CONFIG_RENESAS_DMA) += sh/

3
drivers/dma/bcm2835-dma.c
View File

@ -797,10 +797,7 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
/* stop DMA activity */
if (c->desc) {
if (c->desc->vd.tx.flags & DMA_PREP_INTERRUPT)
vchan_terminate_vdesc(&c->desc->vd);
else
vchan_vdesc_fini(&c->desc->vd);
c->desc = NULL;
bcm2835_dma_abort(c);
}

10
drivers/dma/dma-axi-dmac.c
View File

@ -830,6 +830,7 @@ static int axi_dmac_probe(struct platform_device *pdev)
struct dma_device *dma_dev;
struct axi_dmac *dmac;
struct resource *res;
struct regmap *regmap;
int ret;
dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
@ -921,10 +922,17 @@ static int axi_dmac_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, dmac);
devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config);
regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,
&axi_dmac_regmap_config);
if (IS_ERR(regmap)) {
ret = PTR_ERR(regmap);
goto err_free_irq;
}
return 0;
err_free_irq:
free_irq(dmac->irq, dmac);
err_unregister_of:
of_dma_controller_free(pdev->dev.of_node);
err_unregister_device:

7
drivers/dma/dma-jz4780.c
View File

@ -1021,12 +1021,19 @@ static const struct jz4780_dma_soc_data x1000_dma_soc_data = {
.flags = JZ_SOC_DATA_PROGRAMMABLE_DMA,
};
static const struct jz4780_dma_soc_data x1830_dma_soc_data = {
.nb_channels = 32,
.transfer_ord_max = 7,
.flags = JZ_SOC_DATA_PROGRAMMABLE_DMA,
};
static const struct of_device_id jz4780_dma_dt_match[] = {
{ .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data },
{ .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data },
{ .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data },
{ .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data },
{ .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data },
{ .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data },
{},
};
MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match);

642
drivers/dma/dmaengine.c
View File

@ -60,6 +60,8 @@ static long dmaengine_ref_count;
/* --- sysfs implementation --- */
#define DMA_SLAVE_NAME "slave"
/**
* dev_to_dma_chan - convert a device pointer to its sysfs container object
* @dev - device node
@ -164,130 +166,6 @@ static struct class dma_devclass = {
/* --- client and device registration --- */
#define dma_device_satisfies_mask(device, mask) \
__dma_device_satisfies_mask((device), &(mask))
static int
__dma_device_satisfies_mask(struct dma_device *device,
const dma_cap_mask_t *want)
{
dma_cap_mask_t has;
bitmap_and(has.bits, want->bits, device->cap_mask.bits,
DMA_TX_TYPE_END);
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}
static struct module *dma_chan_to_owner(struct dma_chan *chan)
{
return chan->device->dev->driver->owner;
}
/**
* balance_ref_count - catch up the channel reference count
* @chan - channel to balance ->client_count versus dmaengine_ref_count
*
* balance_ref_count must be called under dma_list_mutex
*/
static void balance_ref_count(struct dma_chan *chan)
{
struct module *owner = dma_chan_to_owner(chan);
while (chan->client_count < dmaengine_ref_count) {
__module_get(owner);
chan->client_count++;
}
}
/**
* dma_chan_get - try to grab a dma channel's parent driver module
* @chan - channel to grab
*
* Must be called under dma_list_mutex
*/
static int dma_chan_get(struct dma_chan *chan)
{
struct module *owner = dma_chan_to_owner(chan);
int ret;
/* The channel is already in use, update client count */
if (chan->client_count) {
__module_get(owner);
goto out;
}
if (!try_module_get(owner))
return -ENODEV;
/* allocate upon first client reference */
if (chan->device->device_alloc_chan_resources) {
ret = chan->device->device_alloc_chan_resources(chan);
if (ret < 0)
goto err_out;
}
if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
balance_ref_count(chan);
out:
chan->client_count++;
return 0;
err_out:
module_put(owner);
return ret;
}
/**
* dma_chan_put - drop a reference to a dma channel's parent driver module
* @chan - channel to release
*
* Must be called under dma_list_mutex
*/
static void dma_chan_put(struct dma_chan *chan)
{
/* This channel is not in use, bail out */
if (!chan->client_count)
return;
chan->client_count--;
module_put(dma_chan_to_owner(chan));
/* This channel is not in use anymore, free it */
if (!chan->client_count && chan->device->device_free_chan_resources) {
/* Make sure all operations have completed */
dmaengine_synchronize(chan);
chan->device->device_free_chan_resources(chan);
}
/* If the channel is used via a DMA request router, free the mapping */
if (chan->router && chan->router->route_free) {
chan->router->route_free(chan->router->dev, chan->route_data);
chan->router = NULL;
chan->route_data = NULL;
}
}
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
enum dma_status status;
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
dma_async_issue_pending(chan);
do {
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
dev_err(chan->device->dev, "%s: timeout!\n", __func__);
return DMA_ERROR;
}
if (status != DMA_IN_PROGRESS)
break;
cpu_relax();
} while (1);
return status;
}
EXPORT_SYMBOL(dma_sync_wait);
/**
* dma_cap_mask_all - enable iteration over all operation types
*/
@ -330,7 +208,7 @@ static int __init dma_channel_table_init(void)
}
if (err) {
pr_err("initialization failure\n");
pr_err("dmaengine dma_channel_table_init failure: %d\n", err);
for_each_dma_cap_mask(cap, dma_cap_mask_all)
free_percpu(channel_table[cap]);
}
@ -340,37 +218,8 @@ static int __init dma_channel_table_init(void)
arch_initcall(dma_channel_table_init);
/**
* dma_find_channel - find a channel to carry out the operation
* @tx_type: transaction type
*/
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
{
return this_cpu_read(channel_table[tx_type]->chan);
}
EXPORT_SYMBOL(dma_find_channel);
/**
* dma_issue_pending_all - flush all pending operations across all channels
*/
void dma_issue_pending_all(void)
{
struct dma_device *device;
struct dma_chan *chan;
rcu_read_lock();
list_for_each_entry_rcu(device, &dma_device_list, global_node) {
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
continue;
list_for_each_entry(chan, &device->channels, device_node)
if (chan->client_count)
device->device_issue_pending(chan);
}
rcu_read_unlock();
}
EXPORT_SYMBOL(dma_issue_pending_all);
/**
* dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu
* dma_chan_is_local - returns true if the channel is in the same numa-node as
* the cpu
*/
static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
{
@ -380,7 +229,8 @@ static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
}
/**
* min_chan - returns the channel with min count and in the same numa-node as the cpu
* min_chan - returns the channel with min count and in the same numa-node as
* the cpu
* @cap: capability to match
* @cpu: cpu index which the channel should be close to
*
@ -460,6 +310,184 @@ static void dma_channel_rebalance(void)
}
}
static int dma_device_satisfies_mask(struct dma_device *device,
const dma_cap_mask_t *want)
{
dma_cap_mask_t has;
bitmap_and(has.bits, want->bits, device->cap_mask.bits,
DMA_TX_TYPE_END);
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}
static struct module *dma_chan_to_owner(struct dma_chan *chan)
{
return chan->device->owner;
}
/**
* balance_ref_count - catch up the channel reference count
* @chan - channel to balance ->client_count versus dmaengine_ref_count
*
* balance_ref_count must be called under dma_list_mutex
*/
static void balance_ref_count(struct dma_chan *chan)
{
struct module *owner = dma_chan_to_owner(chan);
while (chan->client_count < dmaengine_ref_count) {
__module_get(owner);
chan->client_count++;
}
}
static void dma_device_release(struct kref *ref)
{
struct dma_device *device = container_of(ref, struct dma_device, ref);
list_del_rcu(&device->global_node);
dma_channel_rebalance();
if (device->device_release)
device->device_release(device);
}
static void dma_device_put(struct dma_device *device)
{
lockdep_assert_held(&dma_list_mutex);
kref_put(&device->ref, dma_device_release);
}
/**
* dma_chan_get - try to grab a dma channel's parent driver module
* @chan - channel to grab
*
* Must be called under dma_list_mutex
*/
static int dma_chan_get(struct dma_chan *chan)
{
struct module *owner = dma_chan_to_owner(chan);
int ret;
/* The channel is already in use, update client count */
if (chan->client_count) {
__module_get(owner);
goto out;
}
if (!try_module_get(owner))
return -ENODEV;
ret = kref_get_unless_zero(&chan->device->ref);
if (!ret) {
ret = -ENODEV;
goto module_put_out;
}
/* allocate upon first client reference */
if (chan->device->device_alloc_chan_resources) {
ret = chan->device->device_alloc_chan_resources(chan);
if (ret < 0)
goto err_out;
}
if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
balance_ref_count(chan);
out:
chan->client_count++;
return 0;
err_out:
dma_device_put(chan->device);
module_put_out:
module_put(owner);
return ret;
}
/**
* dma_chan_put - drop a reference to a dma channel's parent driver module
* @chan - channel to release
*
* Must be called under dma_list_mutex
*/
static void dma_chan_put(struct dma_chan *chan)
{
/* This channel is not in use, bail out */
if (!chan->client_count)
return;
chan->client_count--;
/* This channel is not in use anymore, free it */
if (!chan->client_count && chan->device->device_free_chan_resources) {
/* Make sure all operations have completed */
dmaengine_synchronize(chan);
chan->device->device_free_chan_resources(chan);
}
/* If the channel is used via a DMA request router, free the mapping */
if (chan->router && chan->router->route_free) {
chan->router->route_free(chan->router->dev, chan->route_data);
chan->router = NULL;
chan->route_data = NULL;
}
dma_device_put(chan->device);
module_put(dma_chan_to_owner(chan));
}
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
enum dma_status status;
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
dma_async_issue_pending(chan);
do {
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
dev_err(chan->device->dev, "%s: timeout!\n", __func__);
return DMA_ERROR;
}
if (status != DMA_IN_PROGRESS)
break;
cpu_relax();
} while (1);
return status;
}
EXPORT_SYMBOL(dma_sync_wait);
/**
* dma_find_channel - find a channel to carry out the operation
* @tx_type: transaction type
*/
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
{
return this_cpu_read(channel_table[tx_type]->chan);
}
EXPORT_SYMBOL(dma_find_channel);
/**
* dma_issue_pending_all - flush all pending operations across all channels
*/
void dma_issue_pending_all(void)
{
struct dma_device *device;
struct dma_chan *chan;
rcu_read_lock();
list_for_each_entry_rcu(device, &dma_device_list, global_node) {
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
continue;
list_for_each_entry(chan, &device->channels, device_node)
if (chan->client_count)
device->device_issue_pending(chan);
}
rcu_read_unlock();
}
EXPORT_SYMBOL(dma_issue_pending_all);
int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
{
struct dma_device *device;
@ -502,7 +530,7 @@ static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
{
struct dma_chan *chan;
if (mask && !__dma_device_satisfies_mask(dev, mask)) {
if (mask && !dma_device_satisfies_mask(dev, mask)) {
dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
return NULL;
}
@ -704,11 +732,11 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name)
if (has_acpi_companion(dev) && !chan)
chan = acpi_dma_request_slave_chan_by_name(dev, name);
if (chan) {
/* Valid channel found or requester needs to be deferred */
if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER)
if (PTR_ERR(chan) == -EPROBE_DEFER)
return chan;
}
if (!IS_ERR_OR_NULL(chan))
goto found;
/* Try to find the channel via the DMA filter map(s) */
mutex_lock(&dma_list_mutex);
@ -728,7 +756,23 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name)
}
mutex_unlock(&dma_list_mutex);
return chan ? chan : ERR_PTR(-EPROBE_DEFER);
if (!IS_ERR_OR_NULL(chan))
goto found;
return ERR_PTR(-EPROBE_DEFER);
found:
chan->slave = dev;
chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);
if (!chan->name)
return ERR_PTR(-ENOMEM);
if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,
DMA_SLAVE_NAME))
dev_err(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))
dev_err(dev, "Cannot create DMA %s symlink\n", chan->name);
return chan;
}
EXPORT_SYMBOL_GPL(dma_request_chan);
@ -786,6 +830,13 @@ void dma_release_channel(struct dma_chan *chan)
/* drop PRIVATE cap enabled by __dma_request_channel() */
if (--chan->device->privatecnt == 0)
dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
if (chan->slave) {
sysfs_remove_link(&chan->slave->kobj, chan->name);
kfree(chan->name);
chan->name = NULL;
chan->slave = NULL;
}
sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
mutex_unlock(&dma_list_mutex);
}
EXPORT_SYMBOL_GPL(dma_release_channel);
@ -834,14 +885,14 @@ EXPORT_SYMBOL(dmaengine_get);
*/
void dmaengine_put(void)
{
struct dma_device *device;
struct dma_device *device, *_d;
struct dma_chan *chan;
mutex_lock(&dma_list_mutex);
dmaengine_ref_count--;
BUG_ON(dmaengine_ref_count < 0);
/* drop channel references */
list_for_each_entry(device, &dma_device_list, global_node) {
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
continue;
list_for_each_entry(chan, &device->channels, device_node)
@ -900,15 +951,115 @@ static int get_dma_id(struct dma_device *device)
return 0;
}
static int __dma_async_device_channel_register(struct dma_device *device,
struct dma_chan *chan,
int chan_id)
{
int rc = 0;
int chancnt = device->chancnt;
atomic_t *idr_ref;
struct dma_chan *tchan;
tchan = list_first_entry_or_null(&device->channels,
struct dma_chan, device_node);
if (tchan->dev) {
idr_ref = tchan->dev->idr_ref;
} else {
idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
if (!idr_ref)
return -ENOMEM;
atomic_set(idr_ref, 0);
}
chan->local = alloc_percpu(typeof(*chan->local));
if (!chan->local)
goto err_out;
chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
if (!chan->dev) {
free_percpu(chan->local);
chan->local = NULL;
goto err_out;
}
/*
* When the chan_id is a negative value, we are dynamically adding
* the channel. Otherwise we are static enumerating.
*/
chan->chan_id = chan_id < 0 ? chancnt : chan_id;
chan->dev->device.class = &dma_devclass;
chan->dev->device.parent = device->dev;
chan->dev->chan = chan;
chan->dev->idr_ref = idr_ref;
chan->dev->dev_id = device->dev_id;
atomic_inc(idr_ref);
dev_set_name(&chan->dev->device, "dma%dchan%d",
device->dev_id, chan->chan_id);
rc = device_register(&chan->dev->device);
if (rc)
goto err_out;
chan->client_count = 0;
device->chancnt = chan->chan_id + 1;
return 0;
err_out:
free_percpu(chan->local);
kfree(chan->dev);
if (atomic_dec_return(idr_ref) == 0)
kfree(idr_ref);
return rc;
}
int dma_async_device_channel_register(struct dma_device *device,
struct dma_chan *chan)
{
int rc;
rc = __dma_async_device_channel_register(device, chan, -1);
if (rc < 0)
return rc;
dma_channel_rebalance();
return 0;
}
EXPORT_SYMBOL_GPL(dma_async_device_channel_register);
static void __dma_async_device_channel_unregister(struct dma_device *device,
struct dma_chan *chan)
{
WARN_ONCE(!device->device_release && chan->client_count,
"%s called while %d clients hold a reference\n",
__func__, chan->client_count);
mutex_lock(&dma_list_mutex);
list_del(&chan->device_node);
device->chancnt--;
chan->dev->chan = NULL;
mutex_unlock(&dma_list_mutex);
device_unregister(&chan->dev->device);
free_percpu(chan->local);
}
void dma_async_device_channel_unregister(struct dma_device *device,
struct dma_chan *chan)
{
__dma_async_device_channel_unregister(device, chan);
dma_channel_rebalance();
}
EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
/**
* dma_async_device_register - registers DMA devices found
* @device: &dma_device
*
* After calling this routine the structure should not be freed except in the
* device_release() callback which will be called after
* dma_async_device_unregister() is called and no further references are taken.
*/
int dma_async_device_register(struct dma_device *device)
{
int chancnt = 0, rc;
int rc, i = 0;
struct dma_chan* chan;
atomic_t *idr_ref;
if (!device)
return -ENODEV;
@ -919,6 +1070,8 @@ int dma_async_device_register(struct dma_device *device)
return -EIO;
}
device->owner = device->dev->driver->owner;
if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) {
dev_err(device->dev,
"Device claims capability %s, but op is not defined\n",
@ -994,65 +1147,29 @@ int dma_async_device_register(struct dma_device *device)
return -EIO;
}
if (!device->device_release)
dev_warn(device->dev,
"WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");
kref_init(&device->ref);
/* note: this only matters in the
* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
*/
if (device_has_all_tx_types(device))
dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
if (!idr_ref)
return -ENOMEM;
rc = get_dma_id(device);
if (rc != 0) {
kfree(idr_ref);
if (rc != 0)
return rc;
}
atomic_set(idr_ref, 0);
/* represent channels in sysfs. Probably want devs too */
list_for_each_entry(chan, &device->channels, device_node) {
rc = -ENOMEM;
chan->local = alloc_percpu(typeof(*chan->local));
if (chan->local == NULL)
goto err_out;
chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
if (chan->dev == NULL) {
free_percpu(chan->local);
chan->local = NULL;
rc = __dma_async_device_channel_register(device, chan, i++);
if (rc < 0)
goto err_out;
}
chan->chan_id = chancnt++;
chan->dev->device.class = &dma_devclass;
chan->dev->device.parent = device->dev;
chan->dev->chan = chan;
chan->dev->idr_ref = idr_ref;
chan->dev->dev_id = device->dev_id;
atomic_inc(idr_ref);
dev_set_name(&chan->dev->device, "dma%dchan%d",
device->dev_id, chan->chan_id);
rc = device_register(&chan->dev->device);
if (rc) {
free_percpu(chan->local);
chan->local = NULL;
kfree(chan->dev);
atomic_dec(idr_ref);
goto err_out;
}
chan->client_count = 0;
}
if (!chancnt) {
dev_err(device->dev, "%s: device has no channels!\n", __func__);
rc = -ENODEV;
goto err_out;
}
device->chancnt = chancnt;
mutex_lock(&dma_list_mutex);
/* take references on public channels */
if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
@ -1080,9 +1197,8 @@ int dma_async_device_register(struct dma_device *device)
err_out:
/* if we never registered a channel just release the idr */
if (atomic_read(idr_ref) == 0) {
if (!device->chancnt) {
ida_free(&dma_ida, device->dev_id);
kfree(idr_ref);
return rc;
}
@ -1108,23 +1224,20 @@ EXPORT_SYMBOL(dma_async_device_register);
*/
void dma_async_device_unregister(struct dma_device *device)
{
struct dma_chan *chan;
struct dma_chan *chan, *n;
list_for_each_entry_safe(chan, n, &device->channels, device_node)
__dma_async_device_channel_unregister(device, chan);
mutex_lock(&dma_list_mutex);
list_del_rcu(&device->global_node);
/*
* setting DMA_PRIVATE ensures the device being torn down will not
* be used in the channel_table
*/
dma_cap_set(DMA_PRIVATE, device->cap_mask);
dma_channel_rebalance();
dma_device_put(device);
mutex_unlock(&dma_list_mutex);
list_for_each_entry(chan, &device->channels, device_node) {
WARN_ONCE(chan->client_count,
"%s called while %d clients hold a reference\n",
__func__, chan->client_count);
mutex_lock(&dma_list_mutex);
chan->dev->chan = NULL;
mutex_unlock(&dma_list_mutex);
device_unregister(&chan->dev->device);
free_percpu(chan->local);
}
}
EXPORT_SYMBOL(dma_async_device_unregister);
@ -1302,6 +1415,79 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
}
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
static inline int desc_check_and_set_metadata_mode(
struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
{
/* Make sure that the metadata mode is not mixed */
if (!desc->desc_metadata_mode) {
if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
desc->desc_metadata_mode = mode;
else
return -ENOTSUPP;
} else if (desc->desc_metadata_mode != mode) {
return -EINVAL;
}
return 0;
}
int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
void *data, size_t len)
{
int ret;
if (!desc)
return -EINVAL;
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
if (ret)
return ret;
if (!desc->metadata_ops || !desc->metadata_ops->attach)
return -ENOTSUPP;
return desc->metadata_ops->attach(desc, data, len);
}
EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
size_t *payload_len, size_t *max_len)
{
int ret;
if (!desc)
return ERR_PTR(-EINVAL);
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
if (ret)
return ERR_PTR(ret);
if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
return ERR_PTR(-ENOTSUPP);
return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
}
EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
size_t payload_len)
{
int ret;
if (!desc)
return -EINVAL;
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
if (ret)
return ret;
if (!desc->metadata_ops || !desc->metadata_ops->set_len)
return -ENOTSUPP;
return desc->metadata_ops->set_len(desc, payload_len);
}
EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
/* dma_wait_for_async_tx - spin wait for a transaction to complete
* @tx: in-flight transaction to wait on
*/
@ -1373,5 +1559,3 @@ static int __init dma_bus_init(void)
return class_register(&dma_devclass);
}
arch_initcall(dma_bus_init);

11
drivers/dma/dmaengine.h
View File

@ -77,6 +77,7 @@ static inline enum dma_status dma_cookie_status(struct dma_chan *chan,
state->last = complete;
state->used = used;
state->residue = 0;
state->in_flight_bytes = 0;
}
return dma_async_is_complete(cookie, complete, used);
}
@ -87,6 +88,13 @@ static inline void dma_set_residue(struct dma_tx_state *state, u32 residue)
state->residue = residue;
}
static inline void dma_set_in_flight_bytes(struct dma_tx_state *state,
u32 in_flight_bytes)
{
if (state)
state->in_flight_bytes = in_flight_bytes;
}
struct dmaengine_desc_callback {
dma_async_tx_callback callback;
dma_async_tx_callback_result callback_result;
@ -171,4 +179,7 @@ dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb)
return (cb->callback) ? true : false;
}
struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
struct dma_chan *dma_get_any_slave_channel(struct dma_device *device);
#endif

8
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
View File

@ -636,14 +636,10 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
vchan_get_all_descriptors(&chan->vc, &head);
/*
* As vchan_dma_desc_free_list can access to desc_allocated list
* we need to call it in vc.lock context.
*/
vchan_dma_desc_free_list(&chan->vc, &head);
spin_unlock_irqrestore(&chan->vc.lock, flags);
vchan_dma_desc_free_list(&chan->vc, &head);
dev_vdbg(dchan2dev(dchan), "terminated: %s\n", axi_chan_name(chan));
return 0;

5
drivers/dma/fsl-edma-common.c
View File

@ -109,10 +109,15 @@ void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,
u32 ch = fsl_chan->vchan.chan.chan_id;
void __iomem *muxaddr;
unsigned int chans_per_mux, ch_off;
int endian_diff[4] = {3, 1, -1, -3};
u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs;
chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr;
ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux;
if (fsl_chan->edma->drvdata->mux_swap)
ch_off += endian_diff[ch_off % 4];
muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux];
slot = EDMAMUX_CHCFG_SOURCE(slot);

1
drivers/dma/fsl-edma-common.h
View File

@ -147,6 +147,7 @@ struct fsl_edma_drvdata {
enum edma_version version;
u32 dmamuxs;
bool has_dmaclk;
bool mux_swap;
int (*setup_irq)(struct platform_device *pdev,
struct fsl_edma_engine *fsl_edma);
};

8
drivers/dma/fsl-edma.c
View File

@ -233,6 +233,13 @@ static struct fsl_edma_drvdata vf610_data = {
.setup_irq = fsl_edma_irq_init,
};
static struct fsl_edma_drvdata ls1028a_data = {
.version = v1,
.dmamuxs = DMAMUX_NR,
.mux_swap = true,
.setup_irq = fsl_edma_irq_init,
};
static struct fsl_edma_drvdata imx7ulp_data = {
.version = v3,
.dmamuxs = 1,
@ -242,6 +249,7 @@ static struct fsl_edma_drvdata imx7ulp_data = {
static const struct of_device_id fsl_edma_dt_ids[] = {
{ .compatible = "fsl,vf610-edma", .data = &vf610_data},
{ .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data},
{ .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data},
{ /* sentinel */ }
};

2
drivers/dma/fsl-qdma.c
View File

@ -304,7 +304,7 @@ static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
if (!fsl_queue->comp_pool && !fsl_queue->comp_pool)
if (!fsl_queue->comp_pool && !fsl_queue->desc_pool)
return;
list_for_each_entry_safe(comp_temp, _comp_temp,

611
drivers/dma/hisi_dma.c Normal file
View File

@ -0,0 +1,611 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2019 HiSilicon Limited. */
#include <linux/bitfield.h>
#include <linux/dmaengine.h>
#include <linux/init.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include "virt-dma.h"
#define HISI_DMA_SQ_BASE_L 0x0
#define HISI_DMA_SQ_BASE_H 0x4
#define HISI_DMA_SQ_DEPTH 0x8
#define HISI_DMA_SQ_TAIL_PTR 0xc
#define HISI_DMA_CQ_BASE_L 0x10
#define HISI_DMA_CQ_BASE_H 0x14
#define HISI_DMA_CQ_DEPTH 0x18
#define HISI_DMA_CQ_HEAD_PTR 0x1c
#define HISI_DMA_CTRL0 0x20
#define HISI_DMA_CTRL0_QUEUE_EN_S 0
#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
#define HISI_DMA_CTRL1 0x24
#define HISI_DMA_CTRL1_QUEUE_RESET_S 0
#define HISI_DMA_Q_FSM_STS 0x30
#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
#define HISI_DMA_INT_STS 0x40
#define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
#define HISI_DMA_INT_MSK 0x44
#define HISI_DMA_MODE 0x217c
#define HISI_DMA_OFFSET 0x100
#define HISI_DMA_MSI_NUM 30
#define HISI_DMA_CHAN_NUM 30
#define HISI_DMA_Q_DEPTH_VAL 1024
#define PCI_BAR_2 2
enum hisi_dma_mode {
EP = 0,
RC,
};
enum hisi_dma_chan_status {
DISABLE = -1,
IDLE = 0,
RUN,
CPL,
PAUSE,
HALT,
ABORT,
WAIT,
BUFFCLR,
};
struct hisi_dma_sqe {
__le32 dw0;
#define OPCODE_MASK GENMASK(3, 0)
#define OPCODE_SMALL_PACKAGE 0x1
#define OPCODE_M2M 0x4
#define LOCAL_IRQ_EN BIT(8)
#define ATTR_SRC_MASK GENMASK(14, 12)
__le32 dw1;
__le32 dw2;
#define ATTR_DST_MASK GENMASK(26, 24)
__le32 length;
__le64 src_addr;
__le64 dst_addr;
};
struct hisi_dma_cqe {
__le32 rsv0;
__le32 rsv1;
__le16 sq_head;
__le16 rsv2;
__le16 rsv3;
__le16 w0;
#define STATUS_MASK GENMASK(15, 1)
#define STATUS_SUCC 0x0
#define VALID_BIT BIT(0)
};
struct hisi_dma_desc {
struct virt_dma_desc vd;
struct hisi_dma_sqe sqe;
};
struct hisi_dma_chan {
struct virt_dma_chan vc;
struct hisi_dma_dev *hdma_dev;
struct hisi_dma_sqe *sq;
struct hisi_dma_cqe *cq;
dma_addr_t sq_dma;
dma_addr_t cq_dma;
u32 sq_tail;
u32 cq_head;
u32 qp_num;
enum hisi_dma_chan_status status;
struct hisi_dma_desc *desc;
};
struct hisi_dma_dev {
struct pci_dev *pdev;
void __iomem *base;
struct dma_device dma_dev;
u32 chan_num;
u32 chan_depth;
struct hisi_dma_chan chan[];
};
static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
{
return container_of(c, struct hisi_dma_chan, vc.chan);
}
static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
{
return container_of(vd, struct hisi_dma_desc, vd);
}
static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
u32 val)
{
writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
}
static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
{
u32 tmp;
tmp = readl_relaxed(addr);
tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
writel_relaxed(tmp, addr);
}
static void hisi_dma_free_irq_vectors(void *data)
{
pci_free_irq_vectors(data);
}
static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
bool pause)
{
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
HISI_DMA_OFFSET;
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
}
static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
bool enable)
{
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
HISI_DMA_OFFSET;
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
}
static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
HISI_DMA_INT_STS_MASK);
}
static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
void __iomem *base = hdma_dev->base;
hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
HISI_DMA_INT_STS_MASK);
hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
}
static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
{
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
HISI_DMA_OFFSET;
hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
}
static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
{
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
}
static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
{
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
u32 index = chan->qp_num, tmp;
int ret;
hisi_dma_pause_dma(hdma_dev, index, true);
hisi_dma_enable_dma(hdma_dev, index, false);
hisi_dma_mask_irq(hdma_dev, index);
ret = readl_relaxed_poll_timeout(hdma_dev->base +
HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
if (ret) {
dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
WARN_ON(1);
}
hisi_dma_do_reset(hdma_dev, index);
hisi_dma_reset_qp_point(hdma_dev, index);
hisi_dma_pause_dma(hdma_dev, index, false);
hisi_dma_enable_dma(hdma_dev, index, true);
hisi_dma_unmask_irq(hdma_dev, index);
ret = readl_relaxed_poll_timeout(hdma_dev->base +
HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
if (ret) {
dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
WARN_ON(1);
}
}
static void hisi_dma_free_chan_resources(struct dma_chan *c)
{
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
hisi_dma_reset_hw_chan(chan);
vchan_free_chan_resources(&chan->vc);
memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
chan->sq_tail = 0;
chan->cq_head = 0;
chan->status = DISABLE;
}
static void hisi_dma_desc_free(struct virt_dma_desc *vd)
{
kfree(to_hisi_dma_desc(vd));
}
static struct dma_async_tx_descriptor *
hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
size_t len, unsigned long flags)
{
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
struct hisi_dma_desc *desc;
desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
if (!desc)
return NULL;
desc->sqe.length = cpu_to_le32(len);
desc->sqe.src_addr = cpu_to_le64(src);
desc->sqe.dst_addr = cpu_to_le64(dst);
return vchan_tx_prep(&chan->vc, &desc->vd, flags);
}
static enum dma_status
hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate)
{
return dma_cookie_status(c, cookie, txstate);
}
static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
{
struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
struct hisi_dma_desc *desc;
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
if (!vd) {
dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
chan->desc = NULL;
return;
}
list_del(&vd->node);
desc = to_hisi_dma_desc(vd);
chan->desc = desc;
memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
/* update other field in sqe */
sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
/* make sure data has been updated in sqe */
wmb();
/* update sq tail, point to new sqe position */
chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
/* update sq_tail to trigger a new task */
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
chan->sq_tail);
}
static void hisi_dma_issue_pending(struct dma_chan *c)
{
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
if (vchan_issue_pending(&chan->vc))
hisi_dma_start_transfer(chan);
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
static int hisi_dma_terminate_all(struct dma_chan *c)
{
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
unsigned long flags;
LIST_HEAD(head);
spin_lock_irqsave(&chan->vc.lock, flags);
hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
if (chan->desc) {
vchan_terminate_vdesc(&chan->desc->vd);
chan->desc = NULL;
}
vchan_get_all_descriptors(&chan->vc, &head);
spin_unlock_irqrestore(&chan->vc.lock, flags);
vchan_dma_desc_free_list(&chan->vc, &head);
hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
return 0;
}
static void hisi_dma_synchronize(struct dma_chan *c)
{
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
vchan_synchronize(&chan->vc);
}
static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
{
size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
struct device *dev = &hdma_dev->pdev->dev;
struct hisi_dma_chan *chan;
int i;
for (i = 0; i < hdma_dev->chan_num; i++) {
chan = &hdma_dev->chan[i];
chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
GFP_KERNEL);
if (!chan->sq)
return -ENOMEM;
chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
GFP_KERNEL);
if (!chan->cq)
return -ENOMEM;
}
return 0;
}
static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
{
struct hisi_dma_chan *chan = &hdma_dev->chan[index];
u32 hw_depth = hdma_dev->chan_depth - 1;
void __iomem *base = hdma_dev->base;
/* set sq, cq base */
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
lower_32_bits(chan->sq_dma));
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
upper_32_bits(chan->sq_dma));
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
lower_32_bits(chan->cq_dma));
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
upper_32_bits(chan->cq_dma));
/* set sq, cq depth */
hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
/* init sq tail and cq head */
hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
}
static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
hisi_dma_init_hw_qp(hdma_dev, qp_index);
hisi_dma_unmask_irq(hdma_dev, qp_index);
hisi_dma_enable_dma(hdma_dev, qp_index, true);
}
static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
}
static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
{
int i;
for (i = 0; i < hdma_dev->chan_num; i++) {
hdma_dev->chan[i].qp_num = i;
hdma_dev->chan[i].hdma_dev = hdma_dev;
hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
hisi_dma_enable_qp(hdma_dev, i);
}
}
static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
{
int i;
for (i = 0; i < hdma_dev->chan_num; i++) {
hisi_dma_disable_qp(hdma_dev, i);
tasklet_kill(&hdma_dev->chan[i].vc.task);
}
}
static irqreturn_t hisi_dma_irq(int irq, void *data)
{
struct hisi_dma_chan *chan = data;
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
struct hisi_dma_desc *desc;
struct hisi_dma_cqe *cqe;
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
desc = chan->desc;
cqe = chan->cq + chan->cq_head;
if (desc) {
if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
chan->cq_head = (chan->cq_head + 1) %
hdma_dev->chan_depth;
hisi_dma_chan_write(hdma_dev->base,
HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
chan->cq_head);
vchan_cookie_complete(&desc->vd);
} else {
dev_err(&hdma_dev->pdev->dev, "task error!\n");
}
chan->desc = NULL;
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
return IRQ_HANDLED;
}
static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
{
struct pci_dev *pdev = hdma_dev->pdev;
int i, ret;
for (i = 0; i < hdma_dev->chan_num; i++) {
ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
hisi_dma_irq, IRQF_SHARED, "hisi_dma",
&hdma_dev->chan[i]);
if (ret)
return ret;
}
return 0;
}
/* This function enables all hw channels in a device */
static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
{
int ret;
ret = hisi_dma_alloc_qps_mem(hdma_dev);
if (ret) {
dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
return ret;
}
ret = hisi_dma_request_qps_irq(hdma_dev);
if (ret) {
dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
return ret;
}
hisi_dma_enable_qps(hdma_dev);
return 0;
}
static void hisi_dma_disable_hw_channels(void *data)
{
hisi_dma_disable_qps(data);
}
static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
enum hisi_dma_mode mode)
{
writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
}
static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct device *dev = &pdev->dev;
struct hisi_dma_dev *hdma_dev;
struct dma_device *dma_dev;
size_t dev_size;
int ret;
ret = pcim_enable_device(pdev);
if (ret) {
dev_err(dev, "failed to enable device mem!\n");
return ret;
}
ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
if (ret) {
dev_err(dev, "failed to remap I/O region!\n");
return ret;
}
ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (ret)
return ret;
ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (ret)
return ret;
dev_size = sizeof(struct hisi_dma_chan) * HISI_DMA_CHAN_NUM +
sizeof(*hdma_dev);
hdma_dev = devm_kzalloc(dev, dev_size, GFP_KERNEL);
if (!hdma_dev)
return -EINVAL;
hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
hdma_dev->pdev = pdev;
hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
pci_set_drvdata(pdev, hdma_dev);
pci_set_master(pdev);
ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
PCI_IRQ_MSI);
if (ret < 0) {
dev_err(dev, "Failed to allocate MSI vectors!\n");
return ret;
}
ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
if (ret)
return ret;
dma_dev = &hdma_dev->dma_dev;
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
dma_dev->device_tx_status = hisi_dma_tx_status;
dma_dev->device_issue_pending = hisi_dma_issue_pending;
dma_dev->device_terminate_all = hisi_dma_terminate_all;
dma_dev->device_synchronize = hisi_dma_synchronize;
dma_dev->directions = BIT(DMA_MEM_TO_MEM);
dma_dev->dev = dev;
INIT_LIST_HEAD(&dma_dev->channels);
hisi_dma_set_mode(hdma_dev, RC);
ret = hisi_dma_enable_hw_channels(hdma_dev);
if (ret < 0) {
dev_err(dev, "failed to enable hw channel!\n");
return ret;
}
ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
hdma_dev);
if (ret)
return ret;
ret = dmaenginem_async_device_register(dma_dev);
if (ret < 0)
dev_err(dev, "failed to register device!\n");
return ret;
}
static const struct pci_device_id hisi_dma_pci_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
{ 0, }
};
static struct pci_driver hisi_dma_pci_driver = {
.name = "hisi_dma",
.id_table = hisi_dma_pci_tbl,
.probe = hisi_dma_probe,
};
module_pci_driver(hisi_dma_pci_driver);
MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
MODULE_LICENSE("GPL v2");
MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);

2
drivers/dma/idxd/Makefile Normal file
View File

@ -0,0 +1,2 @@
obj-$(CONFIG_INTEL_IDXD) += idxd.o
idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o

302
drivers/dma/idxd/cdev.c Normal file
View File

@ -0,0 +1,302 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/sched/task.h>
#include <linux/intel-svm.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/poll.h>
#include <uapi/linux/idxd.h>
#include "registers.h"
#include "idxd.h"
struct idxd_cdev_context {
const char *name;
dev_t devt;
struct ida minor_ida;
};
/*
* ictx is an array based off of accelerator types. enum idxd_type
* is used as index
*/
static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
{ .name = "dsa" },
};
struct idxd_user_context {
struct idxd_wq *wq;
struct task_struct *task;
unsigned int flags;
};
enum idxd_cdev_cleanup {
CDEV_NORMAL = 0,
CDEV_FAILED,
};
static void idxd_cdev_dev_release(struct device *dev)
{
dev_dbg(dev, "releasing cdev device\n");
kfree(dev);
}
static struct device_type idxd_cdev_device_type = {
.name = "idxd_cdev",
.release = idxd_cdev_dev_release,
};
static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
{
struct cdev *cdev = inode->i_cdev;
return container_of(cdev, struct idxd_cdev, cdev);
}
static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev)
{
return container_of(idxd_cdev, struct idxd_wq, idxd_cdev);
}
static inline struct idxd_wq *inode_wq(struct inode *inode)
{
return idxd_cdev_wq(inode_idxd_cdev(inode));
}
static int idxd_cdev_open(struct inode *inode, struct file *filp)
{
struct idxd_user_context *ctx;
struct idxd_device *idxd;
struct idxd_wq *wq;
struct device *dev;
struct idxd_cdev *idxd_cdev;
wq = inode_wq(inode);
idxd = wq->idxd;
dev = &idxd->pdev->dev;
idxd_cdev = &wq->idxd_cdev;
dev_dbg(dev, "%s called\n", __func__);
if (idxd_wq_refcount(wq) > 1 && wq_dedicated(wq))
return -EBUSY;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
ctx->wq = wq;
filp->private_data = ctx;
idxd_wq_get(wq);
return 0;
}
static int idxd_cdev_release(struct inode *node, struct file *filep)
{
struct idxd_user_context *ctx = filep->private_data;
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
dev_dbg(dev, "%s called\n", __func__);
filep->private_data = NULL;
kfree(ctx);
idxd_wq_put(wq);
return 0;
}
static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
const char *func)
{
struct device *dev = &wq->idxd->pdev->dev;
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
dev_info_ratelimited(dev,
"%s: %s: mapping too large: %lu\n",
current->comm, func,
vma->vm_end - vma->vm_start);
return -EINVAL;
}
return 0;
}
static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct idxd_user_context *ctx = filp->private_data;
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
struct pci_dev *pdev = idxd->pdev;
phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
unsigned long pfn;
int rc;
dev_dbg(&pdev->dev, "%s called\n", __func__);
rc = check_vma(wq, vma, __func__);
vma->vm_flags |= VM_DONTCOPY;
pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vma->vm_private_data = ctx;
return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
vma->vm_page_prot);
}
static __poll_t idxd_cdev_poll(struct file *filp,
struct poll_table_struct *wait)
{
struct idxd_user_context *ctx = filp->private_data;
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
unsigned long flags;
__poll_t out = 0;
poll_wait(filp, &idxd_cdev->err_queue, wait);
spin_lock_irqsave(&idxd->dev_lock, flags);
if (idxd->sw_err.valid)
out = EPOLLIN | EPOLLRDNORM;
spin_unlock_irqrestore(&idxd->dev_lock, flags);
return out;
}
static const struct file_operations idxd_cdev_fops = {
.owner = THIS_MODULE,
.open = idxd_cdev_open,
.release = idxd_cdev_release,
.mmap = idxd_cdev_mmap,
.poll = idxd_cdev_poll,
};
int idxd_cdev_get_major(struct idxd_device *idxd)
{
return MAJOR(ictx[idxd->type].devt);
}
static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct idxd_cdev_context *cdev_ctx;
struct device *dev;
int minor, rc;
idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL);
if (!idxd_cdev->dev)
return -ENOMEM;
dev = idxd_cdev->dev;
dev->parent = &idxd->pdev->dev;
dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
idxd->id, wq->id);
dev->bus = idxd_get_bus_type(idxd);
cdev_ctx = &ictx[wq->idxd->type];
minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
if (minor < 0) {
rc = minor;
goto ida_err;
}
dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
dev->type = &idxd_cdev_device_type;
rc = device_register(dev);
if (rc < 0) {
dev_err(&idxd->pdev->dev, "device register failed\n");
put_device(dev);
goto dev_reg_err;
}
idxd_cdev->minor = minor;
return 0;
dev_reg_err:
ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt));
ida_err:
kfree(dev);
idxd_cdev->dev = NULL;
return rc;
}
static void idxd_wq_cdev_cleanup(struct idxd_wq *wq,
enum idxd_cdev_cleanup cdev_state)
{
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct idxd_cdev_context *cdev_ctx;
cdev_ctx = &ictx[wq->idxd->type];
if (cdev_state == CDEV_NORMAL)
cdev_del(&idxd_cdev->cdev);
device_unregister(idxd_cdev->dev);
/*
* The device_type->release() will be called on the device and free
* the allocated struct device. We can just forget it.
*/
ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
idxd_cdev->dev = NULL;
idxd_cdev->minor = -1;
}
int idxd_wq_add_cdev(struct idxd_wq *wq)
{
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct cdev *cdev = &idxd_cdev->cdev;
struct device *dev;
int rc;
rc = idxd_wq_cdev_dev_setup(wq);
if (rc < 0)
return rc;
dev = idxd_cdev->dev;
cdev_init(cdev, &idxd_cdev_fops);
cdev_set_parent(cdev, &dev->kobj);
rc = cdev_add(cdev, dev->devt, 1);
if (rc) {
dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
idxd_wq_cdev_cleanup(wq, CDEV_FAILED);
return rc;
}
init_waitqueue_head(&idxd_cdev->err_queue);
return 0;
}
void idxd_wq_del_cdev(struct idxd_wq *wq)
{
idxd_wq_cdev_cleanup(wq, CDEV_NORMAL);
}
int idxd_cdev_register(void)
{
int rc, i;
for (i = 0; i < IDXD_TYPE_MAX; i++) {
ida_init(&ictx[i].minor_ida);
rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
ictx[i].name);
if (rc)
return rc;
}
return 0;
}
void idxd_cdev_remove(void)
{
int i;
for (i = 0; i < IDXD_TYPE_MAX; i++) {
unregister_chrdev_region(ictx[i].devt, MINORMASK);
ida_destroy(&ictx[i].minor_ida);
}
}

693
drivers/dma/idxd/device.c Normal file
View File

@ -0,0 +1,693 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/dmaengine.h>
#include <uapi/linux/idxd.h>
#include "../dmaengine.h"
#include "idxd.h"
#include "registers.h"
static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout);
static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand);
/* Interrupt control bits */
int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
{
struct pci_dev *pdev = idxd->pdev;
int msixcnt = pci_msix_vec_count(pdev);
union msix_perm perm;
u32 offset;
if (vec_id < 0 || vec_id >= msixcnt)
return -EINVAL;
offset = idxd->msix_perm_offset + vec_id * 8;
perm.bits = ioread32(idxd->reg_base + offset);
perm.ignore = 1;
iowrite32(perm.bits, idxd->reg_base + offset);
return 0;
}
void idxd_mask_msix_vectors(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
int msixcnt = pci_msix_vec_count(pdev);
int i, rc;
for (i = 0; i < msixcnt; i++) {
rc = idxd_mask_msix_vector(idxd, i);
if (rc < 0)
dev_warn(&pdev->dev,
"Failed disabling msix vec %d\n", i);
}
}
int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
{
struct pci_dev *pdev = idxd->pdev;
int msixcnt = pci_msix_vec_count(pdev);
union msix_perm perm;
u32 offset;
if (vec_id < 0 || vec_id >= msixcnt)
return -EINVAL;
offset = idxd->msix_perm_offset + vec_id * 8;
perm.bits = ioread32(idxd->reg_base + offset);
perm.ignore = 0;
iowrite32(perm.bits, idxd->reg_base + offset);
return 0;
}
void idxd_unmask_error_interrupts(struct idxd_device *idxd)
{
union genctrl_reg genctrl;
genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
genctrl.softerr_int_en = 1;
iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
}
void idxd_mask_error_interrupts(struct idxd_device *idxd)
{
union genctrl_reg genctrl;
genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
genctrl.softerr_int_en = 0;
iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
}
static void free_hw_descs(struct idxd_wq *wq)
{
int i;
for (i = 0; i < wq->num_descs; i++)
kfree(wq->hw_descs[i]);
kfree(wq->hw_descs);
}
static int alloc_hw_descs(struct idxd_wq *wq, int num)
{
struct device *dev = &wq->idxd->pdev->dev;
int i;
int node = dev_to_node(dev);
wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
GFP_KERNEL, node);
if (!wq->hw_descs)
return -ENOMEM;
for (i = 0; i < num; i++) {
wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
GFP_KERNEL, node);
if (!wq->hw_descs[i]) {
free_hw_descs(wq);
return -ENOMEM;
}
}
return 0;
}
static void free_descs(struct idxd_wq *wq)
{
int i;
for (i = 0; i < wq->num_descs; i++)
kfree(wq->descs[i]);
kfree(wq->descs);
}
static int alloc_descs(struct idxd_wq *wq, int num)
{
struct device *dev = &wq->idxd->pdev->dev;
int i;
int node = dev_to_node(dev);
wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
GFP_KERNEL, node);
if (!wq->descs)
return -ENOMEM;
for (i = 0; i < num; i++) {
wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
GFP_KERNEL, node);
if (!wq->descs[i]) {
free_descs(wq);
return -ENOMEM;
}
}
return 0;
}
/* WQ control bits */
int idxd_wq_alloc_resources(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct idxd_group *group = wq->group;
struct device *dev = &idxd->pdev->dev;
int rc, num_descs, i;
if (wq->type != IDXD_WQT_KERNEL)
return 0;
num_descs = wq->size +
idxd->hw.gen_cap.max_descs_per_engine * group->num_engines;
wq->num_descs = num_descs;
rc = alloc_hw_descs(wq, num_descs);
if (rc < 0)
return rc;
wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
wq->compls = dma_alloc_coherent(dev, wq->compls_size,
&wq->compls_addr, GFP_KERNEL);
if (!wq->compls) {
rc = -ENOMEM;
goto fail_alloc_compls;
}
rc = alloc_descs(wq, num_descs);
if (rc < 0)
goto fail_alloc_descs;
rc = sbitmap_init_node(&wq->sbmap, num_descs, -1, GFP_KERNEL,
dev_to_node(dev));
if (rc < 0)
goto fail_sbitmap_init;
for (i = 0; i < num_descs; i++) {
struct idxd_desc *desc = wq->descs[i];
desc->hw = wq->hw_descs[i];
desc->completion = &wq->compls[i];
desc->compl_dma = wq->compls_addr +
sizeof(struct dsa_completion_record) * i;
desc->id = i;
desc->wq = wq;
dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
desc->txd.tx_submit = idxd_dma_tx_submit;
}
return 0;
fail_sbitmap_init:
free_descs(wq);
fail_alloc_descs:
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
fail_alloc_compls:
free_hw_descs(wq);
return rc;
}
void idxd_wq_free_resources(struct idxd_wq *wq)
{
struct device *dev = &wq->idxd->pdev->dev;
if (wq->type != IDXD_WQT_KERNEL)
return;
free_hw_descs(wq);
free_descs(wq);
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
sbitmap_free(&wq->sbmap);
}
int idxd_wq_enable(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 status;
int rc;
lockdep_assert_held(&idxd->dev_lock);
if (wq->state == IDXD_WQ_ENABLED) {
dev_dbg(dev, "WQ %d already enabled\n", wq->id);
return -ENXIO;
}
rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_WQ, wq->id);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
if (status != IDXD_CMDSTS_SUCCESS &&
status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
dev_dbg(dev, "WQ enable failed: %#x\n", status);
return -ENXIO;
}
wq->state = IDXD_WQ_ENABLED;
dev_dbg(dev, "WQ %d enabled\n", wq->id);
return 0;
}
int idxd_wq_disable(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 status, operand;
int rc;
lockdep_assert_held(&idxd->dev_lock);
dev_dbg(dev, "Disabling WQ %d\n", wq->id);
if (wq->state != IDXD_WQ_ENABLED) {
dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
return 0;
}
operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_WQ, operand);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
if (status != IDXD_CMDSTS_SUCCESS) {
dev_dbg(dev, "WQ disable failed: %#x\n", status);
return -ENXIO;
}
wq->state = IDXD_WQ_DISABLED;
dev_dbg(dev, "WQ %d disabled\n", wq->id);
return 0;
}
int idxd_wq_map_portal(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct pci_dev *pdev = idxd->pdev;
struct device *dev = &pdev->dev;
resource_size_t start;
start = pci_resource_start(pdev, IDXD_WQ_BAR);
start = start + wq->id * IDXD_PORTAL_SIZE;
wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
if (!wq->dportal)
return -ENOMEM;
dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
return 0;
}
void idxd_wq_unmap_portal(struct idxd_wq *wq)
{
struct device *dev = &wq->idxd->pdev->dev;
devm_iounmap(dev, wq->dportal);
}
/* Device control bits */
static inline bool idxd_is_enabled(struct idxd_device *idxd)
{
union gensts_reg gensts;
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
return true;
return false;
}
static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout)
{
u32 sts, to = timeout;
lockdep_assert_held(&idxd->dev_lock);
sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
while (sts & IDXD_CMDSTS_ACTIVE && --to) {
cpu_relax();
sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
}
if (to == 0 && sts & IDXD_CMDSTS_ACTIVE) {
dev_warn(&idxd->pdev->dev, "%s timed out!\n", __func__);
*status = 0;
return -EBUSY;
}
*status = sts;
return 0;
}
static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand)
{
union idxd_command_reg cmd;
int rc;
u32 status;
lockdep_assert_held(&idxd->dev_lock);
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = cmd_code;
cmd.operand = operand;
dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
__func__, cmd_code, operand);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
return 0;
}
int idxd_device_enable(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int rc;
u32 status;
lockdep_assert_held(&idxd->dev_lock);
if (idxd_is_enabled(idxd)) {
dev_dbg(dev, "Device already enabled\n");
return -ENXIO;
}
rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
/* If the command is successful or if the device was enabled */
if (status != IDXD_CMDSTS_SUCCESS &&
status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
return -ENXIO;
}
idxd->state = IDXD_DEV_ENABLED;
return 0;
}
int idxd_device_disable(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int rc;
u32 status;
lockdep_assert_held(&idxd->dev_lock);
if (!idxd_is_enabled(idxd)) {
dev_dbg(dev, "Device is not enabled\n");
return 0;
}
rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
/* If the command is successful or if the device was disabled */
if (status != IDXD_CMDSTS_SUCCESS &&
!(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
rc = -ENXIO;
return rc;
}
idxd->state = IDXD_DEV_CONF_READY;
return 0;
}
int __idxd_device_reset(struct idxd_device *idxd)
{
u32 status;
int rc;
rc = idxd_cmd_send(idxd, IDXD_CMD_RESET_DEVICE, 0);
if (rc < 0)
return rc;
rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
if (rc < 0)
return rc;
return 0;
}
int idxd_device_reset(struct idxd_device *idxd)
{
unsigned long flags;
int rc;
spin_lock_irqsave(&idxd->dev_lock, flags);
rc = __idxd_device_reset(idxd);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
return rc;
}
/* Device configuration bits */
static void idxd_group_config_write(struct idxd_group *group)
{
struct idxd_device *idxd = group->idxd;
struct device *dev = &idxd->pdev->dev;
int i;
u32 grpcfg_offset;
dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
/* setup GRPWQCFG */
for (i = 0; i < 4; i++) {
grpcfg_offset = idxd->grpcfg_offset +
group->id * 64 + i * sizeof(u64);
iowrite64(group->grpcfg.wqs[i],
idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
group->id, i, grpcfg_offset,
ioread64(idxd->reg_base + grpcfg_offset));
}
/* setup GRPENGCFG */
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
/* setup GRPFLAGS */
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
group->id, grpcfg_offset,
ioread32(idxd->reg_base + grpcfg_offset));
}
static int idxd_groups_config_write(struct idxd_device *idxd)
{
union gencfg_reg reg;
int i;
struct device *dev = &idxd->pdev->dev;
/* Setup bandwidth token limit */
if (idxd->token_limit) {
reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
reg.token_limit = idxd->token_limit;
iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
}
dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
for (i = 0; i < idxd->max_groups; i++) {
struct idxd_group *group = &idxd->groups[i];
idxd_group_config_write(group);
}
return 0;
}
static int idxd_wq_config_write(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
u32 wq_offset;
int i;
if (!wq->group)
return 0;
memset(&wq->wqcfg, 0, sizeof(union wqcfg));
/* byte 0-3 */
wq->wqcfg.wq_size = wq->size;
if (wq->size == 0) {
dev_warn(dev, "Incorrect work queue size: 0\n");
return -EINVAL;
}
/* bytes 4-7 */
wq->wqcfg.wq_thresh = wq->threshold;
/* byte 8-11 */
wq->wqcfg.priv = !!(wq->type == IDXD_WQT_KERNEL);
wq->wqcfg.mode = 1;
wq->wqcfg.priority = wq->priority;
/* bytes 12-15 */
wq->wqcfg.max_xfer_shift = idxd->hw.gen_cap.max_xfer_shift;
wq->wqcfg.max_batch_shift = idxd->hw.gen_cap.max_batch_shift;
dev_dbg(dev, "WQ %d CFGs\n", wq->id);
for (i = 0; i < 8; i++) {
wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32);
iowrite32(wq->wqcfg.bits[i], idxd->reg_base + wq_offset);
dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
wq->id, i, wq_offset,
ioread32(idxd->reg_base + wq_offset));
}
return 0;
}
static int idxd_wqs_config_write(struct idxd_device *idxd)
{
int i, rc;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
rc = idxd_wq_config_write(wq);
if (rc < 0)
return rc;
}
return 0;
}
static void idxd_group_flags_setup(struct idxd_device *idxd)
{
int i;
/* TC-A 0 and TC-B 1 should be defaults */
for (i = 0; i < idxd->max_groups; i++) {
struct idxd_group *group = &idxd->groups[i];
if (group->tc_a == -1)
group->grpcfg.flags.tc_a = 0;
else
group->grpcfg.flags.tc_a = group->tc_a;
if (group->tc_b == -1)
group->grpcfg.flags.tc_b = 1;
else
group->grpcfg.flags.tc_b = group->tc_b;
group->grpcfg.flags.use_token_limit = group->use_token_limit;
group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
if (group->tokens_allowed)
group->grpcfg.flags.tokens_allowed =
group->tokens_allowed;
else
group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
}
}
static int idxd_engines_setup(struct idxd_device *idxd)
{
int i, engines = 0;
struct idxd_engine *eng;
struct idxd_group *group;
for (i = 0; i < idxd->max_groups; i++) {
group = &idxd->groups[i];
group->grpcfg.engines = 0;
}
for (i = 0; i < idxd->max_engines; i++) {
eng = &idxd->engines[i];
group = eng->group;
if (!group)
continue;
group->grpcfg.engines |= BIT(eng->id);
engines++;
}
if (!engines)
return -EINVAL;
return 0;
}
static int idxd_wqs_setup(struct idxd_device *idxd)
{
struct idxd_wq *wq;
struct idxd_group *group;
int i, j, configured = 0;
struct device *dev = &idxd->pdev->dev;
for (i = 0; i < idxd->max_groups; i++) {
group = &idxd->groups[i];
for (j = 0; j < 4; j++)
group->grpcfg.wqs[j] = 0;
}
for (i = 0; i < idxd->max_wqs; i++) {
wq = &idxd->wqs[i];
group = wq->group;
if (!wq->group)
continue;
if (!wq->size)
continue;
if (!wq_dedicated(wq)) {
dev_warn(dev, "No shared workqueue support.\n");
return -EINVAL;
}
group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
configured++;
}
if (configured == 0)
return -EINVAL;
return 0;
}
int idxd_device_config(struct idxd_device *idxd)
{
int rc;
lockdep_assert_held(&idxd->dev_lock);
rc = idxd_wqs_setup(idxd);
if (rc < 0)
return rc;
rc = idxd_engines_setup(idxd);
if (rc < 0)
return rc;
idxd_group_flags_setup(idxd);
rc = idxd_wqs_config_write(idxd);
if (rc < 0)
return rc;
rc = idxd_groups_config_write(idxd);
if (rc < 0)
return rc;
return 0;
}

217
drivers/dma/idxd/dma.c Normal file
View File

@ -0,0 +1,217 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/dmaengine.h>
#include <uapi/linux/idxd.h>
#include "../dmaengine.h"
#include "registers.h"
#include "idxd.h"
static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
{
return container_of(c, struct idxd_wq, dma_chan);
}
void idxd_dma_complete_txd(struct idxd_desc *desc,
enum idxd_complete_type comp_type)
{
struct dma_async_tx_descriptor *tx;
struct dmaengine_result res;
int complete = 1;
if (desc->completion->status == DSA_COMP_SUCCESS)
res.result = DMA_TRANS_NOERROR;
else if (desc->completion->status)
res.result = DMA_TRANS_WRITE_FAILED;
else if (comp_type == IDXD_COMPLETE_ABORT)
res.result = DMA_TRANS_ABORTED;
else
complete = 0;
tx = &desc->txd;
if (complete && tx->cookie) {
dma_cookie_complete(tx);
dma_descriptor_unmap(tx);
dmaengine_desc_get_callback_invoke(tx, &res);
tx->callback = NULL;
tx->callback_result = NULL;
}
}
static void op_flag_setup(unsigned long flags, u32 *desc_flags)
{
*desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
if (flags & DMA_PREP_INTERRUPT)
*desc_flags |= IDXD_OP_FLAG_RCI;
}
static inline void set_completion_address(struct idxd_desc *desc,
u64 *compl_addr)
{
*compl_addr = desc->compl_dma;
}
static inline void idxd_prep_desc_common(struct idxd_wq *wq,
struct dsa_hw_desc *hw, char opcode,
u64 addr_f1, u64 addr_f2, u64 len,
u64 compl, u32 flags)
{
struct idxd_device *idxd = wq->idxd;
hw->flags = flags;
hw->opcode = opcode;
hw->src_addr = addr_f1;
hw->dst_addr = addr_f2;
hw->xfer_size = len;
hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
hw->completion_addr = compl;
/*
* Descriptor completion vectors are 1-8 for MSIX. We will round
* robin through the 8 vectors.
*/
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
hw->int_handle = wq->vec_ptr;
}
static struct dma_async_tx_descriptor *
idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
dma_addr_t dma_src, size_t len, unsigned long flags)
{
struct idxd_wq *wq = to_idxd_wq(c);
u32 desc_flags;
struct idxd_device *idxd = wq->idxd;
struct idxd_desc *desc;
if (wq->state != IDXD_WQ_ENABLED)
return NULL;
if (len > idxd->max_xfer_bytes)
return NULL;
op_flag_setup(flags, &desc_flags);
desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
if (IS_ERR(desc))
return NULL;
idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
dma_src, dma_dest, len, desc->compl_dma,
desc_flags);
desc->txd.flags = flags;
return &desc->txd;
}
static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
{
struct idxd_wq *wq = to_idxd_wq(chan);
struct device *dev = &wq->idxd->pdev->dev;
idxd_wq_get(wq);
dev_dbg(dev, "%s: client_count: %d\n", __func__,
idxd_wq_refcount(wq));
return 0;
}
static void idxd_dma_free_chan_resources(struct dma_chan *chan)
{
struct idxd_wq *wq = to_idxd_wq(chan);
struct device *dev = &wq->idxd->pdev->dev;
idxd_wq_put(wq);
dev_dbg(dev, "%s: client_count: %d\n", __func__,
idxd_wq_refcount(wq));
}
static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
dma_cookie_t cookie,
struct dma_tx_state *txstate)
{
return dma_cookie_status(dma_chan, cookie, txstate);
}
/*
* issue_pending() does not need to do anything since tx_submit() does the job
* already.
*/
static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
{
}
dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct dma_chan *c = tx->chan;
struct idxd_wq *wq = to_idxd_wq(c);
dma_cookie_t cookie;
int rc;
struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
cookie = dma_cookie_assign(tx);
rc = idxd_submit_desc(wq, desc);
if (rc < 0) {
idxd_free_desc(wq, desc);
return rc;
}
return cookie;
}
static void idxd_dma_release(struct dma_device *device)
{
}
int idxd_register_dma_device(struct idxd_device *idxd)
{
struct dma_device *dma = &idxd->dma_dev;
INIT_LIST_HEAD(&dma->channels);
dma->dev = &idxd->pdev->dev;
dma->device_release = idxd_dma_release;
if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
}
dma->device_tx_status = idxd_dma_tx_status;
dma->device_issue_pending = idxd_dma_issue_pending;
dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
dma->device_free_chan_resources = idxd_dma_free_chan_resources;
return dma_async_device_register(&idxd->dma_dev);
}
void idxd_unregister_dma_device(struct idxd_device *idxd)
{
dma_async_device_unregister(&idxd->dma_dev);
}
int idxd_register_dma_channel(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct dma_device *dma = &idxd->dma_dev;
struct dma_chan *chan = &wq->dma_chan;
int rc;
memset(&wq->dma_chan, 0, sizeof(struct dma_chan));
chan->device = dma;
list_add_tail(&chan->device_node, &dma->channels);
rc = dma_async_device_channel_register(dma, chan);
if (rc < 0)
return rc;
return 0;
}
void idxd_unregister_dma_channel(struct idxd_wq *wq)
{
dma_async_device_channel_unregister(&wq->idxd->dma_dev, &wq->dma_chan);
}

316
drivers/dma/idxd/idxd.h Normal file
View File

@ -0,0 +1,316 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#ifndef _IDXD_H_
#define _IDXD_H_
#include <linux/sbitmap.h>
#include <linux/dmaengine.h>
#include <linux/percpu-rwsem.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include "registers.h"
#define IDXD_DRIVER_VERSION "1.00"
extern struct kmem_cache *idxd_desc_pool;
#define IDXD_REG_TIMEOUT 50
#define IDXD_DRAIN_TIMEOUT 5000
enum idxd_type {
IDXD_TYPE_UNKNOWN = -1,
IDXD_TYPE_DSA = 0,
IDXD_TYPE_MAX
};
#define IDXD_NAME_SIZE 128
struct idxd_device_driver {
struct device_driver drv;
};
struct idxd_irq_entry {
struct idxd_device *idxd;
int id;
struct llist_head pending_llist;
struct list_head work_list;
};
struct idxd_group {
struct device conf_dev;
struct idxd_device *idxd;
struct grpcfg grpcfg;
int id;
int num_engines;
int num_wqs;
bool use_token_limit;
u8 tokens_allowed;
u8 tokens_reserved;
int tc_a;
int tc_b;
};
#define IDXD_MAX_PRIORITY 0xf
enum idxd_wq_state {
IDXD_WQ_DISABLED = 0,
IDXD_WQ_ENABLED,
};
enum idxd_wq_flag {
WQ_FLAG_DEDICATED = 0,
};
enum idxd_wq_type {
IDXD_WQT_NONE = 0,
IDXD_WQT_KERNEL,
IDXD_WQT_USER,
};
struct idxd_cdev {
struct cdev cdev;
struct device *dev;
int minor;
struct wait_queue_head err_queue;
};
#define IDXD_ALLOCATED_BATCH_SIZE 128U
#define WQ_NAME_SIZE 1024
#define WQ_TYPE_SIZE 10
enum idxd_op_type {
IDXD_OP_BLOCK = 0,
IDXD_OP_NONBLOCK = 1,
};
enum idxd_complete_type {
IDXD_COMPLETE_NORMAL = 0,
IDXD_COMPLETE_ABORT,
};
struct idxd_wq {
void __iomem *dportal;
struct device conf_dev;
struct idxd_cdev idxd_cdev;
struct idxd_device *idxd;
int id;
enum idxd_wq_type type;
struct idxd_group *group;
int client_count;
struct mutex wq_lock; /* mutex for workqueue */
u32 size;
u32 threshold;
u32 priority;
enum idxd_wq_state state;
unsigned long flags;
union wqcfg wqcfg;
atomic_t dq_count; /* dedicated queue flow control */
u32 vec_ptr; /* interrupt steering */
struct dsa_hw_desc **hw_descs;
int num_descs;
struct dsa_completion_record *compls;
dma_addr_t compls_addr;
int compls_size;
struct idxd_desc **descs;
struct sbitmap sbmap;
struct dma_chan dma_chan;
struct percpu_rw_semaphore submit_lock;
wait_queue_head_t submit_waitq;
char name[WQ_NAME_SIZE + 1];
};
struct idxd_engine {
struct device conf_dev;
int id;
struct idxd_group *group;
struct idxd_device *idxd;
};
/* shadow registers */
struct idxd_hw {
u32 version;
union gen_cap_reg gen_cap;
union wq_cap_reg wq_cap;
union group_cap_reg group_cap;
union engine_cap_reg engine_cap;
struct opcap opcap;
};
enum idxd_device_state {
IDXD_DEV_HALTED = -1,
IDXD_DEV_DISABLED = 0,
IDXD_DEV_CONF_READY,
IDXD_DEV_ENABLED,
};
enum idxd_device_flag {
IDXD_FLAG_CONFIGURABLE = 0,
};
struct idxd_device {
enum idxd_type type;
struct device conf_dev;
struct list_head list;
struct idxd_hw hw;
enum idxd_device_state state;
unsigned long flags;
int id;
int major;
struct pci_dev *pdev;
void __iomem *reg_base;
spinlock_t dev_lock; /* spinlock for device */
struct idxd_group *groups;
struct idxd_wq *wqs;
struct idxd_engine *engines;
int num_groups;
u32 msix_perm_offset;
u32 wqcfg_offset;
u32 grpcfg_offset;
u32 perfmon_offset;
u64 max_xfer_bytes;
u32 max_batch_size;
int max_groups;
int max_engines;
int max_tokens;
int max_wqs;
int max_wq_size;
int token_limit;
int nr_tokens; /* non-reserved tokens */
union sw_err_reg sw_err;
struct msix_entry *msix_entries;
int num_wq_irqs;
struct idxd_irq_entry *irq_entries;
struct dma_device dma_dev;
};
/* IDXD software descriptor */
struct idxd_desc {
struct dsa_hw_desc *hw;
dma_addr_t desc_dma;
struct dsa_completion_record *completion;
dma_addr_t compl_dma;
struct dma_async_tx_descriptor txd;
struct llist_node llnode;
struct list_head list;
int id;
struct idxd_wq *wq;
};
#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
extern struct bus_type dsa_bus_type;
static inline bool wq_dedicated(struct idxd_wq *wq)
{
return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
}
enum idxd_portal_prot {
IDXD_PORTAL_UNLIMITED = 0,
IDXD_PORTAL_LIMITED,
};
static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
{
return prot * 0x1000;
}
static inline int idxd_get_wq_portal_full_offset(int wq_id,
enum idxd_portal_prot prot)
{
return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
}
static inline void idxd_set_type(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
idxd->type = IDXD_TYPE_DSA;
else
idxd->type = IDXD_TYPE_UNKNOWN;
}
static inline void idxd_wq_get(struct idxd_wq *wq)
{
wq->client_count++;
}
static inline void idxd_wq_put(struct idxd_wq *wq)
{
wq->client_count--;
}
static inline int idxd_wq_refcount(struct idxd_wq *wq)
{
return wq->client_count;
};
const char *idxd_get_dev_name(struct idxd_device *idxd);
int idxd_register_bus_type(void);
void idxd_unregister_bus_type(void);
int idxd_setup_sysfs(struct idxd_device *idxd);
void idxd_cleanup_sysfs(struct idxd_device *idxd);
int idxd_register_driver(void);
void idxd_unregister_driver(void);
struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
/* device interrupt control */
irqreturn_t idxd_irq_handler(int vec, void *data);
irqreturn_t idxd_misc_thread(int vec, void *data);
irqreturn_t idxd_wq_thread(int irq, void *data);
void idxd_mask_error_interrupts(struct idxd_device *idxd);
void idxd_unmask_error_interrupts(struct idxd_device *idxd);
void idxd_mask_msix_vectors(struct idxd_device *idxd);
int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
/* device control */
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
int idxd_device_reset(struct idxd_device *idxd);
int __idxd_device_reset(struct idxd_device *idxd);
void idxd_device_cleanup(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
/* work queue control */
int idxd_wq_alloc_resources(struct idxd_wq *wq);
void idxd_wq_free_resources(struct idxd_wq *wq);
int idxd_wq_enable(struct idxd_wq *wq);
int idxd_wq_disable(struct idxd_wq *wq);
int idxd_wq_map_portal(struct idxd_wq *wq);
void idxd_wq_unmap_portal(struct idxd_wq *wq);
/* submission */
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
/* dmaengine */
int idxd_register_dma_device(struct idxd_device *idxd);
void idxd_unregister_dma_device(struct idxd_device *idxd);
int idxd_register_dma_channel(struct idxd_wq *wq);
void idxd_unregister_dma_channel(struct idxd_wq *wq);
void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
void idxd_dma_complete_txd(struct idxd_desc *desc,
enum idxd_complete_type comp_type);
dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
/* cdev */
int idxd_cdev_register(void);
void idxd_cdev_remove(void);
int idxd_cdev_get_major(struct idxd_device *idxd);
int idxd_wq_add_cdev(struct idxd_wq *wq);
void idxd_wq_del_cdev(struct idxd_wq *wq);
#endif

533
drivers/dma/idxd/init.c Normal file
View File

@ -0,0 +1,533 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/aer.h>
#include <linux/fs.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/idr.h>
#include <uapi/linux/idxd.h>
#include <linux/dmaengine.h>
#include "../dmaengine.h"
#include "registers.h"
#include "idxd.h"
MODULE_VERSION(IDXD_DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Intel Corporation");
#define DRV_NAME "idxd"
static struct idr idxd_idrs[IDXD_TYPE_MAX];
static struct mutex idxd_idr_lock;
static struct pci_device_id idxd_pci_tbl[] = {
/* DSA ver 1.0 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
static char *idxd_name[] = {
"dsa",
};
const char *idxd_get_dev_name(struct idxd_device *idxd)
{
return idxd_name[idxd->type];
}
static int idxd_setup_interrupts(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
struct device *dev = &pdev->dev;
struct msix_entry *msix;
struct idxd_irq_entry *irq_entry;
int i, msixcnt;
int rc = 0;
msixcnt = pci_msix_vec_count(pdev);
if (msixcnt < 0) {
dev_err(dev, "Not MSI-X interrupt capable.\n");
goto err_no_irq;
}
idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) *
msixcnt, GFP_KERNEL);
if (!idxd->msix_entries) {
rc = -ENOMEM;
goto err_no_irq;
}
for (i = 0; i < msixcnt; i++)
idxd->msix_entries[i].entry = i;
rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt);
if (rc) {
dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt);
goto err_no_irq;
}
dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
/*
* We implement 1 completion list per MSI-X entry except for
* entry 0, which is for errors and others.
*/
idxd->irq_entries = devm_kcalloc(dev, msixcnt,
sizeof(struct idxd_irq_entry),
GFP_KERNEL);
if (!idxd->irq_entries) {
rc = -ENOMEM;
goto err_no_irq;
}
for (i = 0; i < msixcnt; i++) {
idxd->irq_entries[i].id = i;
idxd->irq_entries[i].idxd = idxd;
}
msix = &idxd->msix_entries[0];
irq_entry = &idxd->irq_entries[0];
rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler,
idxd_misc_thread, 0, "idxd-misc",
irq_entry);
if (rc < 0) {
dev_err(dev, "Failed to allocate misc interrupt.\n");
goto err_no_irq;
}
dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n",
msix->vector);
/* first MSI-X entry is not for wq interrupts */
idxd->num_wq_irqs = msixcnt - 1;
for (i = 1; i < msixcnt; i++) {
msix = &idxd->msix_entries[i];
irq_entry = &idxd->irq_entries[i];
init_llist_head(&idxd->irq_entries[i].pending_llist);
INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
rc = devm_request_threaded_irq(dev, msix->vector,
idxd_irq_handler,
idxd_wq_thread, 0,
"idxd-portal", irq_entry);
if (rc < 0) {
dev_err(dev, "Failed to allocate irq %d.\n",
msix->vector);
goto err_no_irq;
}
dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n",
i, msix->vector);
}
idxd_unmask_error_interrupts(idxd);
return 0;
err_no_irq:
/* Disable error interrupt generation */
idxd_mask_error_interrupts(idxd);
pci_disable_msix(pdev);
dev_err(dev, "No usable interrupts\n");
return rc;
}
static void idxd_wqs_free_lock(struct idxd_device *idxd)
{
int i;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
percpu_free_rwsem(&wq->submit_lock);
}
}
static int idxd_setup_internals(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int i;
idxd->groups = devm_kcalloc(dev, idxd->max_groups,
sizeof(struct idxd_group), GFP_KERNEL);
if (!idxd->groups)
return -ENOMEM;
for (i = 0; i < idxd->max_groups; i++) {
idxd->groups[i].idxd = idxd;
idxd->groups[i].id = i;
idxd->groups[i].tc_a = -1;
idxd->groups[i].tc_b = -1;
}
idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq),
GFP_KERNEL);
if (!idxd->wqs)
return -ENOMEM;
idxd->engines = devm_kcalloc(dev, idxd->max_engines,
sizeof(struct idxd_engine), GFP_KERNEL);
if (!idxd->engines)
return -ENOMEM;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
int rc;
wq->id = i;
wq->idxd = idxd;
mutex_init(&wq->wq_lock);
atomic_set(&wq->dq_count, 0);
init_waitqueue_head(&wq->submit_waitq);
wq->idxd_cdev.minor = -1;
rc = percpu_init_rwsem(&wq->submit_lock);
if (rc < 0) {
idxd_wqs_free_lock(idxd);
return rc;
}
}
for (i = 0; i < idxd->max_engines; i++) {
idxd->engines[i].idxd = idxd;
idxd->engines[i].id = i;
}
return 0;
}
static void idxd_read_table_offsets(struct idxd_device *idxd)
{
union offsets_reg offsets;
struct device *dev = &idxd->pdev->dev;
offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET
+ sizeof(u64));
idxd->grpcfg_offset = offsets.grpcfg * 0x100;
dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
idxd->wqcfg_offset = offsets.wqcfg * 0x100;
dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n",
idxd->wqcfg_offset);
idxd->msix_perm_offset = offsets.msix_perm * 0x100;
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n",
idxd->msix_perm_offset);
idxd->perfmon_offset = offsets.perfmon * 0x100;
dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
}
static void idxd_read_caps(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int i;
/* reading generic capabilities */
idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
if (idxd->hw.gen_cap.config_en)
set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
/* reading group capabilities */
idxd->hw.group_cap.bits =
ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
idxd->max_groups = idxd->hw.group_cap.num_groups;
dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
idxd->max_tokens = idxd->hw.group_cap.total_tokens;
dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
idxd->nr_tokens = idxd->max_tokens;
/* read engine capabilities */
idxd->hw.engine_cap.bits =
ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
idxd->max_engines = idxd->hw.engine_cap.num_engines;
dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
/* read workqueue capabilities */
idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
/* reading operation capabilities */
for (i = 0; i < 4; i++) {
idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
IDXD_OPCAP_OFFSET + i * sizeof(u64));
dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
}
}
static struct idxd_device *idxd_alloc(struct pci_dev *pdev,
void __iomem * const *iomap)
{
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL);
if (!idxd)
return NULL;
idxd->pdev = pdev;
idxd->reg_base = iomap[IDXD_MMIO_BAR];
spin_lock_init(&idxd->dev_lock);
return idxd;
}
static int idxd_probe(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
struct device *dev = &pdev->dev;
int rc;
dev_dbg(dev, "%s entered and resetting device\n", __func__);
rc = idxd_device_reset(idxd);
if (rc < 0)
return rc;
dev_dbg(dev, "IDXD reset complete\n");
idxd_read_caps(idxd);
idxd_read_table_offsets(idxd);
rc = idxd_setup_internals(idxd);
if (rc)
goto err_setup;
rc = idxd_setup_interrupts(idxd);
if (rc)
goto err_setup;
dev_dbg(dev, "IDXD interrupt setup complete.\n");
mutex_lock(&idxd_idr_lock);
idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL);
mutex_unlock(&idxd_idr_lock);
if (idxd->id < 0) {
rc = -ENOMEM;
goto err_idr_fail;
}
idxd->major = idxd_cdev_get_major(idxd);
dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
return 0;
err_idr_fail:
idxd_mask_error_interrupts(idxd);
idxd_mask_msix_vectors(idxd);
err_setup:
return rc;
}
static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
void __iomem * const *iomap;
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
int rc;
unsigned int mask;
rc = pcim_enable_device(pdev);
if (rc)
return rc;
dev_dbg(dev, "Mapping BARs\n");
mask = (1 << IDXD_MMIO_BAR);
rc = pcim_iomap_regions(pdev, mask, DRV_NAME);
if (rc)
return rc;
iomap = pcim_iomap_table(pdev);
if (!iomap)
return -ENOMEM;
dev_dbg(dev, "Set DMA masks\n");
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
dev_dbg(dev, "Alloc IDXD context\n");
idxd = idxd_alloc(pdev, iomap);
if (!idxd)
return -ENOMEM;
idxd_set_type(idxd);
dev_dbg(dev, "Set PCI master\n");
pci_set_master(pdev);
pci_set_drvdata(pdev, idxd);
idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
rc = idxd_probe(idxd);
if (rc) {
dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
return -ENODEV;
}
rc = idxd_setup_sysfs(idxd);
if (rc) {
dev_err(dev, "IDXD sysfs setup failed\n");
return -ENODEV;
}
idxd->state = IDXD_DEV_CONF_READY;
dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
idxd->hw.version);
return 0;
}
static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
{
struct idxd_desc *desc, *itr;
struct llist_node *head;
head = llist_del_all(&ie->pending_llist);
if (!head)
return;
llist_for_each_entry_safe(desc, itr, head, llnode) {
idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
idxd_free_desc(desc->wq, desc);
}
}
static void idxd_flush_work_list(struct idxd_irq_entry *ie)
{
struct idxd_desc *desc, *iter;
list_for_each_entry_safe(desc, iter, &ie->work_list, list) {
list_del(&desc->list);
idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
idxd_free_desc(desc->wq, desc);
}
}
static void idxd_shutdown(struct pci_dev *pdev)
{
struct idxd_device *idxd = pci_get_drvdata(pdev);
int rc, i;
struct idxd_irq_entry *irq_entry;
int msixcnt = pci_msix_vec_count(pdev);
unsigned long flags;
spin_lock_irqsave(&idxd->dev_lock, flags);
rc = idxd_device_disable(idxd);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
if (rc)
dev_err(&pdev->dev, "Disabling device failed\n");
dev_dbg(&pdev->dev, "%s called\n", __func__);
idxd_mask_msix_vectors(idxd);
idxd_mask_error_interrupts(idxd);
for (i = 0; i < msixcnt; i++) {
irq_entry = &idxd->irq_entries[i];
synchronize_irq(idxd->msix_entries[i].vector);
if (i == 0)
continue;
idxd_flush_pending_llist(irq_entry);
idxd_flush_work_list(irq_entry);
}
}
static void idxd_remove(struct pci_dev *pdev)
{
struct idxd_device *idxd = pci_get_drvdata(pdev);
dev_dbg(&pdev->dev, "%s called\n", __func__);
idxd_cleanup_sysfs(idxd);
idxd_shutdown(pdev);
idxd_wqs_free_lock(idxd);
mutex_lock(&idxd_idr_lock);
idr_remove(&idxd_idrs[idxd->type], idxd->id);
mutex_unlock(&idxd_idr_lock);
}
static struct pci_driver idxd_pci_driver = {
.name = DRV_NAME,
.id_table = idxd_pci_tbl,
.probe = idxd_pci_probe,
.remove = idxd_remove,
.shutdown = idxd_shutdown,
};
static int __init idxd_init_module(void)
{
int err, i;
/*
* If the CPU does not support write512, there's no point in
* enumerating the device. We can not utilize it.
*/
if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) {
pr_warn("idxd driver failed to load without MOVDIR64B.\n");
return -ENODEV;
}
pr_info("%s: Intel(R) Accelerator Devices Driver %s\n",
DRV_NAME, IDXD_DRIVER_VERSION);
mutex_init(&idxd_idr_lock);
for (i = 0; i < IDXD_TYPE_MAX; i++)
idr_init(&idxd_idrs[i]);
err = idxd_register_bus_type();
if (err < 0)
return err;
err = idxd_register_driver();
if (err < 0)
goto err_idxd_driver_register;
err = idxd_cdev_register();
if (err)
goto err_cdev_register;
err = pci_register_driver(&idxd_pci_driver);
if (err)
goto err_pci_register;
return 0;
err_pci_register:
idxd_cdev_remove();
err_cdev_register:
idxd_unregister_driver();
err_idxd_driver_register:
idxd_unregister_bus_type();
return err;
}
module_init(idxd_init_module);
static void __exit idxd_exit_module(void)
{
pci_unregister_driver(&idxd_pci_driver);
idxd_cdev_remove();
idxd_unregister_bus_type();
}
module_exit(idxd_exit_module);

261
drivers/dma/idxd/irq.c Normal file
View File

@ -0,0 +1,261 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/dmaengine.h>
#include <uapi/linux/idxd.h>
#include "../dmaengine.h"
#include "idxd.h"
#include "registers.h"
void idxd_device_wqs_clear_state(struct idxd_device *idxd)
{
int i;
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
wq->state = IDXD_WQ_DISABLED;
}
}
static int idxd_restart(struct idxd_device *idxd)
{
int i, rc;
lockdep_assert_held(&idxd->dev_lock);
rc = __idxd_device_reset(idxd);
if (rc < 0)
goto out;
rc = idxd_device_config(idxd);
if (rc < 0)
goto out;
rc = idxd_device_enable(idxd);
if (rc < 0)
goto out;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED) {
rc = idxd_wq_enable(wq);
if (rc < 0) {
dev_warn(&idxd->pdev->dev,
"Unable to re-enable wq %s\n",
dev_name(&wq->conf_dev));
}
}
}
return 0;
out:
idxd_device_wqs_clear_state(idxd);
idxd->state = IDXD_DEV_HALTED;
return rc;
}
irqreturn_t idxd_irq_handler(int vec, void *data)
{
struct idxd_irq_entry *irq_entry = data;
struct idxd_device *idxd = irq_entry->idxd;
idxd_mask_msix_vector(idxd, irq_entry->id);
return IRQ_WAKE_THREAD;
}
irqreturn_t idxd_misc_thread(int vec, void *data)
{
struct idxd_irq_entry *irq_entry = data;
struct idxd_device *idxd = irq_entry->idxd;
struct device *dev = &idxd->pdev->dev;
union gensts_reg gensts;
u32 cause, val = 0;
int i, rc;
bool err = false;
cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
if (cause & IDXD_INTC_ERR) {
spin_lock_bh(&idxd->dev_lock);
for (i = 0; i < 4; i++)
idxd->sw_err.bits[i] = ioread64(idxd->reg_base +
IDXD_SWERR_OFFSET + i * sizeof(u64));
iowrite64(IDXD_SWERR_ACK, idxd->reg_base + IDXD_SWERR_OFFSET);
if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {
int id = idxd->sw_err.wq_idx;
struct idxd_wq *wq = &idxd->wqs[id];
if (wq->type == IDXD_WQT_USER)
wake_up_interruptible(&wq->idxd_cdev.err_queue);
} else {
int i;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
if (wq->type == IDXD_WQT_USER)
wake_up_interruptible(&wq->idxd_cdev.err_queue);
}
}
spin_unlock_bh(&idxd->dev_lock);
val |= IDXD_INTC_ERR;
for (i = 0; i < 4; i++)
dev_warn(dev, "err[%d]: %#16.16llx\n",
i, idxd->sw_err.bits[i]);
err = true;
}
if (cause & IDXD_INTC_CMD) {
/* Driver does use command interrupts */
val |= IDXD_INTC_CMD;
}
if (cause & IDXD_INTC_OCCUPY) {
/* Driver does not utilize occupancy interrupt */
val |= IDXD_INTC_OCCUPY;
}
if (cause & IDXD_INTC_PERFMON_OVFL) {
/*
* Driver does not utilize perfmon counter overflow interrupt
* yet.
*/
val |= IDXD_INTC_PERFMON_OVFL;
}
val ^= cause;
if (val)
dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n",
val);
iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
if (!err)
return IRQ_HANDLED;
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
if (gensts.state == IDXD_DEVICE_STATE_HALT) {
spin_lock_bh(&idxd->dev_lock);
if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) {
rc = idxd_restart(idxd);
if (rc < 0)
dev_err(&idxd->pdev->dev,
"idxd restart failed, device halt.");
} else {
idxd_device_wqs_clear_state(idxd);
idxd->state = IDXD_DEV_HALTED;
dev_err(&idxd->pdev->dev,
"idxd halted, need %s.\n",
gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
"FLR" : "system reset");
}
spin_unlock_bh(&idxd->dev_lock);
}
idxd_unmask_msix_vector(idxd, irq_entry->id);
return IRQ_HANDLED;
}
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
int *processed)
{
struct idxd_desc *desc, *t;
struct llist_node *head;
int queued = 0;
head = llist_del_all(&irq_entry->pending_llist);
if (!head)
return 0;
llist_for_each_entry_safe(desc, t, head, llnode) {
if (desc->completion->status) {
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
idxd_free_desc(desc->wq, desc);
(*processed)++;
} else {
list_add_tail(&desc->list, &irq_entry->work_list);
queued++;
}
}
return queued;
}
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
int *processed)
{
struct list_head *node, *next;
int queued = 0;
if (list_empty(&irq_entry->work_list))
return 0;
list_for_each_safe(node, next, &irq_entry->work_list) {
struct idxd_desc *desc =
container_of(node, struct idxd_desc, list);
if (desc->completion->status) {
list_del(&desc->list);
/* process and callback */
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
idxd_free_desc(desc->wq, desc);
(*processed)++;
} else {
queued++;
}
}
return queued;
}
irqreturn_t idxd_wq_thread(int irq, void *data)
{
struct idxd_irq_entry *irq_entry = data;
int rc, processed = 0, retry = 0;
/*
* There are two lists we are processing. The pending_llist is where
* submmiter adds all the submitted descriptor after sending it to
* the workqueue. It's a lockless singly linked list. The work_list
* is the common linux double linked list. We are in a scenario of
* multiple producers and a single consumer. The producers are all
* the kernel submitters of descriptors, and the consumer is the
* kernel irq handler thread for the msix vector when using threaded
* irq. To work with the restrictions of llist to remain lockless,
* we are doing the following steps:
* 1. Iterate through the work_list and process any completed
* descriptor. Delete the completed entries during iteration.
* 2. llist_del_all() from the pending list.
* 3. Iterate through the llist that was deleted from the pending list
* and process the completed entries.
* 4. If the entry is still waiting on hardware, list_add_tail() to
* the work_list.
* 5. Repeat until no more descriptors.
*/
do {
rc = irq_process_work_list(irq_entry, &processed);
if (rc != 0) {
retry++;
continue;
}
rc = irq_process_pending_llist(irq_entry, &processed);
} while (rc != 0 && retry != 10);
idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id);
if (processed == 0)
return IRQ_NONE;
return IRQ_HANDLED;
}

336
drivers/dma/idxd/registers.h Normal file
View File

@ -0,0 +1,336 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#ifndef _IDXD_REGISTERS_H_
#define _IDXD_REGISTERS_H_
/* PCI Config */
#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25
#define IDXD_MMIO_BAR 0
#define IDXD_WQ_BAR 2
#define IDXD_PORTAL_SIZE 0x4000
/* MMIO Device BAR0 Registers */
#define IDXD_VER_OFFSET 0x00
#define IDXD_VER_MAJOR_MASK 0xf0
#define IDXD_VER_MINOR_MASK 0x0f
#define GET_IDXD_VER_MAJOR(x) (((x) & IDXD_VER_MAJOR_MASK) >> 4)
#define GET_IDXD_VER_MINOR(x) ((x) & IDXD_VER_MINOR_MASK)
union gen_cap_reg {
struct {
u64 block_on_fault:1;
u64 overlap_copy:1;
u64 cache_control_mem:1;
u64 cache_control_cache:1;
u64 rsvd:3;
u64 int_handle_req:1;
u64 dest_readback:1;
u64 drain_readback:1;
u64 rsvd2:6;
u64 max_xfer_shift:5;
u64 max_batch_shift:4;
u64 max_ims_mult:6;
u64 config_en:1;
u64 max_descs_per_engine:8;
u64 rsvd3:24;
};
u64 bits;
} __packed;
#define IDXD_GENCAP_OFFSET 0x10
union wq_cap_reg {
struct {
u64 total_wq_size:16;
u64 num_wqs:8;
u64 rsvd:24;
u64 shared_mode:1;
u64 dedicated_mode:1;
u64 rsvd2:1;
u64 priority:1;
u64 occupancy:1;
u64 occupancy_int:1;
u64 rsvd3:10;
};
u64 bits;
} __packed;
#define IDXD_WQCAP_OFFSET 0x20
union group_cap_reg {
struct {
u64 num_groups:8;
u64 total_tokens:8;
u64 token_en:1;
u64 token_limit:1;
u64 rsvd:46;
};
u64 bits;
} __packed;
#define IDXD_GRPCAP_OFFSET 0x30
union engine_cap_reg {
struct {
u64 num_engines:8;
u64 rsvd:56;
};
u64 bits;
} __packed;
#define IDXD_ENGCAP_OFFSET 0x38
#define IDXD_OPCAP_NOOP 0x0001
#define IDXD_OPCAP_BATCH 0x0002
#define IDXD_OPCAP_MEMMOVE 0x0008
struct opcap {
u64 bits[4];
};
#define IDXD_OPCAP_OFFSET 0x40
#define IDXD_TABLE_OFFSET 0x60
union offsets_reg {
struct {
u64 grpcfg:16;
u64 wqcfg:16;
u64 msix_perm:16;
u64 ims:16;
u64 perfmon:16;
u64 rsvd:48;
};
u64 bits[2];
} __packed;
#define IDXD_GENCFG_OFFSET 0x80
union gencfg_reg {
struct {
u32 token_limit:8;
u32 rsvd:4;
u32 user_int_en:1;
u32 rsvd2:19;
};
u32 bits;
} __packed;
#define IDXD_GENCTRL_OFFSET 0x88
union genctrl_reg {
struct {
u32 softerr_int_en:1;
u32 rsvd:31;
};
u32 bits;
} __packed;
#define IDXD_GENSTATS_OFFSET 0x90
union gensts_reg {
struct {
u32 state:2;
u32 reset_type:2;
u32 rsvd:28;
};
u32 bits;
} __packed;
enum idxd_device_status_state {
IDXD_DEVICE_STATE_DISABLED = 0,
IDXD_DEVICE_STATE_ENABLED,
IDXD_DEVICE_STATE_DRAIN,
IDXD_DEVICE_STATE_HALT,
};
enum idxd_device_reset_type {
IDXD_DEVICE_RESET_SOFTWARE = 0,
IDXD_DEVICE_RESET_FLR,
IDXD_DEVICE_RESET_WARM,
IDXD_DEVICE_RESET_COLD,
};
#define IDXD_INTCAUSE_OFFSET 0x98
#define IDXD_INTC_ERR 0x01
#define IDXD_INTC_CMD 0x02
#define IDXD_INTC_OCCUPY 0x04
#define IDXD_INTC_PERFMON_OVFL 0x08
#define IDXD_CMD_OFFSET 0xa0
union idxd_command_reg {
struct {
u32 operand:20;
u32 cmd:5;
u32 rsvd:6;
u32 int_req:1;
};
u32 bits;
} __packed;
enum idxd_cmd {
IDXD_CMD_ENABLE_DEVICE = 1,
IDXD_CMD_DISABLE_DEVICE,
IDXD_CMD_DRAIN_ALL,
IDXD_CMD_ABORT_ALL,
IDXD_CMD_RESET_DEVICE,
IDXD_CMD_ENABLE_WQ,
IDXD_CMD_DISABLE_WQ,
IDXD_CMD_DRAIN_WQ,
IDXD_CMD_ABORT_WQ,
IDXD_CMD_RESET_WQ,
IDXD_CMD_DRAIN_PASID,
IDXD_CMD_ABORT_PASID,
IDXD_CMD_REQUEST_INT_HANDLE,
};
#define IDXD_CMDSTS_OFFSET 0xa8
union cmdsts_reg {
struct {
u8 err;
u16 result;
u8 rsvd:7;
u8 active:1;
};
u32 bits;
} __packed;
#define IDXD_CMDSTS_ACTIVE 0x80000000
enum idxd_cmdsts_err {
IDXD_CMDSTS_SUCCESS = 0,
IDXD_CMDSTS_INVAL_CMD,
IDXD_CMDSTS_INVAL_WQIDX,
IDXD_CMDSTS_HW_ERR,
/* enable device errors */
IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10,
IDXD_CMDSTS_ERR_CONFIG,
IDXD_CMDSTS_ERR_BUSMASTER_EN,
IDXD_CMDSTS_ERR_PASID_INVAL,
IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE,
IDXD_CMDSTS_ERR_GRP_CONFIG,
IDXD_CMDSTS_ERR_GRP_CONFIG2,
IDXD_CMDSTS_ERR_GRP_CONFIG3,
IDXD_CMDSTS_ERR_GRP_CONFIG4,
/* enable wq errors */
IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20,
IDXD_CMDSTS_ERR_WQ_ENABLED,
IDXD_CMDSTS_ERR_WQ_SIZE,
IDXD_CMDSTS_ERR_WQ_PRIOR,
IDXD_CMDSTS_ERR_WQ_MODE,
IDXD_CMDSTS_ERR_BOF_EN,
IDXD_CMDSTS_ERR_PASID_EN,
IDXD_CMDSTS_ERR_MAX_BATCH_SIZE,
IDXD_CMDSTS_ERR_MAX_XFER_SIZE,
/* disable device errors */
IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31,
/* disable WQ, drain WQ, abort WQ, reset WQ */
IDXD_CMDSTS_ERR_DEV_NOT_EN,
/* request interrupt handle */
IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41,
IDXD_CMDSTS_ERR_NO_HANDLE,
};
#define IDXD_SWERR_OFFSET 0xc0
#define IDXD_SWERR_VALID 0x00000001
#define IDXD_SWERR_OVERFLOW 0x00000002
#define IDXD_SWERR_ACK (IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW)
union sw_err_reg {
struct {
u64 valid:1;
u64 overflow:1;
u64 desc_valid:1;
u64 wq_idx_valid:1;
u64 batch:1;
u64 fault_rw:1;
u64 priv:1;
u64 rsvd:1;
u64 error:8;
u64 wq_idx:8;
u64 rsvd2:8;
u64 operation:8;
u64 pasid:20;
u64 rsvd3:4;
u64 batch_idx:16;
u64 rsvd4:16;
u64 invalid_flags:32;
u64 fault_addr;
u64 rsvd5;
};
u64 bits[4];
} __packed;
union msix_perm {
struct {
u32 rsvd:2;
u32 ignore:1;
u32 pasid_en:1;
u32 rsvd2:8;
u32 pasid:20;
};
u32 bits;
} __packed;
union group_flags {
struct {
u32 tc_a:3;
u32 tc_b:3;
u32 rsvd:1;
u32 use_token_limit:1;
u32 tokens_reserved:8;
u32 rsvd2:4;
u32 tokens_allowed:8;
u32 rsvd3:4;
};
u32 bits;
} __packed;
struct grpcfg {
u64 wqs[4];
u64 engines;
union group_flags flags;
} __packed;
union wqcfg {
struct {
/* bytes 0-3 */
u16 wq_size;
u16 rsvd;
/* bytes 4-7 */
u16 wq_thresh;
u16 rsvd1;
/* bytes 8-11 */
u32 mode:1; /* shared or dedicated */
u32 bof:1; /* block on fault */
u32 rsvd2:2;
u32 priority:4;
u32 pasid:20;
u32 pasid_en:1;
u32 priv:1;
u32 rsvd3:2;
/* bytes 12-15 */
u32 max_xfer_shift:5;
u32 max_batch_shift:4;
u32 rsvd4:23;
/* bytes 16-19 */
u16 occupancy_inth;
u16 occupancy_table_sel:1;
u16 rsvd5:15;
/* bytes 20-23 */
u16 occupancy_limit;
u16 occupancy_int_en:1;
u16 rsvd6:15;
/* bytes 24-27 */
u16 occupancy;
u16 occupancy_int:1;
u16 rsvd7:12;
u16 mode_support:1;
u16 wq_state:2;
/* bytes 28-31 */
u32 rsvd8;
};
u32 bits[8];
} __packed;
#endif

95
drivers/dma/idxd/submit.c Normal file
View File

@ -0,0 +1,95 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <uapi/linux/idxd.h>
#include "idxd.h"
#include "registers.h"
struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
{
struct idxd_desc *desc;
int idx;
struct idxd_device *idxd = wq->idxd;
if (idxd->state != IDXD_DEV_ENABLED)
return ERR_PTR(-EIO);
if (optype == IDXD_OP_BLOCK)
percpu_down_read(&wq->submit_lock);
else if (!percpu_down_read_trylock(&wq->submit_lock))
return ERR_PTR(-EBUSY);
if (!atomic_add_unless(&wq->dq_count, 1, wq->size)) {
int rc;
if (optype == IDXD_OP_NONBLOCK) {
percpu_up_read(&wq->submit_lock);
return ERR_PTR(-EAGAIN);
}
percpu_up_read(&wq->submit_lock);
percpu_down_write(&wq->submit_lock);
rc = wait_event_interruptible(wq->submit_waitq,
atomic_add_unless(&wq->dq_count,
1, wq->size) ||
idxd->state != IDXD_DEV_ENABLED);
percpu_up_write(&wq->submit_lock);
if (rc < 0)
return ERR_PTR(-EINTR);
if (idxd->state != IDXD_DEV_ENABLED)
return ERR_PTR(-EIO);
} else {
percpu_up_read(&wq->submit_lock);
}
idx = sbitmap_get(&wq->sbmap, 0, false);
if (idx < 0) {
atomic_dec(&wq->dq_count);
return ERR_PTR(-EAGAIN);
}
desc = wq->descs[idx];
memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
memset(desc->completion, 0, sizeof(struct dsa_completion_record));
return desc;
}
void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc)
{
atomic_dec(&wq->dq_count);
sbitmap_clear_bit(&wq->sbmap, desc->id);
wake_up(&wq->submit_waitq);
}
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
{
struct idxd_device *idxd = wq->idxd;
int vec = desc->hw->int_handle;
void __iomem *portal;
if (idxd->state != IDXD_DEV_ENABLED)
return -EIO;
portal = wq->dportal + idxd_get_wq_portal_offset(IDXD_PORTAL_UNLIMITED);
/*
* The wmb() flushes writes to coherent DMA data before possibly
* triggering a DMA read. The wmb() is necessary even on UP because
* the recipient is a device.
*/
wmb();
iosubmit_cmds512(portal, desc->hw, 1);
/*
* Pending the descriptor to the lockless list for the irq_entry
* that we designated the descriptor to.
*/
if (desc->hw->flags & IDXD_OP_FLAG_RCI)
llist_add(&desc->llnode,
&idxd->irq_entries[vec].pending_llist);
return 0;
}

1528
drivers/dma/idxd/sysfs.c Normal file
View File

File diff suppressed because it is too large Load Diff

35
drivers/dma/imx-sdma.c
View File

@ -760,11 +760,7 @@ static void sdma_start_desc(struct sdma_channel *sdmac)
return;
}
sdmac->desc = desc = to_sdma_desc(&vd->tx);
/*
* Do not delete the node in desc_issued list in cyclic mode, otherwise
* the desc allocated will never be freed in vchan_dma_desc_free_list
*/
if (!(sdmac->flags & IMX_DMA_SG_LOOP))
list_del(&vd->node);
sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
@ -1071,20 +1067,27 @@ static void sdma_channel_terminate_work(struct work_struct *work)
spin_lock_irqsave(&sdmac->vc.lock, flags);
vchan_get_all_descriptors(&sdmac->vc, &head);
sdmac->desc = NULL;
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
vchan_dma_desc_free_list(&sdmac->vc, &head);
sdmac->context_loaded = false;
}
static int sdma_disable_channel_async(struct dma_chan *chan)
static int sdma_terminate_all(struct dma_chan *chan)
{
struct sdma_channel *sdmac = to_sdma_chan(chan);
unsigned long flags;
spin_lock_irqsave(&sdmac->vc.lock, flags);
sdma_disable_channel(chan);
if (sdmac->desc)
if (sdmac->desc) {
vchan_terminate_vdesc(&sdmac->desc->vd);
sdmac->desc = NULL;
schedule_work(&sdmac->terminate_worker);
}
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
return 0;
}
@ -1324,7 +1327,7 @@ static void sdma_free_chan_resources(struct dma_chan *chan)
struct sdma_channel *sdmac = to_sdma_chan(chan);
struct sdma_engine *sdma = sdmac->sdma;
sdma_disable_channel_async(chan);
sdma_terminate_all(chan);
sdma_channel_synchronize(chan);
@ -1648,7 +1651,7 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
struct dma_tx_state *txstate)
{
struct sdma_channel *sdmac = to_sdma_chan(chan);
struct sdma_desc *desc;
struct sdma_desc *desc = NULL;
u32 residue;
struct virt_dma_desc *vd;
enum dma_status ret;
@ -1659,19 +1662,23 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
return ret;
spin_lock_irqsave(&sdmac->vc.lock, flags);
vd = vchan_find_desc(&sdmac->vc, cookie);
if (vd) {
if (vd)
desc = to_sdma_desc(&vd->tx);
else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
desc = sdmac->desc;
if (desc) {
if (sdmac->flags & IMX_DMA_SG_LOOP)
residue = (desc->num_bd - desc->buf_ptail) *
desc->period_len - desc->chn_real_count;
else
residue = desc->chn_count - desc->chn_real_count;
} else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) {
residue = sdmac->desc->chn_count - sdmac->desc->chn_real_count;
} else {
residue = 0;
}
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
@ -2103,7 +2110,7 @@ static int sdma_probe(struct platform_device *pdev)
sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
sdma->dma_device.device_config = sdma_config;
sdma->dma_device.device_terminate_all = sdma_disable_channel_async;
sdma->dma_device.device_terminate_all = sdma_terminate_all;
sdma->dma_device.device_synchronize = sdma_channel_synchronize;
sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;

30
drivers/dma/ioat/init.c
View File

@ -556,10 +556,6 @@ static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
ioat_kobject_del(ioat_dma);
dma_async_device_unregister(dma);
dma_pool_destroy(ioat_dma->completion_pool);
INIT_LIST_HEAD(&dma->channels);
}
/**
@ -589,7 +585,7 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
for (i = 0; i < dma->chancnt; i++) {
ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
if (!ioat_chan)
break;
@ -624,12 +620,16 @@ static void ioat_free_chan_resources(struct dma_chan *c)
return;
ioat_stop(ioat_chan);
if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) {
ioat_reset_hw(ioat_chan);
/* Put LTR to idle */
if (ioat_dma->version >= IOAT_VER_3_4)
writeb(IOAT_CHAN_LTR_SWSEL_IDLE,
ioat_chan->reg_base + IOAT_CHAN_LTR_SWSEL_OFFSET);
ioat_chan->reg_base +
IOAT_CHAN_LTR_SWSEL_OFFSET);
}
spin_lock_bh(&ioat_chan->cleanup_lock);
spin_lock_bh(&ioat_chan->prep_lock);
@ -1322,16 +1322,28 @@ static struct pci_driver ioat_pci_driver = {
.err_handler = &ioat_err_handler,
};
static void release_ioatdma(struct dma_device *device)
{
struct ioatdma_device *d = to_ioatdma_device(device);
int i;
for (i = 0; i < IOAT_MAX_CHANS; i++)
kfree(d->idx[i]);
dma_pool_destroy(d->completion_pool);
kfree(d);
}
static struct ioatdma_device *
alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
{
struct device *dev = &pdev->dev;
struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
struct ioatdma_device *d = kzalloc(sizeof(*d), GFP_KERNEL);
if (!d)
return NULL;
d->pdev = pdev;
d->reg_base = iobase;
d->dma_dev.device_release = release_ioatdma;
return d;
}
@ -1400,6 +1412,8 @@ static void ioat_remove(struct pci_dev *pdev)
if (!device)
return;
ioat_shutdown(pdev);
dev_err(&pdev->dev, "Removing dma and dca services\n");
if (device->dca) {
unregister_dca_provider(device->dca, &pdev->dev);

3
drivers/dma/mediatek/mtk-uart-apdma.c
View File

@ -430,9 +430,10 @@ static int mtk_uart_apdma_terminate_all(struct dma_chan *chan)
spin_lock_irqsave(&c->vc.lock, flags);
vchan_get_all_descriptors(&c->vc, &head);
vchan_dma_desc_free_list(&c->vc, &head);
spin_unlock_irqrestore(&c->vc.lock, flags);
vchan_dma_desc_free_list(&c->vc, &head);
return 0;
}

2
drivers/dma/of-dma.c
View File

@ -15,6 +15,8 @@
#include <linux/of.h>
#include <linux/of_dma.h>
#include "dmaengine.h"
static LIST_HEAD(of_dma_list);
static DEFINE_MUTEX(of_dma_lock);

3
drivers/dma/owl-dma.c
View File

@ -674,10 +674,11 @@ static int owl_dma_terminate_all(struct dma_chan *chan)
}
vchan_get_all_descriptors(&vchan->vc, &head);
vchan_dma_desc_free_list(&vchan->vc, &head);
spin_unlock_irqrestore(&vchan->vc.lock, flags);
vchan_dma_desc_free_list(&vchan->vc, &head);
return 0;
}

16
drivers/dma/pl330.c
View File

@ -2961,12 +2961,7 @@ static int __maybe_unused pl330_suspend(struct device *dev)
{
struct amba_device *pcdev = to_amba_device(dev);
pm_runtime_disable(dev);
if (!pm_runtime_status_suspended(dev)) {
/* amba did not disable the clock */
amba_pclk_disable(pcdev);
}
pm_runtime_force_suspend(dev);
amba_pclk_unprepare(pcdev);
return 0;
@ -2981,15 +2976,14 @@ static int __maybe_unused pl330_resume(struct device *dev)
if (ret)
return ret;
if (!pm_runtime_status_suspended(dev))
ret = amba_pclk_enable(pcdev);
pm_runtime_enable(dev);
pm_runtime_force_resume(dev);
return ret;
}
static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume);
static const struct dev_pm_ops pl330_pm = {
SET_LATE_SYSTEM_SLEEP_PM_OPS(pl330_suspend, pl330_resume)
};
static int
pl330_probe(struct amba_device *adev, const struct amba_id *id)

639
drivers/dma/plx_dma.c Normal file
View File

@ -0,0 +1,639 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Microsemi Switchtec(tm) PCIe Management Driver
* Copyright (c) 2019, Logan Gunthorpe <logang@deltatee.com>
* Copyright (c) 2019, GigaIO Networks, Inc
*/
#include "dmaengine.h"
#include <linux/circ_buf.h>
#include <linux/dmaengine.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/pci.h>
MODULE_DESCRIPTION("PLX ExpressLane PEX PCI Switch DMA Engine");
MODULE_VERSION("0.1");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Logan Gunthorpe");
#define PLX_REG_DESC_RING_ADDR 0x214
#define PLX_REG_DESC_RING_ADDR_HI 0x218
#define PLX_REG_DESC_RING_NEXT_ADDR 0x21C
#define PLX_REG_DESC_RING_COUNT 0x220
#define PLX_REG_DESC_RING_LAST_ADDR 0x224
#define PLX_REG_DESC_RING_LAST_SIZE 0x228
#define PLX_REG_PREF_LIMIT 0x234
#define PLX_REG_CTRL 0x238
#define PLX_REG_CTRL2 0x23A
#define PLX_REG_INTR_CTRL 0x23C
#define PLX_REG_INTR_STATUS 0x23E
#define PLX_REG_PREF_LIMIT_PREF_FOUR 8
#define PLX_REG_CTRL_GRACEFUL_PAUSE BIT(0)
#define PLX_REG_CTRL_ABORT BIT(1)
#define PLX_REG_CTRL_WRITE_BACK_EN BIT(2)
#define PLX_REG_CTRL_START BIT(3)
#define PLX_REG_CTRL_RING_STOP_MODE BIT(4)
#define PLX_REG_CTRL_DESC_MODE_BLOCK (0 << 5)
#define PLX_REG_CTRL_DESC_MODE_ON_CHIP (1 << 5)
#define PLX_REG_CTRL_DESC_MODE_OFF_CHIP (2 << 5)
#define PLX_REG_CTRL_DESC_INVALID BIT(8)
#define PLX_REG_CTRL_GRACEFUL_PAUSE_DONE BIT(9)
#define PLX_REG_CTRL_ABORT_DONE BIT(10)
#define PLX_REG_CTRL_IMM_PAUSE_DONE BIT(12)
#define PLX_REG_CTRL_IN_PROGRESS BIT(30)
#define PLX_REG_CTRL_RESET_VAL (PLX_REG_CTRL_DESC_INVALID | \
PLX_REG_CTRL_GRACEFUL_PAUSE_DONE | \
PLX_REG_CTRL_ABORT_DONE | \
PLX_REG_CTRL_IMM_PAUSE_DONE)
#define PLX_REG_CTRL_START_VAL (PLX_REG_CTRL_WRITE_BACK_EN | \
PLX_REG_CTRL_DESC_MODE_OFF_CHIP | \
PLX_REG_CTRL_START | \
PLX_REG_CTRL_RESET_VAL)
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_64B 0
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_128B 1
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_256B 2
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_512B 3
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_1KB 4
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_2KB 5
#define PLX_REG_CTRL2_MAX_TXFR_SIZE_4B 7
#define PLX_REG_INTR_CRTL_ERROR_EN BIT(0)
#define PLX_REG_INTR_CRTL_INV_DESC_EN BIT(1)
#define PLX_REG_INTR_CRTL_ABORT_DONE_EN BIT(3)
#define PLX_REG_INTR_CRTL_PAUSE_DONE_EN BIT(4)
#define PLX_REG_INTR_CRTL_IMM_PAUSE_DONE_EN BIT(5)
#define PLX_REG_INTR_STATUS_ERROR BIT(0)
#define PLX_REG_INTR_STATUS_INV_DESC BIT(1)
#define PLX_REG_INTR_STATUS_DESC_DONE BIT(2)
#define PLX_REG_INTR_CRTL_ABORT_DONE BIT(3)
struct plx_dma_hw_std_desc {
__le32 flags_and_size;
__le16 dst_addr_hi;
__le16 src_addr_hi;
__le32 dst_addr_lo;
__le32 src_addr_lo;
};
#define PLX_DESC_SIZE_MASK 0x7ffffff
#define PLX_DESC_FLAG_VALID BIT(31)
#define PLX_DESC_FLAG_INT_WHEN_DONE BIT(30)
#define PLX_DESC_WB_SUCCESS BIT(30)
#define PLX_DESC_WB_RD_FAIL BIT(29)
#define PLX_DESC_WB_WR_FAIL BIT(28)
#define PLX_DMA_RING_COUNT 2048
struct plx_dma_desc {
struct dma_async_tx_descriptor txd;
struct plx_dma_hw_std_desc *hw;
u32 orig_size;
};
struct plx_dma_dev {
struct dma_device dma_dev;
struct dma_chan dma_chan;
struct pci_dev __rcu *pdev;
void __iomem *bar;
struct tasklet_struct desc_task;
spinlock_t ring_lock;
bool ring_active;
int head;
int tail;
struct plx_dma_hw_std_desc *hw_ring;
dma_addr_t hw_ring_dma;
struct plx_dma_desc **desc_ring;
};
static struct plx_dma_dev *chan_to_plx_dma_dev(struct dma_chan *c)
{
return container_of(c, struct plx_dma_dev, dma_chan);
}
static struct plx_dma_desc *to_plx_desc(struct dma_async_tx_descriptor *txd)
{
return container_of(txd, struct plx_dma_desc, txd);
}
static struct plx_dma_desc *plx_dma_get_desc(struct plx_dma_dev *plxdev, int i)
{
return plxdev->desc_ring[i & (PLX_DMA_RING_COUNT - 1)];
}
static void plx_dma_process_desc(struct plx_dma_dev *plxdev)
{
struct dmaengine_result res;
struct plx_dma_desc *desc;
u32 flags;
spin_lock_bh(&plxdev->ring_lock);
while (plxdev->tail != plxdev->head) {
desc = plx_dma_get_desc(plxdev, plxdev->tail);
flags = le32_to_cpu(READ_ONCE(desc->hw->flags_and_size));
if (flags & PLX_DESC_FLAG_VALID)
break;
res.residue = desc->orig_size - (flags & PLX_DESC_SIZE_MASK);
if (flags & PLX_DESC_WB_SUCCESS)
res.result = DMA_TRANS_NOERROR;
else if (flags & PLX_DESC_WB_WR_FAIL)
res.result = DMA_TRANS_WRITE_FAILED;
else
res.result = DMA_TRANS_READ_FAILED;
dma_cookie_complete(&desc->txd);
dma_descriptor_unmap(&desc->txd);
dmaengine_desc_get_callback_invoke(&desc->txd, &res);
desc->txd.callback = NULL;
desc->txd.callback_result = NULL;
plxdev->tail++;
}
spin_unlock_bh(&plxdev->ring_lock);
}
static void plx_dma_abort_desc(struct plx_dma_dev *plxdev)
{
struct dmaengine_result res;
struct plx_dma_desc *desc;
plx_dma_process_desc(plxdev);
spin_lock_bh(&plxdev->ring_lock);
while (plxdev->tail != plxdev->head) {
desc = plx_dma_get_desc(plxdev, plxdev->tail);
res.residue = desc->orig_size;
res.result = DMA_TRANS_ABORTED;
dma_cookie_complete(&desc->txd);
dma_descriptor_unmap(&desc->txd);
dmaengine_desc_get_callback_invoke(&desc->txd, &res);
desc->txd.callback = NULL;
desc->txd.callback_result = NULL;
plxdev->tail++;
}
spin_unlock_bh(&plxdev->ring_lock);
}
static void __plx_dma_stop(struct plx_dma_dev *plxdev)
{
unsigned long timeout = jiffies + msecs_to_jiffies(1000);
u32 val;
val = readl(plxdev->bar + PLX_REG_CTRL);
if (!(val & ~PLX_REG_CTRL_GRACEFUL_PAUSE))
return;
writel(PLX_REG_CTRL_RESET_VAL | PLX_REG_CTRL_GRACEFUL_PAUSE,
plxdev->bar + PLX_REG_CTRL);
while (!time_after(jiffies, timeout)) {
val = readl(plxdev->bar + PLX_REG_CTRL);
if (val & PLX_REG_CTRL_GRACEFUL_PAUSE_DONE)
break;
cpu_relax();
}
if (!(val & PLX_REG_CTRL_GRACEFUL_PAUSE_DONE))
dev_err(plxdev->dma_dev.dev,
"Timeout waiting for graceful pause!\n");
writel(PLX_REG_CTRL_RESET_VAL | PLX_REG_CTRL_GRACEFUL_PAUSE,
plxdev->bar + PLX_REG_CTRL);
writel(0, plxdev->bar + PLX_REG_DESC_RING_COUNT);
writel(0, plxdev->bar + PLX_REG_DESC_RING_ADDR);
writel(0, plxdev->bar + PLX_REG_DESC_RING_ADDR_HI);
writel(0, plxdev->bar + PLX_REG_DESC_RING_NEXT_ADDR);
}
static void plx_dma_stop(struct plx_dma_dev *plxdev)
{
rcu_read_lock();
if (!rcu_dereference(plxdev->pdev)) {
rcu_read_unlock();
return;
}
__plx_dma_stop(plxdev);
rcu_read_unlock();
}
static void plx_dma_desc_task(unsigned long data)
{
struct plx_dma_dev *plxdev = (void *)data;
plx_dma_process_desc(plxdev);
}
static struct dma_async_tx_descriptor *plx_dma_prep_memcpy(struct dma_chan *c,
dma_addr_t dma_dst, dma_addr_t dma_src, size_t len,
unsigned long flags)
__acquires(plxdev->ring_lock)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(c);
struct plx_dma_desc *plxdesc;
spin_lock_bh(&plxdev->ring_lock);
if (!plxdev->ring_active)
goto err_unlock;
if (!CIRC_SPACE(plxdev->head, plxdev->tail, PLX_DMA_RING_COUNT))
goto err_unlock;
if (len > PLX_DESC_SIZE_MASK)
goto err_unlock;
plxdesc = plx_dma_get_desc(plxdev, plxdev->head);
plxdev->head++;
plxdesc->hw->dst_addr_lo = cpu_to_le32(lower_32_bits(dma_dst));
plxdesc->hw->dst_addr_hi = cpu_to_le16(upper_32_bits(dma_dst));
plxdesc->hw->src_addr_lo = cpu_to_le32(lower_32_bits(dma_src));
plxdesc->hw->src_addr_hi = cpu_to_le16(upper_32_bits(dma_src));
plxdesc->orig_size = len;
if (flags & DMA_PREP_INTERRUPT)
len |= PLX_DESC_FLAG_INT_WHEN_DONE;
plxdesc->hw->flags_and_size = cpu_to_le32(len);
plxdesc->txd.flags = flags;
/* return with the lock held, it will be released in tx_submit */
return &plxdesc->txd;
err_unlock:
/*
* Keep sparse happy by restoring an even lock count on
* this lock.
*/
__acquire(plxdev->ring_lock);
spin_unlock_bh(&plxdev->ring_lock);
return NULL;
}
static dma_cookie_t plx_dma_tx_submit(struct dma_async_tx_descriptor *desc)
__releases(plxdev->ring_lock)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(desc->chan);
struct plx_dma_desc *plxdesc = to_plx_desc(desc);
dma_cookie_t cookie;
cookie = dma_cookie_assign(desc);
/*
* Ensure the descriptor updates are visible to the dma device
* before setting the valid bit.
*/
wmb();
plxdesc->hw->flags_and_size |= cpu_to_le32(PLX_DESC_FLAG_VALID);
spin_unlock_bh(&plxdev->ring_lock);
return cookie;
}
static enum dma_status plx_dma_tx_status(struct dma_chan *chan,
dma_cookie_t cookie, struct dma_tx_state *txstate)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
enum dma_status ret;
ret = dma_cookie_status(chan, cookie, txstate);
if (ret == DMA_COMPLETE)
return ret;
plx_dma_process_desc(plxdev);
return dma_cookie_status(chan, cookie, txstate);
}
static void plx_dma_issue_pending(struct dma_chan *chan)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
rcu_read_lock();
if (!rcu_dereference(plxdev->pdev)) {
rcu_read_unlock();
return;
}
/*
* Ensure the valid bits are visible before starting the
* DMA engine.
*/
wmb();
writew(PLX_REG_CTRL_START_VAL, plxdev->bar + PLX_REG_CTRL);
rcu_read_unlock();
}
static irqreturn_t plx_dma_isr(int irq, void *devid)
{
struct plx_dma_dev *plxdev = devid;
u32 status;
status = readw(plxdev->bar + PLX_REG_INTR_STATUS);
if (!status)
return IRQ_NONE;
if (status & PLX_REG_INTR_STATUS_DESC_DONE && plxdev->ring_active)
tasklet_schedule(&plxdev->desc_task);
writew(status, plxdev->bar + PLX_REG_INTR_STATUS);
return IRQ_HANDLED;
}
static int plx_dma_alloc_desc(struct plx_dma_dev *plxdev)
{
struct plx_dma_desc *desc;
int i;
plxdev->desc_ring = kcalloc(PLX_DMA_RING_COUNT,
sizeof(*plxdev->desc_ring), GFP_KERNEL);
if (!plxdev->desc_ring)
return -ENOMEM;
for (i = 0; i < PLX_DMA_RING_COUNT; i++) {
desc = kzalloc(sizeof(*desc), GFP_KERNEL);
if (!desc)
goto free_and_exit;
dma_async_tx_descriptor_init(&desc->txd, &plxdev->dma_chan);
desc->txd.tx_submit = plx_dma_tx_submit;
desc->hw = &plxdev->hw_ring[i];
plxdev->desc_ring[i] = desc;
}
return 0;
free_and_exit:
for (i = 0; i < PLX_DMA_RING_COUNT; i++)
kfree(plxdev->desc_ring[i]);
kfree(plxdev->desc_ring);
return -ENOMEM;
}
static int plx_dma_alloc_chan_resources(struct dma_chan *chan)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
size_t ring_sz = PLX_DMA_RING_COUNT * sizeof(*plxdev->hw_ring);
int rc;
plxdev->head = plxdev->tail = 0;
plxdev->hw_ring = dma_alloc_coherent(plxdev->dma_dev.dev, ring_sz,
&plxdev->hw_ring_dma, GFP_KERNEL);
if (!plxdev->hw_ring)
return -ENOMEM;
rc = plx_dma_alloc_desc(plxdev);
if (rc)
goto out_free_hw_ring;
rcu_read_lock();
if (!rcu_dereference(plxdev->pdev)) {
rcu_read_unlock();
rc = -ENODEV;
goto out_free_hw_ring;
}
writel(PLX_REG_CTRL_RESET_VAL, plxdev->bar + PLX_REG_CTRL);
writel(lower_32_bits(plxdev->hw_ring_dma),
plxdev->bar + PLX_REG_DESC_RING_ADDR);
writel(upper_32_bits(plxdev->hw_ring_dma),
plxdev->bar + PLX_REG_DESC_RING_ADDR_HI);
writel(lower_32_bits(plxdev->hw_ring_dma),
plxdev->bar + PLX_REG_DESC_RING_NEXT_ADDR);
writel(PLX_DMA_RING_COUNT, plxdev->bar + PLX_REG_DESC_RING_COUNT);
writel(PLX_REG_PREF_LIMIT_PREF_FOUR, plxdev->bar + PLX_REG_PREF_LIMIT);
plxdev->ring_active = true;
rcu_read_unlock();
return PLX_DMA_RING_COUNT;
out_free_hw_ring:
dma_free_coherent(plxdev->dma_dev.dev, ring_sz, plxdev->hw_ring,
plxdev->hw_ring_dma);
return rc;
}
static void plx_dma_free_chan_resources(struct dma_chan *chan)
{
struct plx_dma_dev *plxdev = chan_to_plx_dma_dev(chan);
size_t ring_sz = PLX_DMA_RING_COUNT * sizeof(*plxdev->hw_ring);
struct pci_dev *pdev;
int irq = -1;
int i;
spin_lock_bh(&plxdev->ring_lock);
plxdev->ring_active = false;
spin_unlock_bh(&plxdev->ring_lock);
plx_dma_stop(plxdev);
rcu_read_lock();
pdev = rcu_dereference(plxdev->pdev);
if (pdev)
irq = pci_irq_vector(pdev, 0);
rcu_read_unlock();
if (irq > 0)
synchronize_irq(irq);
tasklet_kill(&plxdev->desc_task);
plx_dma_abort_desc(plxdev);
for (i = 0; i < PLX_DMA_RING_COUNT; i++)
kfree(plxdev->desc_ring[i]);
kfree(plxdev->desc_ring);
dma_free_coherent(plxdev->dma_dev.dev, ring_sz, plxdev->hw_ring,
plxdev->hw_ring_dma);
}
static void plx_dma_release(struct dma_device *dma_dev)
{
struct plx_dma_dev *plxdev =
container_of(dma_dev, struct plx_dma_dev, dma_dev);
put_device(dma_dev->dev);
kfree(plxdev);
}
static int plx_dma_create(struct pci_dev *pdev)
{
struct plx_dma_dev *plxdev;
struct dma_device *dma;
struct dma_chan *chan;
int rc;
plxdev = kzalloc(sizeof(*plxdev), GFP_KERNEL);
if (!plxdev)
return -ENOMEM;
rc = request_irq(pci_irq_vector(pdev, 0), plx_dma_isr, 0,
KBUILD_MODNAME, plxdev);
if (rc) {
kfree(plxdev);
return rc;
}
spin_lock_init(&plxdev->ring_lock);
tasklet_init(&plxdev->desc_task, plx_dma_desc_task,
(unsigned long)plxdev);
RCU_INIT_POINTER(plxdev->pdev, pdev);
plxdev->bar = pcim_iomap_table(pdev)[0];
dma = &plxdev->dma_dev;
dma->chancnt = 1;
INIT_LIST_HEAD(&dma->channels);
dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->copy_align = DMAENGINE_ALIGN_1_BYTE;
dma->dev = get_device(&pdev->dev);
dma->device_alloc_chan_resources = plx_dma_alloc_chan_resources;
dma->device_free_chan_resources = plx_dma_free_chan_resources;
dma->device_prep_dma_memcpy = plx_dma_prep_memcpy;
dma->device_issue_pending = plx_dma_issue_pending;
dma->device_tx_status = plx_dma_tx_status;
dma->device_release = plx_dma_release;
chan = &plxdev->dma_chan;
chan->device = dma;
dma_cookie_init(chan);
list_add_tail(&chan->device_node, &dma->channels);
rc = dma_async_device_register(dma);
if (rc) {
pci_err(pdev, "Failed to register dma device: %d\n", rc);
free_irq(pci_irq_vector(pdev, 0), plxdev);
kfree(plxdev);
return rc;
}
pci_set_drvdata(pdev, plxdev);
return 0;
}
static int plx_dma_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
{
int rc;
rc = pcim_enable_device(pdev);
if (rc)
return rc;
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
if (rc)
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
if (rc)
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
rc = pcim_iomap_regions(pdev, 1, KBUILD_MODNAME);
if (rc)
return rc;
rc = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
if (rc <= 0)
return rc;
pci_set_master(pdev);
rc = plx_dma_create(pdev);
if (rc)
goto err_free_irq_vectors;
pci_info(pdev, "PLX DMA Channel Registered\n");
return 0;
err_free_irq_vectors:
pci_free_irq_vectors(pdev);
return rc;
}
static void plx_dma_remove(struct pci_dev *pdev)
{
struct plx_dma_dev *plxdev = pci_get_drvdata(pdev);
free_irq(pci_irq_vector(pdev, 0), plxdev);
rcu_assign_pointer(plxdev->pdev, NULL);
synchronize_rcu();
spin_lock_bh(&plxdev->ring_lock);
plxdev->ring_active = false;
spin_unlock_bh(&plxdev->ring_lock);
__plx_dma_stop(plxdev);
plx_dma_abort_desc(plxdev);
plxdev->bar = NULL;
dma_async_device_unregister(&plxdev->dma_dev);
pci_free_irq_vectors(pdev);
}
static const struct pci_device_id plx_dma_pci_tbl[] = {
{
.vendor = PCI_VENDOR_ID_PLX,
.device = 0x87D0,
.subvendor = PCI_ANY_ID,
.subdevice = PCI_ANY_ID,
.class = PCI_CLASS_SYSTEM_OTHER << 8,
.class_mask = 0xFFFFFFFF,
},
{0}
};
MODULE_DEVICE_TABLE(pci, plx_dma_pci_tbl);
static struct pci_driver plx_dma_pci_driver = {
.name = KBUILD_MODNAME,
.id_table = plx_dma_pci_tbl,
.probe = plx_dma_probe,
.remove = plx_dma_remove,
};
module_pci_driver(plx_dma_pci_driver);

24
drivers/dma/s3c24xx-dma.c
View File

@ -519,15 +519,6 @@ static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
s3c24xx_dma_start_next_sg(s3cchan, txd);
}
static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
struct s3c24xx_dma_chan *s3cchan)
{
LIST_HEAD(head);
vchan_get_all_descriptors(&s3cchan->vc, &head);
vchan_dma_desc_free_list(&s3cchan->vc, &head);
}
/*
* Try to allocate a physical channel. When successful, assign it to
* this virtual channel, and initiate the next descriptor. The
@ -709,8 +700,9 @@ static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
{
struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
LIST_HEAD(head);
unsigned long flags;
int ret = 0;
int ret;
spin_lock_irqsave(&s3cchan->vc.lock, flags);
@ -734,7 +726,15 @@ static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
}
/* Dequeue jobs not yet fired as well */
s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
vchan_get_all_descriptors(&s3cchan->vc, &head);
spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
vchan_dma_desc_free_list(&s3cchan->vc, &head);
return 0;
unlock:
spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
@ -1198,7 +1198,7 @@ static int s3c24xx_dma_probe(struct platform_device *pdev)
/* Basic sanity check */
if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
dev_err(&pdev->dev, "too many dma channels %d, max %d\n",
pdata->num_phy_channels, MAX_DMA_CHANNELS);
return -EINVAL;
}

4
drivers/dma/sf-pdma/sf-pdma.c
View File

@ -155,9 +155,9 @@ static void sf_pdma_free_chan_resources(struct dma_chan *dchan)
kfree(chan->desc);
chan->desc = NULL;
vchan_get_all_descriptors(&chan->vchan, &head);
vchan_dma_desc_free_list(&chan->vchan, &head);
sf_pdma_disclaim_chan(chan);
spin_unlock_irqrestore(&chan->vchan.lock, flags);
vchan_dma_desc_free_list(&chan->vchan, &head);
}
static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan,
@ -220,8 +220,8 @@ static int sf_pdma_terminate_all(struct dma_chan *dchan)
chan->desc = NULL;
chan->xfer_err = false;
vchan_get_all_descriptors(&chan->vchan, &head);
vchan_dma_desc_free_list(&chan->vchan, &head);
spin_unlock_irqrestore(&chan->vchan.lock, flags);
vchan_dma_desc_free_list(&chan->vchan, &head);
return 0;
}

44
drivers/dma/sun4i-dma.c
View File

@ -669,43 +669,41 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
dma_addr_t src, dest;
u32 endpoints;
int nr_periods, offset, plength, i;
u8 ram_type, io_mode, linear_mode;
if (!is_slave_direction(dir)) {
dev_err(chan2dev(chan), "Invalid DMA direction\n");
return NULL;
}
if (vchan->is_dedicated) {
/*
* As we are using this just for audio data, we need to use
* normal DMA. There is nothing stopping us from supporting
* dedicated DMA here as well, so if a client comes up and
* requires it, it will be simple to implement it.
*/
dev_err(chan2dev(chan),
"Cyclic transfers are only supported on Normal DMA\n");
return NULL;
}
contract = generate_dma_contract();
if (!contract)
return NULL;
contract->is_cyclic = 1;
/* Figure out the endpoints and the address we need */
if (vchan->is_dedicated) {
io_mode = SUN4I_DDMA_ADDR_MODE_IO;
linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
} else {
io_mode = SUN4I_NDMA_ADDR_MODE_IO;
linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
}
if (dir == DMA_MEM_TO_DEV) {
src = buf;
dest = sconfig->dst_addr;
endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
SUN4I_DMA_CFG_DST_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO);
endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type);
} else {
src = sconfig->src_addr;
dest = buf;
endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
SUN4I_DMA_CFG_SRC_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO) |
SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
}
/*
@ -747,8 +745,13 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
dest = buf + offset;
/* Make the promise */
if (vchan->is_dedicated)
promise = generate_ddma_promise(chan, src, dest,
plength, sconfig);
else
promise = generate_ndma_promise(chan, src, dest,
plength, sconfig, dir);
if (!promise) {
/* TODO: should we free everything? */
return NULL;
@ -885,12 +888,13 @@ static int sun4i_dma_terminate_all(struct dma_chan *chan)
}
spin_lock_irqsave(&vchan->vc.lock, flags);
vchan_dma_desc_free_list(&vchan->vc, &head);
/* Clear these so the vchan is usable again */
vchan->processing = NULL;
vchan->pchan = NULL;
spin_unlock_irqrestore(&vchan->vc.lock, flags);
vchan_dma_desc_free_list(&vchan->vc, &head);
return 0;
}

24
drivers/dma/ti/Kconfig
View File

@ -34,5 +34,29 @@ config DMA_OMAP
Enable support for the TI sDMA (System DMA or DMA4) controller. This
DMA engine is found on OMAP and DRA7xx parts.
config TI_K3_UDMA
bool "Texas Instruments UDMA support"
depends on ARCH_K3 || COMPILE_TEST
depends on TI_SCI_PROTOCOL
depends on TI_SCI_INTA_IRQCHIP
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
select TI_K3_RINGACC
select TI_K3_PSIL
help
Enable support for the TI UDMA (Unified DMA) controller. This
DMA engine is used in AM65x and j721e.
config TI_K3_UDMA_GLUE_LAYER
bool "Texas Instruments UDMA Glue layer for non DMAengine users"
depends on ARCH_K3 || COMPILE_TEST
depends on TI_K3_UDMA
help
Say y here to support the K3 NAVSS DMA glue interface
If unsure, say N.
config TI_K3_PSIL
bool
config TI_DMA_CROSSBAR
bool

3
drivers/dma/ti/Makefile
View File

@ -2,4 +2,7 @@
obj-$(CONFIG_TI_CPPI41) += cppi41.o
obj-$(CONFIG_TI_EDMA) += edma.o
obj-$(CONFIG_DMA_OMAP) += omap-dma.o
obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o
obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o
obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o k3-psil-am654.o k3-psil-j721e.o
obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o

39
drivers/dma/ti/edma.c
View File

@ -2289,13 +2289,6 @@ static int edma_probe(struct platform_device *pdev)
if (!info)
return -ENODEV;
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
if (ret < 0) {
dev_err(dev, "pm_runtime_get_sync() failed\n");
return ret;
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
if (ret)
return ret;
@ -2326,27 +2319,33 @@ static int edma_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, ecc);
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
if (ret < 0) {
dev_err(dev, "pm_runtime_get_sync() failed\n");
pm_runtime_disable(dev);
return ret;
}
/* Get eDMA3 configuration from IP */
ret = edma_setup_from_hw(dev, info, ecc);
if (ret)
return ret;
goto err_disable_pm;
/* Allocate memory based on the information we got from the IP */
ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
sizeof(*ecc->slave_chans), GFP_KERNEL);
if (!ecc->slave_chans)
return -ENOMEM;
ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
sizeof(unsigned long), GFP_KERNEL);
if (!ecc->slot_inuse)
return -ENOMEM;
ecc->channels_mask = devm_kcalloc(dev,
BITS_TO_LONGS(ecc->num_channels),
sizeof(unsigned long), GFP_KERNEL);
if (!ecc->channels_mask)
return -ENOMEM;
if (!ecc->slave_chans || !ecc->slot_inuse || !ecc->channels_mask) {
ret = -ENOMEM;
goto err_disable_pm;
}
/* Mark all channels available initially */
bitmap_fill(ecc->channels_mask, ecc->num_channels);
@ -2388,7 +2387,7 @@ static int edma_probe(struct platform_device *pdev)
ecc);
if (ret) {
dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
return ret;
goto err_disable_pm;
}
ecc->ccint = irq;
}
@ -2404,7 +2403,7 @@ static int edma_probe(struct platform_device *pdev)
ecc);
if (ret) {
dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
return ret;
goto err_disable_pm;
}
ecc->ccerrint = irq;
}
@ -2412,7 +2411,8 @@ static int edma_probe(struct platform_device *pdev)
ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
if (ecc->dummy_slot < 0) {
dev_err(dev, "Can't allocate PaRAM dummy slot\n");
return ecc->dummy_slot;
ret = ecc->dummy_slot;
goto err_disable_pm;
}
queue_priority_mapping = info->queue_priority_mapping;
@ -2512,6 +2512,9 @@ static int edma_probe(struct platform_device *pdev)
err_reg1:
edma_free_slot(ecc, ecc->dummy_slot);
err_disable_pm:
pm_runtime_put_sync(dev);
pm_runtime_disable(dev);
return ret;
}
@ -2542,6 +2545,8 @@ static int edma_remove(struct platform_device *pdev)
if (ecc->dma_memcpy)
dma_async_device_unregister(ecc->dma_memcpy);
edma_free_slot(ecc, ecc->dummy_slot);
pm_runtime_put_sync(dev);
pm_runtime_disable(dev);
return 0;
}

175
drivers/dma/ti/k3-psil-am654.c Normal file
View File

@ -0,0 +1,175 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
*/
#include <linux/kernel.h>
#include "k3-psil-priv.h"
#define PSIL_PDMA_XY_TR(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
}, \
}
#define PSIL_PDMA_XY_PKT(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
.pkt_mode = 1, \
}, \
}
#define PSIL_ETHERNET(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 16, \
}, \
}
#define PSIL_SA2UL(x, tx) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 64, \
.notdpkt = tx, \
}, \
}
/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
static struct psil_ep am654_src_ep_map[] = {
/* SA2UL */
PSIL_SA2UL(0x4000, 0),
PSIL_SA2UL(0x4001, 0),
PSIL_SA2UL(0x4002, 0),
PSIL_SA2UL(0x4003, 0),
/* PRU_ICSSG0 */
PSIL_ETHERNET(0x4100),
PSIL_ETHERNET(0x4101),
PSIL_ETHERNET(0x4102),
PSIL_ETHERNET(0x4103),
/* PRU_ICSSG1 */
PSIL_ETHERNET(0x4200),
PSIL_ETHERNET(0x4201),
PSIL_ETHERNET(0x4202),
PSIL_ETHERNET(0x4203),
/* PRU_ICSSG2 */
PSIL_ETHERNET(0x4300),
PSIL_ETHERNET(0x4301),
PSIL_ETHERNET(0x4302),
PSIL_ETHERNET(0x4303),
/* PDMA0 - McASPs */
PSIL_PDMA_XY_TR(0x4400),
PSIL_PDMA_XY_TR(0x4401),
PSIL_PDMA_XY_TR(0x4402),
/* PDMA1 - SPI0-4 */
PSIL_PDMA_XY_PKT(0x4500),
PSIL_PDMA_XY_PKT(0x4501),
PSIL_PDMA_XY_PKT(0x4502),
PSIL_PDMA_XY_PKT(0x4503),
PSIL_PDMA_XY_PKT(0x4504),
PSIL_PDMA_XY_PKT(0x4505),
PSIL_PDMA_XY_PKT(0x4506),
PSIL_PDMA_XY_PKT(0x4507),
PSIL_PDMA_XY_PKT(0x4508),
PSIL_PDMA_XY_PKT(0x4509),
PSIL_PDMA_XY_PKT(0x450a),
PSIL_PDMA_XY_PKT(0x450b),
PSIL_PDMA_XY_PKT(0x450c),
PSIL_PDMA_XY_PKT(0x450d),
PSIL_PDMA_XY_PKT(0x450e),
PSIL_PDMA_XY_PKT(0x450f),
PSIL_PDMA_XY_PKT(0x4510),
PSIL_PDMA_XY_PKT(0x4511),
PSIL_PDMA_XY_PKT(0x4512),
PSIL_PDMA_XY_PKT(0x4513),
/* PDMA1 - USART0-2 */
PSIL_PDMA_XY_PKT(0x4514),
PSIL_PDMA_XY_PKT(0x4515),
PSIL_PDMA_XY_PKT(0x4516),
/* CPSW0 */
PSIL_ETHERNET(0x7000),
/* MCU_PDMA0 - ADCs */
PSIL_PDMA_XY_TR(0x7100),
PSIL_PDMA_XY_TR(0x7101),
PSIL_PDMA_XY_TR(0x7102),
PSIL_PDMA_XY_TR(0x7103),
/* MCU_PDMA1 - MCU_SPI0-2 */
PSIL_PDMA_XY_PKT(0x7200),
PSIL_PDMA_XY_PKT(0x7201),
PSIL_PDMA_XY_PKT(0x7202),
PSIL_PDMA_XY_PKT(0x7203),
PSIL_PDMA_XY_PKT(0x7204),
PSIL_PDMA_XY_PKT(0x7205),
PSIL_PDMA_XY_PKT(0x7206),
PSIL_PDMA_XY_PKT(0x7207),
PSIL_PDMA_XY_PKT(0x7208),
PSIL_PDMA_XY_PKT(0x7209),
PSIL_PDMA_XY_PKT(0x720a),
PSIL_PDMA_XY_PKT(0x720b),
/* MCU_PDMA1 - MCU_USART0 */
PSIL_PDMA_XY_PKT(0x7212),
};
/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */
static struct psil_ep am654_dst_ep_map[] = {
/* SA2UL */
PSIL_SA2UL(0xc000, 1),
PSIL_SA2UL(0xc001, 1),
/* PRU_ICSSG0 */
PSIL_ETHERNET(0xc100),
PSIL_ETHERNET(0xc101),
PSIL_ETHERNET(0xc102),
PSIL_ETHERNET(0xc103),
PSIL_ETHERNET(0xc104),
PSIL_ETHERNET(0xc105),
PSIL_ETHERNET(0xc106),
PSIL_ETHERNET(0xc107),
/* PRU_ICSSG1 */
PSIL_ETHERNET(0xc200),
PSIL_ETHERNET(0xc201),
PSIL_ETHERNET(0xc202),
PSIL_ETHERNET(0xc203),
PSIL_ETHERNET(0xc204),
PSIL_ETHERNET(0xc205),
PSIL_ETHERNET(0xc206),
PSIL_ETHERNET(0xc207),
/* PRU_ICSSG2 */
PSIL_ETHERNET(0xc300),
PSIL_ETHERNET(0xc301),
PSIL_ETHERNET(0xc302),
PSIL_ETHERNET(0xc303),
PSIL_ETHERNET(0xc304),
PSIL_ETHERNET(0xc305),
PSIL_ETHERNET(0xc306),
PSIL_ETHERNET(0xc307),
/* CPSW0 */
PSIL_ETHERNET(0xf000),
PSIL_ETHERNET(0xf001),
PSIL_ETHERNET(0xf002),
PSIL_ETHERNET(0xf003),
PSIL_ETHERNET(0xf004),
PSIL_ETHERNET(0xf005),
PSIL_ETHERNET(0xf006),
PSIL_ETHERNET(0xf007),
};
struct psil_ep_map am654_ep_map = {
.name = "am654",
.src = am654_src_ep_map,
.src_count = ARRAY_SIZE(am654_src_ep_map),
.dst = am654_dst_ep_map,
.dst_count = ARRAY_SIZE(am654_dst_ep_map),
};

222
drivers/dma/ti/k3-psil-j721e.c Normal file
View File

@ -0,0 +1,222 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
*/
#include <linux/kernel.h>
#include "k3-psil-priv.h"
#define PSIL_PDMA_XY_TR(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
}, \
}
#define PSIL_PDMA_XY_PKT(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
.pkt_mode = 1, \
}, \
}
#define PSIL_PDMA_MCASP(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
.pdma_acc32 = 1, \
.pdma_burst = 1, \
}, \
}
#define PSIL_ETHERNET(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 16, \
}, \
}
#define PSIL_SA2UL(x, tx) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 64, \
.notdpkt = tx, \
}, \
}
/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
static struct psil_ep j721e_src_ep_map[] = {
/* SA2UL */
PSIL_SA2UL(0x4000, 0),
PSIL_SA2UL(0x4001, 0),
PSIL_SA2UL(0x4002, 0),
PSIL_SA2UL(0x4003, 0),
/* PRU_ICSSG0 */
PSIL_ETHERNET(0x4100),
PSIL_ETHERNET(0x4101),
PSIL_ETHERNET(0x4102),
PSIL_ETHERNET(0x4103),
/* PRU_ICSSG1 */
PSIL_ETHERNET(0x4200),
PSIL_ETHERNET(0x4201),
PSIL_ETHERNET(0x4202),
PSIL_ETHERNET(0x4203),
/* PDMA6 (PSIL_PDMA_MCASP_G0) - McASP0-2 */
PSIL_PDMA_MCASP(0x4400),
PSIL_PDMA_MCASP(0x4401),
PSIL_PDMA_MCASP(0x4402),
/* PDMA7 (PSIL_PDMA_MCASP_G1) - McASP3-11 */
PSIL_PDMA_MCASP(0x4500),
PSIL_PDMA_MCASP(0x4501),
PSIL_PDMA_MCASP(0x4502),
PSIL_PDMA_MCASP(0x4503),
PSIL_PDMA_MCASP(0x4504),
PSIL_PDMA_MCASP(0x4505),
PSIL_PDMA_MCASP(0x4506),
PSIL_PDMA_MCASP(0x4507),
PSIL_PDMA_MCASP(0x4508),
/* PDMA8 (PDMA_MISC_G0) - SPI0-1 */
PSIL_PDMA_XY_PKT(0x4600),
PSIL_PDMA_XY_PKT(0x4601),
PSIL_PDMA_XY_PKT(0x4602),
PSIL_PDMA_XY_PKT(0x4603),
PSIL_PDMA_XY_PKT(0x4604),
PSIL_PDMA_XY_PKT(0x4605),
PSIL_PDMA_XY_PKT(0x4606),
PSIL_PDMA_XY_PKT(0x4607),
/* PDMA9 (PDMA_MISC_G1) - SPI2-3 */
PSIL_PDMA_XY_PKT(0x460c),
PSIL_PDMA_XY_PKT(0x460d),
PSIL_PDMA_XY_PKT(0x460e),
PSIL_PDMA_XY_PKT(0x460f),
PSIL_PDMA_XY_PKT(0x4610),
PSIL_PDMA_XY_PKT(0x4611),
PSIL_PDMA_XY_PKT(0x4612),
PSIL_PDMA_XY_PKT(0x4613),
/* PDMA10 (PDMA_MISC_G2) - SPI4-5 */
PSIL_PDMA_XY_PKT(0x4618),
PSIL_PDMA_XY_PKT(0x4619),
PSIL_PDMA_XY_PKT(0x461a),
PSIL_PDMA_XY_PKT(0x461b),
PSIL_PDMA_XY_PKT(0x461c),
PSIL_PDMA_XY_PKT(0x461d),
PSIL_PDMA_XY_PKT(0x461e),
PSIL_PDMA_XY_PKT(0x461f),
/* PDMA11 (PDMA_MISC_G3) */
PSIL_PDMA_XY_PKT(0x4624),
PSIL_PDMA_XY_PKT(0x4625),
PSIL_PDMA_XY_PKT(0x4626),
PSIL_PDMA_XY_PKT(0x4627),
PSIL_PDMA_XY_PKT(0x4628),
PSIL_PDMA_XY_PKT(0x4629),
PSIL_PDMA_XY_PKT(0x4630),
PSIL_PDMA_XY_PKT(0x463a),
/* PDMA13 (PDMA_USART_G0) - UART0-1 */
PSIL_PDMA_XY_PKT(0x4700),
PSIL_PDMA_XY_PKT(0x4701),
/* PDMA14 (PDMA_USART_G1) - UART2-3 */
PSIL_PDMA_XY_PKT(0x4702),
PSIL_PDMA_XY_PKT(0x4703),
/* PDMA15 (PDMA_USART_G2) - UART4-9 */
PSIL_PDMA_XY_PKT(0x4704),
PSIL_PDMA_XY_PKT(0x4705),
PSIL_PDMA_XY_PKT(0x4706),
PSIL_PDMA_XY_PKT(0x4707),
PSIL_PDMA_XY_PKT(0x4708),
PSIL_PDMA_XY_PKT(0x4709),
/* CPSW9 */
PSIL_ETHERNET(0x4a00),
/* CPSW0 */
PSIL_ETHERNET(0x7000),
/* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */
PSIL_PDMA_XY_PKT(0x7100),
PSIL_PDMA_XY_PKT(0x7101),
PSIL_PDMA_XY_PKT(0x7102),
PSIL_PDMA_XY_PKT(0x7103),
/* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */
PSIL_PDMA_XY_PKT(0x7200),
PSIL_PDMA_XY_PKT(0x7201),
PSIL_PDMA_XY_PKT(0x7202),
PSIL_PDMA_XY_PKT(0x7203),
PSIL_PDMA_XY_PKT(0x7204),
PSIL_PDMA_XY_PKT(0x7205),
PSIL_PDMA_XY_PKT(0x7206),
PSIL_PDMA_XY_PKT(0x7207),
/* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */
PSIL_PDMA_XY_PKT(0x7300),
/* MCU_PDMA_ADC - ADC0-1 */
PSIL_PDMA_XY_TR(0x7400),
PSIL_PDMA_XY_TR(0x7401),
PSIL_PDMA_XY_TR(0x7402),
PSIL_PDMA_XY_TR(0x7403),
/* SA2UL */
PSIL_SA2UL(0x7500, 0),
PSIL_SA2UL(0x7501, 0),
};
/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */
static struct psil_ep j721e_dst_ep_map[] = {
/* SA2UL */
PSIL_SA2UL(0xc000, 1),
PSIL_SA2UL(0xc001, 1),
/* PRU_ICSSG0 */
PSIL_ETHERNET(0xc100),
PSIL_ETHERNET(0xc101),
PSIL_ETHERNET(0xc102),
PSIL_ETHERNET(0xc103),
PSIL_ETHERNET(0xc104),
PSIL_ETHERNET(0xc105),
PSIL_ETHERNET(0xc106),
PSIL_ETHERNET(0xc107),
/* PRU_ICSSG1 */
PSIL_ETHERNET(0xc200),
PSIL_ETHERNET(0xc201),
PSIL_ETHERNET(0xc202),
PSIL_ETHERNET(0xc203),
PSIL_ETHERNET(0xc204),
PSIL_ETHERNET(0xc205),
PSIL_ETHERNET(0xc206),
PSIL_ETHERNET(0xc207),
/* CPSW9 */
PSIL_ETHERNET(0xca00),
PSIL_ETHERNET(0xca01),
PSIL_ETHERNET(0xca02),
PSIL_ETHERNET(0xca03),
PSIL_ETHERNET(0xca04),
PSIL_ETHERNET(0xca05),
PSIL_ETHERNET(0xca06),
PSIL_ETHERNET(0xca07),
/* CPSW0 */
PSIL_ETHERNET(0xf000),
PSIL_ETHERNET(0xf001),
PSIL_ETHERNET(0xf002),
PSIL_ETHERNET(0xf003),
PSIL_ETHERNET(0xf004),
PSIL_ETHERNET(0xf005),
PSIL_ETHERNET(0xf006),
PSIL_ETHERNET(0xf007),
/* SA2UL */
PSIL_SA2UL(0xf500, 1),
};
struct psil_ep_map j721e_ep_map = {
.name = "j721e",
.src = j721e_src_ep_map,
.src_count = ARRAY_SIZE(j721e_src_ep_map),
.dst = j721e_dst_ep_map,
.dst_count = ARRAY_SIZE(j721e_dst_ep_map),
};

43
drivers/dma/ti/k3-psil-priv.h Normal file
View File

@ -0,0 +1,43 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
*/
#ifndef K3_PSIL_PRIV_H_
#define K3_PSIL_PRIV_H_
#include <linux/dma/k3-psil.h>
struct psil_ep {
u32 thread_id;
struct psil_endpoint_config ep_config;
};
/**
* struct psil_ep_map - PSI-L thread ID configuration maps
* @name: Name of the map, set it to the name of the SoC
* @src: Array of source PSI-L thread configurations
* @src_count: Number of entries in the src array
* @dst: Array of destination PSI-L thread configurations
* @dst_count: Number of entries in the dst array
*
* In case of symmetric configuration for a matching src/dst thread (for example
* 0x4400 and 0xc400) only the src configuration can be present. If no dst
* configuration found the code will look for (dst_thread_id & ~0x8000) to find
* the symmetric match.
*/
struct psil_ep_map {
char *name;
struct psil_ep *src;
int src_count;
struct psil_ep *dst;
int dst_count;
};
struct psil_endpoint_config *psil_get_ep_config(u32 thread_id);
/* SoC PSI-L endpoint maps */
extern struct psil_ep_map am654_ep_map;
extern struct psil_ep_map j721e_ep_map;
#endif /* K3_PSIL_PRIV_H_ */

90
drivers/dma/ti/k3-psil.c Normal file
View File

@ -0,0 +1,90 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
*/
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include "k3-psil-priv.h"
static DEFINE_MUTEX(ep_map_mutex);
static struct psil_ep_map *soc_ep_map;
struct psil_endpoint_config *psil_get_ep_config(u32 thread_id)
{
int i;
mutex_lock(&ep_map_mutex);
if (!soc_ep_map) {
if (of_machine_is_compatible("ti,am654")) {
soc_ep_map = &am654_ep_map;
} else if (of_machine_is_compatible("ti,j721e")) {
soc_ep_map = &j721e_ep_map;
} else {
pr_err("PSIL: No compatible machine found for map\n");
return ERR_PTR(-ENOTSUPP);
}
pr_debug("%s: Using map for %s\n", __func__, soc_ep_map->name);
}
mutex_unlock(&ep_map_mutex);
if (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET && soc_ep_map->dst) {
/* check in destination thread map */
for (i = 0; i < soc_ep_map->dst_count; i++) {
if (soc_ep_map->dst[i].thread_id == thread_id)
return &soc_ep_map->dst[i].ep_config;
}
}
thread_id &= ~K3_PSIL_DST_THREAD_ID_OFFSET;
if (soc_ep_map->src) {
for (i = 0; i < soc_ep_map->src_count; i++) {
if (soc_ep_map->src[i].thread_id == thread_id)
return &soc_ep_map->src[i].ep_config;
}
}
return ERR_PTR(-ENOENT);
}
EXPORT_SYMBOL_GPL(psil_get_ep_config);
int psil_set_new_ep_config(struct device *dev, const char *name,
struct psil_endpoint_config *ep_config)
{
struct psil_endpoint_config *dst_ep_config;
struct of_phandle_args dma_spec;
u32 thread_id;
int index;
if (!dev || !dev->of_node)
return -EINVAL;
index = of_property_match_string(dev->of_node, "dma-names", name);
if (index < 0)
return index;
if (of_parse_phandle_with_args(dev->of_node, "dmas", "#dma-cells",
index, &dma_spec))
return -ENOENT;
thread_id = dma_spec.args[0];
dst_ep_config = psil_get_ep_config(thread_id);
if (IS_ERR(dst_ep_config)) {
pr_err("PSIL: thread ID 0x%04x not defined in map\n",
thread_id);
of_node_put(dma_spec.np);
return PTR_ERR(dst_ep_config);
}
memcpy(dst_ep_config, ep_config, sizeof(*dst_ep_config));
of_node_put(dma_spec.np);
return 0;
}
EXPORT_SYMBOL_GPL(psil_set_new_ep_config);

1198
drivers/dma/ti/k3-udma-glue.c Normal file
View File

File diff suppressed because it is too large Load Diff

133
drivers/dma/ti/k3-udma-private.c Normal file
View File

@ -0,0 +1,133 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
*/
int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
{
return navss_psil_pair(ud, src_thread, dst_thread);
}
EXPORT_SYMBOL(xudma_navss_psil_pair);
int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
{
return navss_psil_unpair(ud, src_thread, dst_thread);
}
EXPORT_SYMBOL(xudma_navss_psil_unpair);
struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property)
{
struct device_node *udma_node = np;
struct platform_device *pdev;
struct udma_dev *ud;
if (property) {
udma_node = of_parse_phandle(np, property, 0);
if (!udma_node) {
pr_err("UDMA node is not found\n");
return ERR_PTR(-ENODEV);
}
}
pdev = of_find_device_by_node(udma_node);
if (!pdev) {
pr_debug("UDMA device not found\n");
return ERR_PTR(-EPROBE_DEFER);
}
if (np != udma_node)
of_node_put(udma_node);
ud = platform_get_drvdata(pdev);
if (!ud) {
pr_debug("UDMA has not been probed\n");
return ERR_PTR(-EPROBE_DEFER);
}
return ud;
}
EXPORT_SYMBOL(of_xudma_dev_get);
u32 xudma_dev_get_psil_base(struct udma_dev *ud)
{
return ud->psil_base;
}
EXPORT_SYMBOL(xudma_dev_get_psil_base);
struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud)
{
return &ud->tisci_rm;
}
EXPORT_SYMBOL(xudma_dev_get_tisci_rm);
int xudma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
{
return __udma_alloc_gp_rflow_range(ud, from, cnt);
}
EXPORT_SYMBOL(xudma_alloc_gp_rflow_range);
int xudma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
{
return __udma_free_gp_rflow_range(ud, from, cnt);
}
EXPORT_SYMBOL(xudma_free_gp_rflow_range);
bool xudma_rflow_is_gp(struct udma_dev *ud, int id)
{
return !test_bit(id, ud->rflow_gp_map);
}
EXPORT_SYMBOL(xudma_rflow_is_gp);
#define XUDMA_GET_PUT_RESOURCE(res) \
struct udma_##res *xudma_##res##_get(struct udma_dev *ud, int id) \
{ \
return __udma_reserve_##res(ud, false, id); \
} \
EXPORT_SYMBOL(xudma_##res##_get); \
\
void xudma_##res##_put(struct udma_dev *ud, struct udma_##res *p) \
{ \
clear_bit(p->id, ud->res##_map); \
} \
EXPORT_SYMBOL(xudma_##res##_put)
XUDMA_GET_PUT_RESOURCE(tchan);
XUDMA_GET_PUT_RESOURCE(rchan);
struct udma_rflow *xudma_rflow_get(struct udma_dev *ud, int id)
{
return __udma_get_rflow(ud, id);
}
EXPORT_SYMBOL(xudma_rflow_get);
void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p)
{
__udma_put_rflow(ud, p);
}
EXPORT_SYMBOL(xudma_rflow_put);
#define XUDMA_GET_RESOURCE_ID(res) \
int xudma_##res##_get_id(struct udma_##res *p) \
{ \
return p->id; \
} \
EXPORT_SYMBOL(xudma_##res##_get_id)
XUDMA_GET_RESOURCE_ID(tchan);
XUDMA_GET_RESOURCE_ID(rchan);
XUDMA_GET_RESOURCE_ID(rflow);
/* Exported register access functions */
#define XUDMA_RT_IO_FUNCTIONS(res) \
u32 xudma_##res##rt_read(struct udma_##res *p, int reg) \
{ \
return udma_##res##rt_read(p, reg); \
} \
EXPORT_SYMBOL(xudma_##res##rt_read); \
\
void xudma_##res##rt_write(struct udma_##res *p, int reg, u32 val) \
{ \
udma_##res##rt_write(p, reg, val); \
} \
EXPORT_SYMBOL(xudma_##res##rt_write)
XUDMA_RT_IO_FUNCTIONS(tchan);
XUDMA_RT_IO_FUNCTIONS(rchan);

3432
drivers/dma/ti/k3-udma.c Normal file
View File

File diff suppressed because it is too large Load Diff

151
drivers/dma/ti/k3-udma.h Normal file
View File

@ -0,0 +1,151 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
*/
#ifndef K3_UDMA_H_
#define K3_UDMA_H_
#include <linux/soc/ti/ti_sci_protocol.h>
/* Global registers */
#define UDMA_REV_REG 0x0
#define UDMA_PERF_CTL_REG 0x4
#define UDMA_EMU_CTL_REG 0x8
#define UDMA_PSIL_TO_REG 0x10
#define UDMA_UTC_CTL_REG 0x1c
#define UDMA_CAP_REG(i) (0x20 + ((i) * 4))
#define UDMA_RX_FLOW_ID_FW_OES_REG 0x80
#define UDMA_RX_FLOW_ID_FW_STATUS_REG 0x88
/* TX chan RT regs */
#define UDMA_TCHAN_RT_CTL_REG 0x0
#define UDMA_TCHAN_RT_SWTRIG_REG 0x8
#define UDMA_TCHAN_RT_STDATA_REG 0x80
#define UDMA_TCHAN_RT_PEER_REG(i) (0x200 + ((i) * 0x4))
#define UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG \
UDMA_TCHAN_RT_PEER_REG(0) /* PSI-L: 0x400 */
#define UDMA_TCHAN_RT_PEER_STATIC_TR_Z_REG \
UDMA_TCHAN_RT_PEER_REG(1) /* PSI-L: 0x401 */
#define UDMA_TCHAN_RT_PEER_BCNT_REG \
UDMA_TCHAN_RT_PEER_REG(4) /* PSI-L: 0x404 */
#define UDMA_TCHAN_RT_PEER_RT_EN_REG \
UDMA_TCHAN_RT_PEER_REG(8) /* PSI-L: 0x408 */
#define UDMA_TCHAN_RT_PCNT_REG 0x400
#define UDMA_TCHAN_RT_BCNT_REG 0x408
#define UDMA_TCHAN_RT_SBCNT_REG 0x410
/* RX chan RT regs */
#define UDMA_RCHAN_RT_CTL_REG 0x0
#define UDMA_RCHAN_RT_SWTRIG_REG 0x8
#define UDMA_RCHAN_RT_STDATA_REG 0x80
#define UDMA_RCHAN_RT_PEER_REG(i) (0x200 + ((i) * 0x4))
#define UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG \
UDMA_RCHAN_RT_PEER_REG(0) /* PSI-L: 0x400 */
#define UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG \
UDMA_RCHAN_RT_PEER_REG(1) /* PSI-L: 0x401 */
#define UDMA_RCHAN_RT_PEER_BCNT_REG \
UDMA_RCHAN_RT_PEER_REG(4) /* PSI-L: 0x404 */
#define UDMA_RCHAN_RT_PEER_RT_EN_REG \
UDMA_RCHAN_RT_PEER_REG(8) /* PSI-L: 0x408 */
#define UDMA_RCHAN_RT_PCNT_REG 0x400
#define UDMA_RCHAN_RT_BCNT_REG 0x408
#define UDMA_RCHAN_RT_SBCNT_REG 0x410
/* UDMA_TCHAN_RT_CTL_REG/UDMA_RCHAN_RT_CTL_REG */
#define UDMA_CHAN_RT_CTL_EN BIT(31)
#define UDMA_CHAN_RT_CTL_TDOWN BIT(30)
#define UDMA_CHAN_RT_CTL_PAUSE BIT(29)
#define UDMA_CHAN_RT_CTL_FTDOWN BIT(28)
#define UDMA_CHAN_RT_CTL_ERROR BIT(0)
/* UDMA_TCHAN_RT_PEER_RT_EN_REG/UDMA_RCHAN_RT_PEER_RT_EN_REG (PSI-L: 0x408) */
#define UDMA_PEER_RT_EN_ENABLE BIT(31)
#define UDMA_PEER_RT_EN_TEARDOWN BIT(30)
#define UDMA_PEER_RT_EN_PAUSE BIT(29)
#define UDMA_PEER_RT_EN_FLUSH BIT(28)
#define UDMA_PEER_RT_EN_IDLE BIT(1)
/*
* UDMA_TCHAN_RT_PEER_STATIC_TR_XY_REG /
* UDMA_RCHAN_RT_PEER_STATIC_TR_XY_REG
*/
#define PDMA_STATIC_TR_X_MASK GENMASK(26, 24)
#define PDMA_STATIC_TR_X_SHIFT (24)
#define PDMA_STATIC_TR_Y_MASK GENMASK(11, 0)
#define PDMA_STATIC_TR_Y_SHIFT (0)
#define PDMA_STATIC_TR_Y(x) \
(((x) << PDMA_STATIC_TR_Y_SHIFT) & PDMA_STATIC_TR_Y_MASK)
#define PDMA_STATIC_TR_X(x) \
(((x) << PDMA_STATIC_TR_X_SHIFT) & PDMA_STATIC_TR_X_MASK)
#define PDMA_STATIC_TR_XY_ACC32 BIT(30)
#define PDMA_STATIC_TR_XY_BURST BIT(31)
/*
* UDMA_TCHAN_RT_PEER_STATIC_TR_Z_REG /
* UDMA_RCHAN_RT_PEER_STATIC_TR_Z_REG
*/
#define PDMA_STATIC_TR_Z(x, mask) ((x) & (mask))
struct udma_dev;
struct udma_tchan;
struct udma_rchan;
struct udma_rflow;
enum udma_rm_range {
RM_RANGE_TCHAN = 0,
RM_RANGE_RCHAN,
RM_RANGE_RFLOW,
RM_RANGE_LAST,
};
struct udma_tisci_rm {
const struct ti_sci_handle *tisci;
const struct ti_sci_rm_udmap_ops *tisci_udmap_ops;
u32 tisci_dev_id;
/* tisci information for PSI-L thread pairing/unpairing */
const struct ti_sci_rm_psil_ops *tisci_psil_ops;
u32 tisci_navss_dev_id;
struct ti_sci_resource *rm_ranges[RM_RANGE_LAST];
};
/* Direct access to UDMA low lever resources for the glue layer */
int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread);
int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
u32 dst_thread);
struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property);
void xudma_dev_put(struct udma_dev *ud);
u32 xudma_dev_get_psil_base(struct udma_dev *ud);
struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud);
int xudma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt);
int xudma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt);
struct udma_tchan *xudma_tchan_get(struct udma_dev *ud, int id);
struct udma_rchan *xudma_rchan_get(struct udma_dev *ud, int id);
struct udma_rflow *xudma_rflow_get(struct udma_dev *ud, int id);
void xudma_tchan_put(struct udma_dev *ud, struct udma_tchan *p);
void xudma_rchan_put(struct udma_dev *ud, struct udma_rchan *p);
void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p);
int xudma_tchan_get_id(struct udma_tchan *p);
int xudma_rchan_get_id(struct udma_rchan *p);
int xudma_rflow_get_id(struct udma_rflow *p);
u32 xudma_tchanrt_read(struct udma_tchan *tchan, int reg);
void xudma_tchanrt_write(struct udma_tchan *tchan, int reg, u32 val);
u32 xudma_rchanrt_read(struct udma_rchan *rchan, int reg);
void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val);
bool xudma_rflow_is_gp(struct udma_dev *ud, int id);
#endif /* K3_UDMA_H_ */

8
drivers/dma/virt-dma.c
View File

@ -114,13 +114,8 @@ void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head)
struct virt_dma_desc *vd, *_vd;
list_for_each_entry_safe(vd, _vd, head, node) {
if (dmaengine_desc_test_reuse(&vd->tx)) {
list_move_tail(&vd->node, &vc->desc_allocated);
} else {
dev_dbg(vc->chan.device->dev, "txd %p: freeing\n", vd);
list_del(&vd->node);
vc->desc_free(vd);
}
vchan_vdesc_fini(vd);
}
}
EXPORT_SYMBOL_GPL(vchan_dma_desc_free_list);
@ -134,6 +129,7 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev)
INIT_LIST_HEAD(&vc->desc_submitted);
INIT_LIST_HEAD(&vc->desc_issued);
INIT_LIST_HEAD(&vc->desc_completed);
INIT_LIST_HEAD(&vc->desc_terminated);
tasklet_init(&vc->task, vchan_complete, (unsigned long)vc);

27
drivers/dma/virt-dma.h
View File

@ -31,9 +31,9 @@ struct virt_dma_chan {
struct list_head desc_submitted;
struct list_head desc_issued;
struct list_head desc_completed;
struct list_head desc_terminated;
struct virt_dma_desc *cyclic;
struct virt_dma_desc *vd_terminated;
};
static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan)
@ -113,10 +113,15 @@ static inline void vchan_vdesc_fini(struct virt_dma_desc *vd)
{
struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
if (dmaengine_desc_test_reuse(&vd->tx))
if (dmaengine_desc_test_reuse(&vd->tx)) {
unsigned long flags;
spin_lock_irqsave(&vc->lock, flags);
list_add(&vd->node, &vc->desc_allocated);
else
spin_unlock_irqrestore(&vc->lock, flags);
} else {
vc->desc_free(vd);
}
}
/**
@ -141,11 +146,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd)
{
struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
/* free up stuck descriptor */
if (vc->vd_terminated)
vchan_vdesc_fini(vc->vd_terminated);
list_add_tail(&vd->node, &vc->desc_terminated);
vc->vd_terminated = vd;
if (vc->cyclic == vd)
vc->cyclic = NULL;
}
@ -179,6 +181,7 @@ static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc,
list_splice_tail_init(&vc->desc_submitted, head);
list_splice_tail_init(&vc->desc_issued, head);
list_splice_tail_init(&vc->desc_completed, head);
list_splice_tail_init(&vc->desc_terminated, head);
}
static inline void vchan_free_chan_resources(struct virt_dma_chan *vc)
@ -207,16 +210,18 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc)
*/
static inline void vchan_synchronize(struct virt_dma_chan *vc)
{
LIST_HEAD(head);
unsigned long flags;
tasklet_kill(&vc->task);
spin_lock_irqsave(&vc->lock, flags);
if (vc->vd_terminated) {
vchan_vdesc_fini(vc->vd_terminated);
vc->vd_terminated = NULL;
}
list_splice_tail_init(&vc->desc_terminated, &head);
spin_unlock_irqrestore(&vc->lock, flags);
vchan_dma_desc_free_list(vc, &head);
}
#endif

24
drivers/dma/xilinx/zynqmp_dma.c
View File

@ -123,10 +123,12 @@
/* Max transfer size per descriptor */
#define ZYNQMP_DMA_MAX_TRANS_LEN 0x40000000
/* Max burst lengths */
#define ZYNQMP_DMA_MAX_DST_BURST_LEN 32768U
#define ZYNQMP_DMA_MAX_SRC_BURST_LEN 32768U
/* Reset values for data attributes */
#define ZYNQMP_DMA_AXCACHE_VAL 0xF
#define ZYNQMP_DMA_ARLEN_RST_VAL 0xF
#define ZYNQMP_DMA_AWLEN_RST_VAL 0xF
#define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F
@ -534,17 +536,19 @@ static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status)
static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)
{
u32 val;
u32 val, burst_val;
val = readl(chan->regs + ZYNQMP_DMA_CTRL0);
val |= ZYNQMP_DMA_POINT_TYPE_SG;
writel(val, chan->regs + ZYNQMP_DMA_CTRL0);
val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);
burst_val = __ilog2_u32(chan->src_burst_len);
val = (val & ~ZYNQMP_DMA_ARLEN) |
(chan->src_burst_len << ZYNQMP_DMA_ARLEN_OFST);
((burst_val << ZYNQMP_DMA_ARLEN_OFST) & ZYNQMP_DMA_ARLEN);
burst_val = __ilog2_u32(chan->dst_burst_len);
val = (val & ~ZYNQMP_DMA_AWLEN) |
(chan->dst_burst_len << ZYNQMP_DMA_AWLEN_OFST);
((burst_val << ZYNQMP_DMA_AWLEN_OFST) & ZYNQMP_DMA_AWLEN);
writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);
}
@ -560,8 +564,10 @@ static int zynqmp_dma_device_config(struct dma_chan *dchan,
{
struct zynqmp_dma_chan *chan = to_chan(dchan);
chan->src_burst_len = config->src_maxburst;
chan->dst_burst_len = config->dst_maxburst;
chan->src_burst_len = clamp(config->src_maxburst, 1U,
ZYNQMP_DMA_MAX_SRC_BURST_LEN);
chan->dst_burst_len = clamp(config->dst_maxburst, 1U,
ZYNQMP_DMA_MAX_DST_BURST_LEN);
return 0;
}
@ -887,8 +893,8 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
return PTR_ERR(chan->regs);
chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;
chan->dst_burst_len = ZYNQMP_DMA_AWLEN_RST_VAL;
chan->src_burst_len = ZYNQMP_DMA_ARLEN_RST_VAL;
chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN;
chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN;
err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);
if (err < 0) {
dev_err(&pdev->dev, "missing xlnx,bus-width property\n");

11
drivers/soc/ti/Kconfig
View File

@ -80,6 +80,17 @@ config TI_SCI_PM_DOMAINS
called ti_sci_pm_domains. Note this is needed early in boot before
rootfs may be available.
config TI_K3_RINGACC
bool "K3 Ring accelerator Sub System"
depends on ARCH_K3 || COMPILE_TEST
depends on TI_SCI_INTA_IRQCHIP
help
Say y here to support the K3 Ring accelerator module.
The Ring Accelerator (RINGACC or RA) provides hardware acceleration
to enable straightforward passing of work between a producer
and a consumer. There is one RINGACC module per NAVSS on TI AM65x SoCs
If unsure, say N.
endif # SOC_TI
config TI_SCI_INTA_MSI_DOMAIN

1
drivers/soc/ti/Makefile
View File

@ -10,3 +10,4 @@ obj-$(CONFIG_ARCH_OMAP2PLUS) += omap_prm.o
obj-$(CONFIG_WKUP_M3_IPC) += wkup_m3_ipc.o
obj-$(CONFIG_TI_SCI_PM_DOMAINS) += ti_sci_pm_domains.o
obj-$(CONFIG_TI_SCI_INTA_MSI_DOMAIN) += ti_sci_inta_msi.o
obj-$(CONFIG_TI_K3_RINGACC) += k3-ringacc.o

1157
drivers/soc/ti/k3-ringacc.c Normal file
View File

File diff suppressed because it is too large Load Diff

39
include/dt-bindings/dma/x1830-dma.h Normal file
View File

@ -0,0 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* This header provides macros for X1830 DMA bindings.
*
* Copyright (c) 2019 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
*/
#ifndef __DT_BINDINGS_DMA_X1830_DMA_H__
#define __DT_BINDINGS_DMA_X1830_DMA_H__
/*
* Request type numbers for the X1830 DMA controller (written to the DRTn
* register for the channel).
*/
#define X1830_DMA_I2S0_TX 0x6
#define X1830_DMA_I2S0_RX 0x7
#define X1830_DMA_AUTO 0x8
#define X1830_DMA_SADC_RX 0x9
#define X1830_DMA_UART1_TX 0x12
#define X1830_DMA_UART1_RX 0x13
#define X1830_DMA_UART0_TX 0x14
#define X1830_DMA_UART0_RX 0x15
#define X1830_DMA_SSI0_TX 0x16
#define X1830_DMA_SSI0_RX 0x17
#define X1830_DMA_SSI1_TX 0x18
#define X1830_DMA_SSI1_RX 0x19
#define X1830_DMA_MSC0_TX 0x1a
#define X1830_DMA_MSC0_RX 0x1b
#define X1830_DMA_MSC1_TX 0x1c
#define X1830_DMA_MSC1_RX 0x1d
#define X1830_DMA_DMIC_RX 0x21
#define X1830_DMA_SMB0_TX 0x24
#define X1830_DMA_SMB0_RX 0x25
#define X1830_DMA_SMB1_TX 0x26
#define X1830_DMA_SMB1_RX 0x27
#define X1830_DMA_DES_TX 0x2e
#define X1830_DMA_DES_RX 0x2f
#endif /* __DT_BINDINGS_DMA_X1830_DMA_H__ */

71
include/linux/dma/k3-psil.h Normal file
View File

@ -0,0 +1,71 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
*/
#ifndef K3_PSIL_H_
#define K3_PSIL_H_
#include <linux/types.h>
#define K3_PSIL_DST_THREAD_ID_OFFSET 0x8000
struct device;
/**
* enum udma_tp_level - Channel Throughput Levels
* @UDMA_TP_NORMAL: Normal channel
* @UDMA_TP_HIGH: High Throughput channel
* @UDMA_TP_ULTRAHIGH: Ultra High Throughput channel
*/
enum udma_tp_level {
UDMA_TP_NORMAL = 0,
UDMA_TP_HIGH,
UDMA_TP_ULTRAHIGH,
UDMA_TP_LAST,
};
/**
* enum psil_endpoint_type - PSI-L Endpoint type
* @PSIL_EP_NATIVE: Normal channel
* @PSIL_EP_PDMA_XY: XY mode PDMA
* @PSIL_EP_PDMA_MCAN: MCAN mode PDMA
* @PSIL_EP_PDMA_AASRC: AASRC mode PDMA
*/
enum psil_endpoint_type {
PSIL_EP_NATIVE = 0,
PSIL_EP_PDMA_XY,
PSIL_EP_PDMA_MCAN,
PSIL_EP_PDMA_AASRC,
};
/**
* struct psil_endpoint_config - PSI-L Endpoint configuration
* @ep_type: PSI-L endpoint type
* @pkt_mode: If set, the channel must be in Packet mode, otherwise in
* TR mode
* @notdpkt: TDCM must be suppressed on the TX channel
* @needs_epib: Endpoint needs EPIB
* @psd_size: If set, PSdata is used by the endpoint
* @channel_tpl: Desired throughput level for the channel
* @pdma_acc32: ACC32 must be enabled on the PDMA side
* @pdma_burst: BURST must be enabled on the PDMA side
*/
struct psil_endpoint_config {
enum psil_endpoint_type ep_type;
unsigned pkt_mode:1;
unsigned notdpkt:1;
unsigned needs_epib:1;
u32 psd_size;
enum udma_tp_level channel_tpl;
/* PDMA properties, valid for PSIL_EP_PDMA_* */
unsigned pdma_acc32:1;
unsigned pdma_burst:1;
};
int psil_set_new_ep_config(struct device *dev, const char *name,
struct psil_endpoint_config *ep_config);
#endif /* K3_PSIL_H_ */

134
include/linux/dma/k3-udma-glue.h Normal file
View File

@ -0,0 +1,134 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
*/
#ifndef K3_UDMA_GLUE_H_
#define K3_UDMA_GLUE_H_
#include <linux/types.h>
#include <linux/soc/ti/k3-ringacc.h>
#include <linux/dma/ti-cppi5.h>
struct k3_udma_glue_tx_channel_cfg {
struct k3_ring_cfg tx_cfg;
struct k3_ring_cfg txcq_cfg;
bool tx_pause_on_err;
bool tx_filt_einfo;
bool tx_filt_pswords;
bool tx_supr_tdpkt;
u32 swdata_size;
};
struct k3_udma_glue_tx_channel;
struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
const char *name, struct k3_udma_glue_tx_channel_cfg *cfg);
void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn);
int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
struct cppi5_host_desc_t *desc_tx,
dma_addr_t desc_dma);
int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
dma_addr_t *desc_dma);
int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn);
void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn);
void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
bool sync);
void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
void *data, void (*cleanup)(void *data, dma_addr_t desc_dma));
u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn);
u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn);
int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn);
enum {
K3_UDMA_GLUE_SRC_TAG_LO_KEEP = 0,
K3_UDMA_GLUE_SRC_TAG_LO_USE_FLOW_REG = 1,
K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_FLOW_ID = 2,
K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG = 4,
};
/**
* k3_udma_glue_rx_flow_cfg - UDMA RX flow cfg
*
* @rx_cfg: RX ring configuration
* @rxfdq_cfg: RX free Host PD ring configuration
* @ring_rxq_id: RX ring id (or -1 for any)
* @ring_rxfdq0_id: RX free Host PD ring (FDQ) if (or -1 for any)
* @rx_error_handling: Rx Error Handling Mode (0 - drop, 1 - re-try)
* @src_tag_lo_sel: Rx Source Tag Low Byte Selector in Host PD
*/
struct k3_udma_glue_rx_flow_cfg {
struct k3_ring_cfg rx_cfg;
struct k3_ring_cfg rxfdq_cfg;
int ring_rxq_id;
int ring_rxfdq0_id;
bool rx_error_handling;
int src_tag_lo_sel;
};
/**
* k3_udma_glue_rx_channel_cfg - UDMA RX channel cfg
*
* @psdata_size: SW Data is present in Host PD of @swdata_size bytes
* @flow_id_base: first flow_id used by channel.
* if @flow_id_base = -1 - range of GP rflows will be
* allocated dynamically.
* @flow_id_num: number of RX flows used by channel
* @flow_id_use_rxchan_id: use RX channel id as flow id,
* used only if @flow_id_num = 1
* @remote indication that RX channel is remote - some remote CPU
* core owns and control the RX channel. Linux Host only
* allowed to attach and configure RX Flow within RX
* channel. if set - not RX channel operation will be
* performed by K3 NAVSS DMA glue interface.
* @def_flow_cfg default RX flow configuration,
* used only if @flow_id_num = 1
*/
struct k3_udma_glue_rx_channel_cfg {
u32 swdata_size;
int flow_id_base;
int flow_id_num;
bool flow_id_use_rxchan_id;
bool remote;
struct k3_udma_glue_rx_flow_cfg *def_flow_cfg;
};
struct k3_udma_glue_rx_channel;
struct k3_udma_glue_rx_channel *k3_udma_glue_request_rx_chn(
struct device *dev,
const char *name,
struct k3_udma_glue_rx_channel_cfg *cfg);
void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn);
void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
bool sync);
int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_num, struct cppi5_host_desc_t *desc_tx,
dma_addr_t desc_dma);
int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_num, dma_addr_t *desc_dma);
int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_idx, struct k3_udma_glue_rx_flow_cfg *flow_cfg);
u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_idx);
u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn);
int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_num);
void k3_udma_glue_rx_put_irq(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_num);
void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_num, void *data,
void (*cleanup)(void *data, dma_addr_t desc_dma),
bool skip_fdq);
int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_idx);
int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
u32 flow_idx);
#endif /* K3_UDMA_GLUE_H_ */

1059
include/linux/dma/ti-cppi5.h Normal file
View File

File diff suppressed because it is too large Load Diff

159
include/linux/dmaengine.h
View File

@ -219,6 +219,62 @@ typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
* @bytes_transferred: byte counter
*/
/**
* enum dma_desc_metadata_mode - per descriptor metadata mode types supported
* @DESC_METADATA_CLIENT - the metadata buffer is allocated/provided by the
* client driver and it is attached (via the dmaengine_desc_attach_metadata()
* helper) to the descriptor.
*
* Client drivers interested to use this mode can follow:
* - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
* 1. prepare the descriptor (dmaengine_prep_*)
* construct the metadata in the client's buffer
* 2. use dmaengine_desc_attach_metadata() to attach the buffer to the
* descriptor
* 3. submit the transfer
* - DMA_DEV_TO_MEM:
* 1. prepare the descriptor (dmaengine_prep_*)
* 2. use dmaengine_desc_attach_metadata() to attach the buffer to the
* descriptor
* 3. submit the transfer
* 4. when the transfer is completed, the metadata should be available in the
* attached buffer
*
* @DESC_METADATA_ENGINE - the metadata buffer is allocated/managed by the DMA
* driver. The client driver can ask for the pointer, maximum size and the
* currently used size of the metadata and can directly update or read it.
* dmaengine_desc_get_metadata_ptr() and dmaengine_desc_set_metadata_len() is
* provided as helper functions.
*
* Note: the metadata area for the descriptor is no longer valid after the
* transfer has been completed (valid up to the point when the completion
* callback returns if used).
*
* Client drivers interested to use this mode can follow:
* - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
* 1. prepare the descriptor (dmaengine_prep_*)
* 2. use dmaengine_desc_get_metadata_ptr() to get the pointer to the engine's
* metadata area
* 3. update the metadata at the pointer
* 4. use dmaengine_desc_set_metadata_len() to tell the DMA engine the amount
* of data the client has placed into the metadata buffer
* 5. submit the transfer
* - DMA_DEV_TO_MEM:
* 1. prepare the descriptor (dmaengine_prep_*)
* 2. submit the transfer
* 3. on transfer completion, use dmaengine_desc_get_metadata_ptr() to get the
* pointer to the engine's metadata area
* 4. Read out the metadata from the pointer
*
* Note: the two mode is not compatible and clients must use one mode for a
* descriptor.
*/
enum dma_desc_metadata_mode {
DESC_METADATA_NONE = 0,
DESC_METADATA_CLIENT = BIT(0),
DESC_METADATA_ENGINE = BIT(1),
};
struct dma_chan_percpu {
/* stats */
unsigned long memcpy_count;
@ -238,10 +294,12 @@ struct dma_router {
/**
* struct dma_chan - devices supply DMA channels, clients use them
* @device: ptr to the dma device who supplies this channel, always !%NULL
* @slave: ptr to the device using this channel
* @cookie: last cookie value returned to client
* @completed_cookie: last completed cookie for this channel
* @chan_id: channel ID for sysfs
* @dev: class device for sysfs
* @name: backlink name for sysfs
* @device_node: used to add this to the device chan list
* @local: per-cpu pointer to a struct dma_chan_percpu
* @client_count: how many clients are using this channel
@ -252,12 +310,14 @@ struct dma_router {
*/
struct dma_chan {
struct dma_device *device;
struct device *slave;
dma_cookie_t cookie;
dma_cookie_t completed_cookie;
/* sysfs */
int chan_id;
struct dma_chan_dev *dev;
const char *name;
struct list_head device_node;
struct dma_chan_percpu __percpu *local;
@ -475,6 +535,18 @@ struct dmaengine_unmap_data {
dma_addr_t addr[0];
};
struct dma_async_tx_descriptor;
struct dma_descriptor_metadata_ops {
int (*attach)(struct dma_async_tx_descriptor *desc, void *data,
size_t len);
void *(*get_ptr)(struct dma_async_tx_descriptor *desc,
size_t *payload_len, size_t *max_len);
int (*set_len)(struct dma_async_tx_descriptor *desc,
size_t payload_len);
};
/**
* struct dma_async_tx_descriptor - async transaction descriptor
* ---dma generic offload fields---
@ -488,6 +560,11 @@ struct dmaengine_unmap_data {
* descriptor pending. To be pushed on .issue_pending() call
* @callback: routine to call after this operation is complete
* @callback_param: general parameter to pass to the callback routine
* @desc_metadata_mode: core managed metadata mode to protect mixed use of
* DESC_METADATA_CLIENT or DESC_METADATA_ENGINE. Otherwise
* DESC_METADATA_NONE
* @metadata_ops: DMA driver provided metadata mode ops, need to be set by the
* DMA driver if metadata mode is supported with the descriptor
* ---async_tx api specific fields---
* @next: at completion submit this descriptor
* @parent: pointer to the next level up in the dependency chain
@ -504,6 +581,8 @@ struct dma_async_tx_descriptor {
dma_async_tx_callback_result callback_result;
void *callback_param;
struct dmaengine_unmap_data *unmap;
enum dma_desc_metadata_mode desc_metadata_mode;
struct dma_descriptor_metadata_ops *metadata_ops;
#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
struct dma_async_tx_descriptor *next;
struct dma_async_tx_descriptor *parent;
@ -611,11 +690,13 @@ static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descr
* @residue: the remaining number of bytes left to transmit
* on the selected transfer for states DMA_IN_PROGRESS and
* DMA_PAUSED if this is implemented in the driver, else 0
* @in_flight_bytes: amount of data in bytes cached by the DMA.
*/
struct dma_tx_state {
dma_cookie_t last;
dma_cookie_t used;
u32 residue;
u32 in_flight_bytes;
};
/**
@ -666,6 +747,7 @@ struct dma_filter {
* @global_node: list_head for global dma_device_list
* @filter: information for device/slave to filter function/param mapping
* @cap_mask: one or more dma_capability flags
* @desc_metadata_modes: supported metadata modes by the DMA device
* @max_xor: maximum number of xor sources, 0 if no capability
* @max_pq: maximum number of PQ sources and PQ-continue capability
* @copy_align: alignment shift for memcpy operations
@ -674,6 +756,7 @@ struct dma_filter {
* @fill_align: alignment shift for memset operations
* @dev_id: unique device ID
* @dev: struct device reference for dma mapping api
* @owner: owner module (automatically set based on the provided dev)
* @src_addr_widths: bit mask of src addr widths the device supports
* Width is specified in bytes, e.g. for a device supporting
* a width of 4 the mask should have BIT(4) set.
@ -718,15 +801,21 @@ struct dma_filter {
* will just return a simple status code
* @device_issue_pending: push pending transactions to hardware
* @descriptor_reuse: a submitted transfer can be resubmitted after completion
* @device_release: called sometime atfer dma_async_device_unregister() is
* called and there are no further references to this structure. This
* must be implemented to free resources however many existing drivers
* do not and are therefore not safe to unbind while in use.
*
*/
struct dma_device {
struct kref ref;
unsigned int chancnt;
unsigned int privatecnt;
struct list_head channels;
struct list_head global_node;
struct dma_filter filter;
dma_cap_mask_t cap_mask;
enum dma_desc_metadata_mode desc_metadata_modes;
unsigned short max_xor;
unsigned short max_pq;
enum dmaengine_alignment copy_align;
@ -737,6 +826,7 @@ struct dma_device {
int dev_id;
struct device *dev;
struct module *owner;
u32 src_addr_widths;
u32 dst_addr_widths;
@ -800,6 +890,7 @@ struct dma_device {
dma_cookie_t cookie,
struct dma_tx_state *txstate);
void (*device_issue_pending)(struct dma_chan *chan);
void (*device_release)(struct dma_device *dev);
};
static inline int dmaengine_slave_config(struct dma_chan *chan,
@ -902,6 +993,41 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy(
len, flags);
}
static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan,
enum dma_desc_metadata_mode mode)
{
if (!chan)
return false;
return !!(chan->device->desc_metadata_modes & mode);
}
#ifdef CONFIG_DMA_ENGINE
int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
void *data, size_t len);
void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
size_t *payload_len, size_t *max_len);
int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
size_t payload_len);
#else /* CONFIG_DMA_ENGINE */
static inline int dmaengine_desc_attach_metadata(
struct dma_async_tx_descriptor *desc, void *data, size_t len)
{
return -EINVAL;
}
static inline void *dmaengine_desc_get_metadata_ptr(
struct dma_async_tx_descriptor *desc, size_t *payload_len,
size_t *max_len)
{
return NULL;
}
static inline int dmaengine_desc_set_metadata_len(
struct dma_async_tx_descriptor *desc, size_t payload_len)
{
return -EINVAL;
}
#endif /* CONFIG_DMA_ENGINE */
/**
* dmaengine_terminate_all() - Terminate all active DMA transfers
* @chan: The channel for which to terminate the transfers
@ -1402,16 +1528,16 @@ static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc)
int dma_async_device_register(struct dma_device *device);
int dmaenginem_async_device_register(struct dma_device *device);
void dma_async_device_unregister(struct dma_device *device);
int dma_async_device_channel_register(struct dma_device *device,
struct dma_chan *chan);
void dma_async_device_channel_unregister(struct dma_device *device,
struct dma_chan *chan);
void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
struct dma_chan *dma_get_any_slave_channel(struct dma_device *device);
#define dma_request_channel(mask, x, y) \
__dma_request_channel(&(mask), x, y, NULL)
#define dma_request_slave_channel_compat(mask, x, y, dev, name) \
__dma_request_slave_channel_compat(&(mask), x, y, dev, name)
static inline struct dma_chan
*__dma_request_slave_channel_compat(const dma_cap_mask_t *mask,
*dma_request_slave_channel_compat(const dma_cap_mask_t mask,
dma_filter_fn fn, void *fn_param,
struct device *dev, const char *name)
{
@ -1424,6 +1550,25 @@ static inline struct dma_chan
if (!fn || !fn_param)
return NULL;
return __dma_request_channel(mask, fn, fn_param, NULL);
return __dma_request_channel(&mask, fn, fn_param, NULL);
}
static inline char *
dmaengine_get_direction_text(enum dma_transfer_direction dir)
{
switch (dir) {
case DMA_DEV_TO_MEM:
return "DEV_TO_MEM";
case DMA_MEM_TO_DEV:
return "MEM_TO_DEV";
case DMA_MEM_TO_MEM:
return "MEM_TO_MEM";
case DMA_DEV_TO_DEV:
return "DEV_TO_DEV";
default:
break;
}
return "invalid";
}
#endif /* DMAENGINE_H */

244
include/linux/soc/ti/k3-ringacc.h Normal file
View File

@ -0,0 +1,244 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* K3 Ring Accelerator (RA) subsystem interface
*
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
*/
#ifndef __SOC_TI_K3_RINGACC_API_H_
#define __SOC_TI_K3_RINGACC_API_H_
#include <linux/types.h>
struct device_node;
/**
* enum k3_ring_mode - &struct k3_ring_cfg mode
*
* RA ring operational modes
*
* @K3_RINGACC_RING_MODE_RING: Exposed Ring mode for SW direct access
* @K3_RINGACC_RING_MODE_MESSAGE: Messaging mode. Messaging mode requires
* that all accesses to the queue must go through this IP so that all
* accesses to the memory are controlled and ordered. This IP then
* controls the entire state of the queue, and SW has no directly control,
* such as through doorbells and cannot access the storage memory directly.
* This is particularly useful when more than one SW or HW entity can be
* the producer and/or consumer at the same time
* @K3_RINGACC_RING_MODE_CREDENTIALS: Credentials mode is message mode plus
* stores credentials with each message, requiring the element size to be
* doubled to fit the credentials. Any exposed memory should be protected
* by a firewall from unwanted access
*/
enum k3_ring_mode {
K3_RINGACC_RING_MODE_RING = 0,
K3_RINGACC_RING_MODE_MESSAGE,
K3_RINGACC_RING_MODE_CREDENTIALS,
K3_RINGACC_RING_MODE_INVALID
};
/**
* enum k3_ring_size - &struct k3_ring_cfg elm_size
*
* RA ring element's sizes in bytes.
*/
enum k3_ring_size {
K3_RINGACC_RING_ELSIZE_4 = 0,
K3_RINGACC_RING_ELSIZE_8,
K3_RINGACC_RING_ELSIZE_16,
K3_RINGACC_RING_ELSIZE_32,
K3_RINGACC_RING_ELSIZE_64,
K3_RINGACC_RING_ELSIZE_128,
K3_RINGACC_RING_ELSIZE_256,
K3_RINGACC_RING_ELSIZE_INVALID
};
struct k3_ringacc;
struct k3_ring;
/**
* enum k3_ring_cfg - RA ring configuration structure
*
* @size: Ring size, number of elements
* @elm_size: Ring element size
* @mode: Ring operational mode
* @flags: Ring configuration flags. Possible values:
* @K3_RINGACC_RING_SHARED: when set allows to request the same ring
* few times. It's usable when the same ring is used as Free Host PD ring
* for different flows, for example.
* Note: Locking should be done by consumer if required
*/
struct k3_ring_cfg {
u32 size;
enum k3_ring_size elm_size;
enum k3_ring_mode mode;
#define K3_RINGACC_RING_SHARED BIT(1)
u32 flags;
};
#define K3_RINGACC_RING_ID_ANY (-1)
/**
* of_k3_ringacc_get_by_phandle - find a RA by phandle property
* @np: device node
* @propname: property name containing phandle on RA node
*
* Returns pointer on the RA - struct k3_ringacc
* or -ENODEV if not found,
* or -EPROBE_DEFER if not yet registered
*/
struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
const char *property);
#define K3_RINGACC_RING_USE_PROXY BIT(1)
/**
* k3_ringacc_request_ring - request ring from ringacc
* @ringacc: pointer on ringacc
* @id: ring id or K3_RINGACC_RING_ID_ANY for any general purpose ring
* @flags:
* @K3_RINGACC_RING_USE_PROXY: if set - proxy will be allocated and
* used to access ring memory. Sopported only for rings in
* Message/Credentials/Queue mode.
*
* Returns pointer on the Ring - struct k3_ring
* or NULL in case of failure.
*/
struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
int id, u32 flags);
/**
* k3_ringacc_ring_reset - ring reset
* @ring: pointer on Ring
*
* Resets ring internal state ((hw)occ, (hw)idx).
*/
void k3_ringacc_ring_reset(struct k3_ring *ring);
/**
* k3_ringacc_ring_reset - ring reset for DMA rings
* @ring: pointer on Ring
*
* Resets ring internal state ((hw)occ, (hw)idx). Should be used for rings
* which are read by K3 UDMA, like TX or Free Host PD rings.
*/
void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ);
/**
* k3_ringacc_ring_free - ring free
* @ring: pointer on Ring
*
* Resets ring and free all alocated resources.
*/
int k3_ringacc_ring_free(struct k3_ring *ring);
/**
* k3_ringacc_get_ring_id - Get the Ring ID
* @ring: pointer on ring
*
* Returns the Ring ID
*/
u32 k3_ringacc_get_ring_id(struct k3_ring *ring);
/**
* k3_ringacc_get_ring_irq_num - Get the irq number for the ring
* @ring: pointer on ring
*
* Returns the interrupt number which can be used to request the interrupt
*/
int k3_ringacc_get_ring_irq_num(struct k3_ring *ring);
/**
* k3_ringacc_ring_cfg - ring configure
* @ring: pointer on ring
* @cfg: Ring configuration parameters (see &struct k3_ring_cfg)
*
* Configures ring, including ring memory allocation.
* Returns 0 on success, errno otherwise.
*/
int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg);
/**
* k3_ringacc_ring_get_size - get ring size
* @ring: pointer on ring
*
* Returns ring size in number of elements.
*/
u32 k3_ringacc_ring_get_size(struct k3_ring *ring);
/**
* k3_ringacc_ring_get_free - get free elements
* @ring: pointer on ring
*
* Returns number of free elements in the ring.
*/
u32 k3_ringacc_ring_get_free(struct k3_ring *ring);
/**
* k3_ringacc_ring_get_occ - get ring occupancy
* @ring: pointer on ring
*
* Returns total number of valid entries on the ring
*/
u32 k3_ringacc_ring_get_occ(struct k3_ring *ring);
/**
* k3_ringacc_ring_is_full - checks if ring is full
* @ring: pointer on ring
*
* Returns true if the ring is full
*/
u32 k3_ringacc_ring_is_full(struct k3_ring *ring);
/**
* k3_ringacc_ring_push - push element to the ring tail
* @ring: pointer on ring
* @elem: pointer on ring element buffer
*
* Push one ring element to the ring tail. Size of the ring element is
* determined by ring configuration &struct k3_ring_cfg elm_size.
*
* Returns 0 on success, errno otherwise.
*/
int k3_ringacc_ring_push(struct k3_ring *ring, void *elem);
/**
* k3_ringacc_ring_pop - pop element from the ring head
* @ring: pointer on ring
* @elem: pointer on ring element buffer
*
* Push one ring element from the ring head. Size of the ring element is
* determined by ring configuration &struct k3_ring_cfg elm_size..
*
* Returns 0 on success, errno otherwise.
*/
int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem);
/**
* k3_ringacc_ring_push_head - push element to the ring head
* @ring: pointer on ring
* @elem: pointer on ring element buffer
*
* Push one ring element to the ring head. Size of the ring element is
* determined by ring configuration &struct k3_ring_cfg elm_size.
*
* Returns 0 on success, errno otherwise.
* Not Supported by ring modes: K3_RINGACC_RING_MODE_RING
*/
int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem);
/**
* k3_ringacc_ring_pop_tail - pop element from the ring tail
* @ring: pointer on ring
* @elem: pointer on ring element buffer
*
* Push one ring element from the ring tail. Size of the ring element is
* determined by ring configuration &struct k3_ring_cfg elm_size.
*
* Returns 0 on success, errno otherwise.
* Not Supported by ring modes: K3_RINGACC_RING_MODE_RING
*/
int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem);
u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring);
#endif /* __SOC_TI_K3_RINGACC_API_H_ */

228
include/uapi/linux/idxd.h Normal file
View File

@ -0,0 +1,228 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#ifndef _USR_IDXD_H_
#define _USR_IDXD_H_
#ifdef __KERNEL__
#include <linux/types.h>
#else
#include <stdint.h>
#endif
/* Descriptor flags */
#define IDXD_OP_FLAG_FENCE 0x0001
#define IDXD_OP_FLAG_BOF 0x0002
#define IDXD_OP_FLAG_CRAV 0x0004
#define IDXD_OP_FLAG_RCR 0x0008
#define IDXD_OP_FLAG_RCI 0x0010
#define IDXD_OP_FLAG_CRSTS 0x0020
#define IDXD_OP_FLAG_CR 0x0080
#define IDXD_OP_FLAG_CC 0x0100
#define IDXD_OP_FLAG_ADDR1_TCS 0x0200
#define IDXD_OP_FLAG_ADDR2_TCS 0x0400
#define IDXD_OP_FLAG_ADDR3_TCS 0x0800
#define IDXD_OP_FLAG_CR_TCS 0x1000
#define IDXD_OP_FLAG_STORD 0x2000
#define IDXD_OP_FLAG_DRDBK 0x4000
#define IDXD_OP_FLAG_DSTS 0x8000
/* Opcode */
enum dsa_opcode {
DSA_OPCODE_NOOP = 0,
DSA_OPCODE_BATCH,
DSA_OPCODE_DRAIN,
DSA_OPCODE_MEMMOVE,
DSA_OPCODE_MEMFILL,
DSA_OPCODE_COMPARE,
DSA_OPCODE_COMPVAL,
DSA_OPCODE_CR_DELTA,
DSA_OPCODE_AP_DELTA,
DSA_OPCODE_DUALCAST,
DSA_OPCODE_CRCGEN = 0x10,
DSA_OPCODE_COPY_CRC,
DSA_OPCODE_DIF_CHECK,
DSA_OPCODE_DIF_INS,
DSA_OPCODE_DIF_STRP,
DSA_OPCODE_DIF_UPDT,
DSA_OPCODE_CFLUSH = 0x20,
};
/* Completion record status */
enum dsa_completion_status {
DSA_COMP_NONE = 0,
DSA_COMP_SUCCESS,
DSA_COMP_SUCCESS_PRED,
DSA_COMP_PAGE_FAULT_NOBOF,
DSA_COMP_PAGE_FAULT_IR,
DSA_COMP_BATCH_FAIL,
DSA_COMP_BATCH_PAGE_FAULT,
DSA_COMP_DR_OFFSET_NOINC,
DSA_COMP_DR_OFFSET_ERANGE,
DSA_COMP_DIF_ERR,
DSA_COMP_BAD_OPCODE = 0x10,
DSA_COMP_INVALID_FLAGS,
DSA_COMP_NOZERO_RESERVE,
DSA_COMP_XFER_ERANGE,
DSA_COMP_DESC_CNT_ERANGE,
DSA_COMP_DR_ERANGE,
DSA_COMP_OVERLAP_BUFFERS,
DSA_COMP_DCAST_ERR,
DSA_COMP_DESCLIST_ALIGN,
DSA_COMP_INT_HANDLE_INVAL,
DSA_COMP_CRA_XLAT,
DSA_COMP_CRA_ALIGN,
DSA_COMP_ADDR_ALIGN,
DSA_COMP_PRIV_BAD,
DSA_COMP_TRAFFIC_CLASS_CONF,
DSA_COMP_PFAULT_RDBA,
DSA_COMP_HW_ERR1,
DSA_COMP_HW_ERR_DRB,
DSA_COMP_TRANSLATION_FAIL,
};
#define DSA_COMP_STATUS_MASK 0x7f
#define DSA_COMP_STATUS_WRITE 0x80
struct dsa_batch_desc {
uint32_t pasid:20;
uint32_t rsvd:11;
uint32_t priv:1;
uint32_t flags:24;
uint32_t opcode:8;
uint64_t completion_addr;
uint64_t desc_list_addr;
uint64_t rsvd1;
uint32_t desc_count;
uint16_t interrupt_handle;
uint16_t rsvd2;
uint8_t rsvd3[24];
} __attribute__((packed));
struct dsa_hw_desc {
uint32_t pasid:20;
uint32_t rsvd:11;
uint32_t priv:1;
uint32_t flags:24;
uint32_t opcode:8;
uint64_t completion_addr;
union {
uint64_t src_addr;
uint64_t rdback_addr;
uint64_t pattern;
};
union {
uint64_t dst_addr;
uint64_t rdback_addr2;
uint64_t src2_addr;
uint64_t comp_pattern;
};
uint32_t xfer_size;
uint16_t int_handle;
uint16_t rsvd1;
union {
uint8_t expected_res;
struct {
uint64_t delta_addr;
uint32_t max_delta_size;
};
uint32_t delta_rec_size;
uint64_t dest2;
/* CRC */
struct {
uint32_t crc_seed;
uint32_t crc_rsvd;
uint64_t seed_addr;
};
/* DIF check or strip */
struct {
uint8_t src_dif_flags;
uint8_t dif_chk_res;
uint8_t dif_chk_flags;
uint8_t dif_chk_res2[5];
uint32_t chk_ref_tag_seed;
uint16_t chk_app_tag_mask;
uint16_t chk_app_tag_seed;
};
/* DIF insert */
struct {
uint8_t dif_ins_res;
uint8_t dest_dif_flag;
uint8_t dif_ins_flags;
uint8_t dif_ins_res2[13];
uint32_t ins_ref_tag_seed;
uint16_t ins_app_tag_mask;
uint16_t ins_app_tag_seed;
};
/* DIF update */
struct {
uint8_t src_upd_flags;
uint8_t upd_dest_flags;
uint8_t dif_upd_flags;
uint8_t dif_upd_res[5];
uint32_t src_ref_tag_seed;
uint16_t src_app_tag_mask;
uint16_t src_app_tag_seed;
uint32_t dest_ref_tag_seed;
uint16_t dest_app_tag_mask;
uint16_t dest_app_tag_seed;
};
uint8_t op_specific[24];
};
} __attribute__((packed));
struct dsa_raw_desc {
uint64_t field[8];
} __attribute__((packed));
/*
* The status field will be modified by hardware, therefore it should be
* volatile and prevent the compiler from optimize the read.
*/
struct dsa_completion_record {
volatile uint8_t status;
union {
uint8_t result;
uint8_t dif_status;
};
uint16_t rsvd;
uint32_t bytes_completed;
uint64_t fault_addr;
union {
uint16_t delta_rec_size;
uint16_t crc_val;
/* DIF check & strip */
struct {
uint32_t dif_chk_ref_tag;
uint16_t dif_chk_app_tag_mask;
uint16_t dif_chk_app_tag;
};
/* DIF insert */
struct {
uint64_t dif_ins_res;
uint32_t dif_ins_ref_tag;
uint16_t dif_ins_app_tag_mask;
uint16_t dif_ins_app_tag;
};
/* DIF update */
struct {
uint32_t dif_upd_src_ref_tag;
uint16_t dif_upd_src_app_tag_mask;
uint16_t dif_upd_src_app_tag;
uint32_t dif_upd_dest_ref_tag;
uint16_t dif_upd_dest_app_tag_mask;
uint16_t dif_upd_dest_app_tag;
};
uint8_t op_specific[16];
};
} __attribute__((packed));
struct dsa_raw_completion_record {
uint64_t field[4];
} __attribute__((packed));
#endif