mirror of
https://github.com/edk2-porting/linux-next.git
synced 2024-12-22 12:14:01 +08:00
mtd: nand: sunxi: add support for DMA assisted operations
The sunxi NAND controller is able to pipeline ECC operations only when operated in DMA mode, which improves a lot NAND throughput while keeping CPU usage low. Signed-off-by: Boris Brezillon <boris.brezillon@free-electrons.com>
This commit is contained in:
Boris Brezillon
parent
decba6d478
commit
614049a8d9
@ -153,6 +153,7 @@
|
||||
|
||||
/* define bit use in NFC_ECC_ST */
|
||||
#define NFC_ECC_ERR(x) BIT(x)
|
||||
#define NFC_ECC_ERR_MSK GENMASK(15, 0)
|
||||
#define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
|
||||
#define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
|
||||
|
||||
@ -273,6 +274,7 @@ struct sunxi_nfc {
|
||||
unsigned long clk_rate;
|
||||
struct list_head chips;
|
||||
struct completion complete;
|
||||
struct dma_chan *dmac;
|
||||
};
|
||||
|
||||
static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
|
||||
@ -365,6 +367,67 @@ static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
|
||||
int chunksize, int nchunks,
|
||||
enum dma_data_direction ddir,
|
||||
struct scatterlist *sg)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
|
||||
struct dma_async_tx_descriptor *dmad;
|
||||
enum dma_transfer_direction tdir;
|
||||
dma_cookie_t dmat;
|
||||
int ret;
|
||||
|
||||
if (ddir == DMA_FROM_DEVICE)
|
||||
tdir = DMA_DEV_TO_MEM;
|
||||
else
|
||||
tdir = DMA_MEM_TO_DEV;
|
||||
|
||||
sg_init_one(sg, buf, nchunks * chunksize);
|
||||
ret = dma_map_sg(nfc->dev, sg, 1, ddir);
|
||||
if (!ret)
|
||||
return -ENOMEM;
|
||||
|
||||
dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
|
||||
if (IS_ERR(dmad)) {
|
||||
ret = PTR_ERR(dmad);
|
||||
goto err_unmap_buf;
|
||||
}
|
||||
|
||||
writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
|
||||
nfc->regs + NFC_REG_CTL);
|
||||
writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
|
||||
writel(chunksize, nfc->regs + NFC_REG_CNT);
|
||||
dmat = dmaengine_submit(dmad);
|
||||
|
||||
ret = dma_submit_error(dmat);
|
||||
if (ret)
|
||||
goto err_clr_dma_flag;
|
||||
|
||||
return 0;
|
||||
|
||||
err_clr_dma_flag:
|
||||
writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
|
||||
nfc->regs + NFC_REG_CTL);
|
||||
|
||||
err_unmap_buf:
|
||||
dma_unmap_sg(nfc->dev, sg, 1, ddir);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
|
||||
enum dma_data_direction ddir,
|
||||
struct scatterlist *sg)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
|
||||
|
||||
dma_unmap_sg(nfc->dev, sg, 1, ddir);
|
||||
writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
|
||||
nfc->regs + NFC_REG_CTL);
|
||||
}
|
||||
|
||||
static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
@ -822,17 +885,15 @@ static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
|
||||
int step, bool *erased)
|
||||
int step, u32 status, bool *erased)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
|
||||
struct nand_ecc_ctrl *ecc = &nand->ecc;
|
||||
u32 status, tmp;
|
||||
u32 tmp;
|
||||
|
||||
*erased = false;
|
||||
|
||||
status = readl(nfc->regs + NFC_REG_ECC_ST);
|
||||
|
||||
if (status & NFC_ECC_ERR(step))
|
||||
return -EBADMSG;
|
||||
|
||||
@ -898,6 +959,7 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
|
||||
*cur_off = oob_off + ecc->bytes + 4;
|
||||
|
||||
ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
|
||||
readl(nfc->regs + NFC_REG_ECC_ST),
|
||||
&erased);
|
||||
if (erased)
|
||||
return 1;
|
||||
@ -967,6 +1029,128 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
|
||||
*cur_off = mtd->oobsize + mtd->writesize;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
|
||||
int oob_required, int page,
|
||||
int nchunks)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
bool randomized = nand->options & NAND_NEED_SCRAMBLING;
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
|
||||
struct nand_ecc_ctrl *ecc = &nand->ecc;
|
||||
unsigned int max_bitflips = 0;
|
||||
int ret, i, raw_mode = 0;
|
||||
struct scatterlist sg;
|
||||
u32 status;
|
||||
|
||||
ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
|
||||
DMA_FROM_DEVICE, &sg);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
sunxi_nfc_hw_ecc_enable(mtd);
|
||||
sunxi_nfc_randomizer_config(mtd, page, false);
|
||||
sunxi_nfc_randomizer_enable(mtd);
|
||||
|
||||
writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
|
||||
NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
|
||||
|
||||
dma_async_issue_pending(nfc->dmac);
|
||||
|
||||
writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
|
||||
nfc->regs + NFC_REG_CMD);
|
||||
|
||||
ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
|
||||
if (ret)
|
||||
dmaengine_terminate_all(nfc->dmac);
|
||||
|
||||
sunxi_nfc_randomizer_disable(mtd);
|
||||
sunxi_nfc_hw_ecc_disable(mtd);
|
||||
|
||||
sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
|
||||
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
status = readl(nfc->regs + NFC_REG_ECC_ST);
|
||||
|
||||
for (i = 0; i < nchunks; i++) {
|
||||
int data_off = i * ecc->size;
|
||||
int oob_off = i * (ecc->bytes + 4);
|
||||
u8 *data = buf + data_off;
|
||||
u8 *oob = nand->oob_poi + oob_off;
|
||||
bool erased;
|
||||
|
||||
ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
|
||||
oob_required ? oob : NULL,
|
||||
i, status, &erased);
|
||||
|
||||
/* ECC errors are handled in the second loop. */
|
||||
if (ret < 0)
|
||||
continue;
|
||||
|
||||
if (oob_required && !erased) {
|
||||
/* TODO: use DMA to retrieve OOB */
|
||||
nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
|
||||
nand->read_buf(mtd, oob, ecc->bytes + 4);
|
||||
|
||||
sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
|
||||
!i, page);
|
||||
}
|
||||
|
||||
if (erased)
|
||||
raw_mode = 1;
|
||||
|
||||
sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
|
||||
}
|
||||
|
||||
if (status & NFC_ECC_ERR_MSK) {
|
||||
for (i = 0; i < nchunks; i++) {
|
||||
int data_off = i * ecc->size;
|
||||
int oob_off = i * (ecc->bytes + 4);
|
||||
u8 *data = buf + data_off;
|
||||
u8 *oob = nand->oob_poi + oob_off;
|
||||
|
||||
if (!(status & NFC_ECC_ERR(i)))
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Re-read the data with the randomizer disabled to
|
||||
* identify bitflips in erased pages.
|
||||
*/
|
||||
if (randomized) {
|
||||
/* TODO: use DMA to read page in raw mode */
|
||||
nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
|
||||
data_off, -1);
|
||||
nand->read_buf(mtd, data, ecc->size);
|
||||
}
|
||||
|
||||
/* TODO: use DMA to retrieve OOB */
|
||||
nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
|
||||
nand->read_buf(mtd, oob, ecc->bytes + 4);
|
||||
|
||||
ret = nand_check_erased_ecc_chunk(data, ecc->size,
|
||||
oob, ecc->bytes + 4,
|
||||
NULL, 0,
|
||||
ecc->strength);
|
||||
if (ret >= 0)
|
||||
raw_mode = 1;
|
||||
|
||||
sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
|
||||
}
|
||||
}
|
||||
|
||||
if (oob_required)
|
||||
sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
|
||||
NULL, !raw_mode,
|
||||
page);
|
||||
|
||||
return max_bitflips;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
|
||||
const u8 *data, int data_off,
|
||||
const u8 *oob, int oob_off,
|
||||
@ -1065,6 +1249,23 @@ static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
|
||||
return max_bitflips;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
|
||||
struct nand_chip *chip, u8 *buf,
|
||||
int oob_required, int page)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
|
||||
chip->ecc.steps);
|
||||
if (ret >= 0)
|
||||
return ret;
|
||||
|
||||
/* Fallback to PIO mode */
|
||||
chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
|
||||
|
||||
return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
|
||||
struct nand_chip *chip,
|
||||
u32 data_offs, u32 readlen,
|
||||
@ -1098,6 +1299,25 @@ static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
|
||||
return max_bitflips;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
|
||||
struct nand_chip *chip,
|
||||
u32 data_offs, u32 readlen,
|
||||
u8 *buf, int page)
|
||||
{
|
||||
int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
|
||||
int ret;
|
||||
|
||||
ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
|
||||
if (ret >= 0)
|
||||
return ret;
|
||||
|
||||
/* Fallback to PIO mode */
|
||||
chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
|
||||
|
||||
return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
|
||||
buf, page);
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
|
||||
struct nand_chip *chip,
|
||||
const uint8_t *buf, int oob_required,
|
||||
@ -1130,6 +1350,69 @@ static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
|
||||
struct nand_chip *chip,
|
||||
const u8 *buf,
|
||||
int oob_required,
|
||||
int page)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
|
||||
struct nand_ecc_ctrl *ecc = &nand->ecc;
|
||||
struct scatterlist sg;
|
||||
int ret, i;
|
||||
|
||||
ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
|
||||
DMA_TO_DEVICE, &sg);
|
||||
if (ret)
|
||||
goto pio_fallback;
|
||||
|
||||
for (i = 0; i < ecc->steps; i++) {
|
||||
const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
|
||||
|
||||
sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
|
||||
}
|
||||
|
||||
sunxi_nfc_hw_ecc_enable(mtd);
|
||||
sunxi_nfc_randomizer_config(mtd, page, false);
|
||||
sunxi_nfc_randomizer_enable(mtd);
|
||||
|
||||
writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
|
||||
nfc->regs + NFC_REG_RCMD_SET);
|
||||
|
||||
dma_async_issue_pending(nfc->dmac);
|
||||
|
||||
writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
|
||||
NFC_DATA_TRANS | NFC_ACCESS_DIR,
|
||||
nfc->regs + NFC_REG_CMD);
|
||||
|
||||
ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
|
||||
if (ret)
|
||||
dmaengine_terminate_all(nfc->dmac);
|
||||
|
||||
sunxi_nfc_randomizer_disable(mtd);
|
||||
sunxi_nfc_hw_ecc_disable(mtd);
|
||||
|
||||
sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
|
||||
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
|
||||
/* TODO: use DMA to transfer extra OOB bytes ? */
|
||||
sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
|
||||
NULL, page);
|
||||
|
||||
return 0;
|
||||
|
||||
pio_fallback:
|
||||
return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
|
||||
}
|
||||
|
||||
static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
|
||||
struct nand_chip *chip,
|
||||
uint8_t *buf, int oob_required,
|
||||
@ -1550,14 +1833,27 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
|
||||
struct nand_ecc_ctrl *ecc,
|
||||
struct device_node *np)
|
||||
{
|
||||
struct nand_chip *nand = mtd_to_nand(mtd);
|
||||
struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
|
||||
struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
|
||||
int ret;
|
||||
|
||||
ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ecc->read_page = sunxi_nfc_hw_ecc_read_page;
|
||||
ecc->write_page = sunxi_nfc_hw_ecc_write_page;
|
||||
if (nfc->dmac) {
|
||||
ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
|
||||
ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
|
||||
ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
|
||||
nand->options |= NAND_USE_BOUNCE_BUFFER;
|
||||
} else {
|
||||
ecc->read_page = sunxi_nfc_hw_ecc_read_page;
|
||||
ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
|
||||
ecc->write_page = sunxi_nfc_hw_ecc_write_page;
|
||||
}
|
||||
|
||||
/* TODO: support DMA for raw accesses */
|
||||
ecc->read_oob_raw = nand_read_oob_std;
|
||||
ecc->write_oob_raw = nand_write_oob_std;
|
||||
ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
|
||||
@ -1881,16 +2177,34 @@ static int sunxi_nfc_probe(struct platform_device *pdev)
|
||||
if (ret)
|
||||
goto out_mod_clk_unprepare;
|
||||
|
||||
nfc->dmac = dma_request_slave_channel(dev, "rxtx");
|
||||
if (nfc->dmac) {
|
||||
struct dma_slave_config dmac_cfg = { };
|
||||
|
||||
dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
|
||||
dmac_cfg.dst_addr = dmac_cfg.src_addr;
|
||||
dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
|
||||
dmac_cfg.src_maxburst = 4;
|
||||
dmac_cfg.dst_maxburst = 4;
|
||||
dmaengine_slave_config(nfc->dmac, &dmac_cfg);
|
||||
} else {
|
||||
dev_warn(dev, "failed to request rxtx DMA channel\n");
|
||||
}
|
||||
|
||||
platform_set_drvdata(pdev, nfc);
|
||||
|
||||
ret = sunxi_nand_chips_init(dev, nfc);
|
||||
if (ret) {
|
||||
dev_err(dev, "failed to init nand chips\n");
|
||||
goto out_mod_clk_unprepare;
|
||||
goto out_release_dmac;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
out_release_dmac:
|
||||
if (nfc->dmac)
|
||||
dma_release_channel(nfc->dmac);
|
||||
out_mod_clk_unprepare:
|
||||
clk_disable_unprepare(nfc->mod_clk);
|
||||
out_ahb_clk_unprepare:
|
||||
@ -1904,6 +2218,8 @@ static int sunxi_nfc_remove(struct platform_device *pdev)
|
||||
struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
|
||||
|
||||
sunxi_nand_chips_cleanup(nfc);
|
||||
if (nfc->dmac)
|
||||
dma_release_channel(nfc->dmac);
|
||||
clk_disable_unprepare(nfc->mod_clk);
|
||||
clk_disable_unprepare(nfc->ahb_clk);
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user