From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Paolo Sabatino Date: Sun, 12 Jan 2025 12:36:50 +0100 Subject: pl330: fix dma engine periodic transfers --- drivers/dma/pl330.c | 277 +++++++--- 1 file changed, 186 insertions(+), 91 deletions(-) diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 111111111111..222222222222 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -239,6 +239,7 @@ enum pl330_byteswap { #define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr)) #define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr)) +#define BYTE_MOD_BURST_LEN(b, ccr) (((b) / BRST_SIZE(ccr)) % BRST_LEN(ccr)) /* * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req @@ -455,9 +456,6 @@ struct dma_pl330_chan { enum dma_data_direction dir; struct dma_slave_config slave_config; - /* for cyclic capability */ - bool cyclic; - /* for runtime pm tracking */ bool active; }; @@ -545,6 +543,10 @@ struct dma_pl330_desc { unsigned peri:5; /* Hook to attach to DMAC's list of reqs with due callback */ struct list_head rqd; + + /* For cyclic capability */ + bool cyclic; + size_t num_periods; }; struct _xfer_spec { @@ -1368,6 +1370,108 @@ static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], return off; } +static int _period(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[], + unsigned long bursts, const struct _xfer_spec *pxs, int ev) +{ + unsigned int lcnt1, ljmp1; + int cyc, off = 0, num_dregs = 0; + struct _arg_LPEND lpend; + struct pl330_xfer *x = &pxs->desc->px; + + if (bursts > 256) { + lcnt1 = 256; + cyc = bursts / 256; + } else { + lcnt1 = bursts; + cyc = 1; + } + + /* loop1 */ + off += _emit_LP(dry_run, &buf[off], 1, lcnt1); + ljmp1 = off; + off += _bursts(pl330, dry_run, &buf[off], pxs, cyc); + lpend.cond = ALWAYS; + lpend.forever = false; + lpend.loop = 1; + lpend.bjump = off - ljmp1; + off += _emit_LPEND(dry_run, &buf[off], &lpend); + + /* remainder */ + lcnt1 = bursts - (lcnt1 * cyc); + + if (lcnt1) { + off += _emit_LP(dry_run, &buf[off], 1, lcnt1); + ljmp1 = off; + off += _bursts(pl330, dry_run, &buf[off], pxs, 1); + lpend.cond = ALWAYS; + lpend.forever = false; + lpend.loop = 1; + lpend.bjump = off - ljmp1; + off += _emit_LPEND(dry_run, &buf[off], &lpend); + } + + num_dregs = BYTE_MOD_BURST_LEN(x->bytes, pxs->ccr); + + if (num_dregs) { + off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs); + off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); + } + + off += _emit_SEV(dry_run, &buf[off], ev); + + return off; +} + +static inline int _loop_cyclic(struct pl330_dmac *pl330, unsigned int dry_run, + u8 buf[], unsigned long bursts, + const struct _xfer_spec *pxs, int ev) +{ + int off, periods, residue, i; + unsigned int lcnt0, ljmp0, ljmpfe; + struct _arg_LPEND lpend; + struct pl330_xfer *x = &pxs->desc->px; + + off = 0; + ljmpfe = off; + lcnt0 = pxs->desc->num_periods; + periods = 1; + + while (lcnt0 > 256) { + periods++; + lcnt0 = pxs->desc->num_periods / periods; + } + + residue = pxs->desc->num_periods % periods; + + /* forever loop */ + off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr); + off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); + + /* loop0 */ + off += _emit_LP(dry_run, &buf[off], 0, lcnt0); + ljmp0 = off; + + for (i = 0; i < periods; i++) + off += _period(pl330, dry_run, &buf[off], bursts, pxs, ev); + + lpend.cond = ALWAYS; + lpend.forever = false; + lpend.loop = 0; + lpend.bjump = off - ljmp0; + off += _emit_LPEND(dry_run, &buf[off], &lpend); + + for (i = 0; i < residue; i++) + off += _period(pl330, dry_run, &buf[off], bursts, pxs, ev); + + lpend.cond = ALWAYS; + lpend.forever = true; + lpend.loop = 1; + lpend.bjump = off - ljmpfe; + off += _emit_LPEND(dry_run, &buf[off], &lpend); + + return off; +} + static inline int _setup_loops(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[], const struct _xfer_spec *pxs) @@ -1407,6 +1511,21 @@ static inline int _setup_xfer(struct pl330_dmac *pl330, return off; } +static inline int _setup_xfer_cyclic(struct pl330_dmac *pl330, + unsigned int dry_run, u8 buf[], + const struct _xfer_spec *pxs, int ev) +{ + struct pl330_xfer *x = &pxs->desc->px; + u32 ccr = pxs->ccr; + unsigned long bursts = BYTE_TO_BURST(x->bytes, ccr); + int off = 0; + + /* Setup Loop(s) */ + off += _loop_cyclic(pl330, dry_run, &buf[off], bursts, pxs, ev); + + return off; +} + /* * A req is a sequence of one or more xfer units. * Returns the number of bytes taken to setup the MC for the req. @@ -1424,12 +1543,17 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run, /* DMAMOV CCR, ccr */ off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); - off += _setup_xfer(pl330, dry_run, &buf[off], pxs); + if (!pxs->desc->cyclic) { + off += _setup_xfer(pl330, dry_run, &buf[off], pxs); - /* DMASEV peripheral/event */ - off += _emit_SEV(dry_run, &buf[off], thrd->ev); - /* DMAEND */ - off += _emit_END(dry_run, &buf[off]); + /* DMASEV peripheral/event */ + off += _emit_SEV(dry_run, &buf[off], thrd->ev); + /* DMAEND */ + off += _emit_END(dry_run, &buf[off]); + } else { + off += _setup_xfer_cyclic(pl330, dry_run, &buf[off], + pxs, thrd->ev); + } return off; } @@ -1703,15 +1827,17 @@ static int pl330_update(struct pl330_dmac *pl330) /* Detach the req */ descdone = thrd->req[active].desc; - thrd->req[active].desc = NULL; - - thrd->req_running = -1; - - /* Get going again ASAP */ - pl330_start_thread(thrd); - - /* For now, just make a list of callbacks to be done */ - list_add_tail(&descdone->rqd, &pl330->req_done); + if (descdone) { + if (!descdone->cyclic) { + thrd->req[active].desc = NULL; + thrd->req_running = -1; + /* Get going again ASAP */ + pl330_start_thread(thrd); + } + + /* For now, just make a list of callbacks to be done */ + list_add_tail(&descdone->rqd, &pl330->req_done); + } } } @@ -2076,12 +2202,25 @@ static void pl330_tasklet(struct tasklet_struct *t) spin_lock_irqsave(&pch->lock, flags); /* Pick up ripe tomatoes */ - list_for_each_entry_safe(desc, _dt, &pch->work_list, node) + list_for_each_entry_safe(desc, _dt, &pch->work_list, node) { if (desc->status == DONE) { - if (!pch->cyclic) + if (!desc->cyclic) { dma_cookie_complete(&desc->txd); - list_move_tail(&desc->node, &pch->completed_list); + list_move_tail(&desc->node, &pch->completed_list); + } else { + struct dmaengine_desc_callback cb; + + desc->status = BUSY; + dmaengine_desc_get_callback(&desc->txd, &cb); + + if (dmaengine_desc_callback_valid(&cb)) { + spin_unlock_irqrestore(&pch->lock, flags); + dmaengine_desc_callback_invoke(&cb, NULL); + spin_lock_irqsave(&pch->lock, flags); + } + } } + } /* Try to submit a req imm. next to the last completed cookie */ fill_queue(pch); @@ -2107,20 +2246,8 @@ static void pl330_tasklet(struct tasklet_struct *t) dmaengine_desc_get_callback(&desc->txd, &cb); - if (pch->cyclic) { - desc->status = PREP; - list_move_tail(&desc->node, &pch->work_list); - if (power_down) { - pch->active = true; - spin_lock(&pch->thread->dmac->lock); - pl330_start_thread(pch->thread); - spin_unlock(&pch->thread->dmac->lock); - power_down = false; - } - } else { - desc->status = FREE; - list_move_tail(&desc->node, &pch->dmac->desc_pool); - } + desc->status = FREE; + list_move_tail(&desc->node, &pch->dmac->desc_pool); dma_descriptor_unmap(&desc->txd); @@ -2168,7 +2295,6 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan) spin_lock_irqsave(&pl330->lock, flags); dma_cookie_init(chan); - pch->cyclic = false; pch->thread = pl330_request_channel(pl330); if (!pch->thread) { @@ -2367,8 +2493,7 @@ static void pl330_free_chan_resources(struct dma_chan *chan) pl330_release_channel(pch->thread); pch->thread = NULL; - if (pch->cyclic) - list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); + list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); spin_unlock_irqrestore(&pl330->lock, flags); pm_runtime_mark_last_busy(pch->dmac->ddma.dev); @@ -2431,7 +2556,7 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, /* Check in pending list */ list_for_each_entry(desc, &pch->work_list, node) { - if (desc->status == DONE) + if (desc->status == DONE && !desc->cyclic) transferred = desc->bytes_requested; else if (running && desc == running) transferred = @@ -2516,10 +2641,7 @@ static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) /* Assign cookies to all nodes */ while (!list_empty(&last->node)) { desc = list_entry(last->node.next, struct dma_pl330_desc, node); - if (pch->cyclic) { - desc->txd.callback = last->txd.callback; - desc->txd.callback_param = last->txd.callback_param; - } + desc->last = false; dma_cookie_assign(&desc->txd); @@ -2622,6 +2744,9 @@ static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) desc->peri = peri_id ? pch->chan.chan_id : 0; desc->rqcfg.pcfg = &pch->dmac->pcfg; + desc->cyclic = false; + desc->num_periods = 1; + dma_async_tx_descriptor_init(&desc->txd, &pch->chan); return desc; @@ -2685,12 +2810,10 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( size_t period_len, enum dma_transfer_direction direction, unsigned long flags) { - struct dma_pl330_desc *desc = NULL, *first = NULL; + struct dma_pl330_desc *desc = NULL; struct dma_pl330_chan *pch = to_pchan(chan); - struct pl330_dmac *pl330 = pch->dmac; - unsigned int i; - dma_addr_t dst; - dma_addr_t src; + dma_addr_t dst = 0; + dma_addr_t src = 0; if (len % period_len != 0) return NULL; @@ -2706,33 +2829,14 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( if (!pl330_prep_slave_fifo(pch, direction)) return NULL; - for (i = 0; i < len / period_len; i++) { - desc = pl330_get_desc(pch); - if (!desc) { - unsigned long iflags; - - dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", - __func__, __LINE__); - - if (!first) - return NULL; - - spin_lock_irqsave(&pl330->pool_lock, iflags); - - while (!list_empty(&first->node)) { - desc = list_entry(first->node.next, - struct dma_pl330_desc, node); - list_move_tail(&desc->node, &pl330->desc_pool); - } - - list_move_tail(&first->node, &pl330->desc_pool); - - spin_unlock_irqrestore(&pl330->pool_lock, iflags); - - return NULL; - } + desc = pl330_get_desc(pch); + if (!desc) { + dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", + __func__, __LINE__); + return NULL; + } - switch (direction) { + switch (direction) { case DMA_MEM_TO_DEV: desc->rqcfg.src_inc = 1; desc->rqcfg.dst_inc = 0; @@ -2746,27 +2850,18 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( dst = dma_addr; break; default: - break; - } - - desc->rqtype = direction; - desc->rqcfg.brst_size = pch->burst_sz; - desc->rqcfg.brst_len = pch->burst_len; - desc->bytes_requested = period_len; - fill_px(&desc->px, dst, src, period_len); - - if (!first) - first = desc; - else - list_add_tail(&desc->node, &first->node); - - dma_addr += period_len; + break; } - if (!desc) - return NULL; + desc->rqtype = direction; + desc->rqcfg.brst_size = pch->burst_sz; + desc->rqcfg.brst_len = pch->burst_len; + desc->bytes_requested = len; + fill_px(&desc->px, dst, src, period_len); - pch->cyclic = true; + desc->cyclic = true; + desc->num_periods = len / period_len; + desc->txd.flags = flags; return &desc->txd; } -- Armbian