host1x/hw/cdma_hw.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Tegra host1x Command DMA
0004  *
0005  * Copyright (c) 2010-2013, NVIDIA Corporation.
0006  */
0007
0008 #include <linux/slab.h>
0009 #include <linux/scatterlist.h>
0010 #include <linux/dma-mapping.h>
0011
0012 #include "../cdma.h"
0013 #include "../channel.h"
0014 #include "../dev.h"
0015 #include "../debug.h"
0016
0017 /*
0018  * Put the restart at the end of pushbuffer memory
0019  */
0020 static void push_buffer_init(struct push_buffer *pb)
0021 {
0022     *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
0023 }
0024
0025 /*
0026  * Increment timedout buffer's syncpt via CPU.
0027  */
0028 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
0029                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
0030 {
0031     unsigned int i;
0032
0033     for (i = 0; i < syncpt_incrs; i++)
0034         host1x_syncpt_incr(cdma->timeout.syncpt);
0035
0036     /* after CPU incr, ensure shadow is up to date */
0037     host1x_syncpt_load(cdma->timeout.syncpt);
0038 }
0039
0040 /*
0041  * Start channel DMA
0042  */
0043 static void cdma_start(struct host1x_cdma *cdma)
0044 {
0045     struct host1x_channel *ch = cdma_to_channel(cdma);
0046     u64 start, end;
0047
0048     if (cdma->running)
0049         return;
0050
0051     cdma->last_pos = cdma->push_buffer.pos;
0052     start = cdma->push_buffer.dma;
0053     end = cdma->push_buffer.size + 4;
0054
0055     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
0056              HOST1X_CHANNEL_DMACTRL);
0057
0058     /* set base, put and end pointer */
0059     host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
0060 #if HOST1X_HW >= 6
0061     host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
0062 #endif
0063     host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
0064 #if HOST1X_HW >= 6
0065     host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
0066 #endif
0067     host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
0068 #if HOST1X_HW >= 6
0069     host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
0070 #endif
0071
0072     /* reset GET */
0073     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
0074              HOST1X_CHANNEL_DMACTRL_DMAGETRST |
0075              HOST1X_CHANNEL_DMACTRL_DMAINITGET,
0076              HOST1X_CHANNEL_DMACTRL);
0077
0078     /* start the command DMA */
0079     host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
0080
0081     cdma->running = true;
0082 }
0083
0084 /*
0085  * Similar to cdma_start(), but rather than starting from an idle
0086  * state (where DMA GET is set to DMA PUT), on a timeout we restore
0087  * DMA GET from an explicit value (so DMA may again be pending).
0088  */
0089 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
0090 {
0091     struct host1x *host1x = cdma_to_host1x(cdma);
0092     struct host1x_channel *ch = cdma_to_channel(cdma);
0093     u64 start, end;
0094
0095     if (cdma->running)
0096         return;
0097
0098     cdma->last_pos = cdma->push_buffer.pos;
0099
0100     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
0101              HOST1X_CHANNEL_DMACTRL);
0102
0103     start = cdma->push_buffer.dma;
0104     end = cdma->push_buffer.size + 4;
0105
0106     /* set base, end pointer (all of memory) */
0107     host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
0108 #if HOST1X_HW >= 6
0109     host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
0110 #endif
0111     host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
0112 #if HOST1X_HW >= 6
0113     host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
0114 #endif
0115
0116     /* set GET, by loading the value in PUT (then reset GET) */
0117     host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
0118     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
0119              HOST1X_CHANNEL_DMACTRL_DMAGETRST |
0120              HOST1X_CHANNEL_DMACTRL_DMAINITGET,
0121              HOST1X_CHANNEL_DMACTRL);
0122
0123     dev_dbg(host1x->dev,
0124         "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
0125         host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
0126         host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
0127         cdma->last_pos);
0128
0129     /* deassert GET reset and set PUT */
0130     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
0131              HOST1X_CHANNEL_DMACTRL);
0132     host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
0133
0134     /* start the command DMA */
0135     host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
0136
0137     cdma->running = true;
0138 }
0139
0140 /*
0141  * Kick channel DMA into action by writing its PUT offset (if it has changed)
0142  */
0143 static void cdma_flush(struct host1x_cdma *cdma)
0144 {
0145     struct host1x_channel *ch = cdma_to_channel(cdma);
0146
0147     if (cdma->push_buffer.pos != cdma->last_pos) {
0148         host1x_ch_writel(ch, cdma->push_buffer.pos,
0149                  HOST1X_CHANNEL_DMAPUT);
0150         cdma->last_pos = cdma->push_buffer.pos;
0151     }
0152 }
0153
0154 static void cdma_stop(struct host1x_cdma *cdma)
0155 {
0156     struct host1x_channel *ch = cdma_to_channel(cdma);
0157
0158     mutex_lock(&cdma->lock);
0159
0160     if (cdma->running) {
0161         host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
0162         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
0163                  HOST1X_CHANNEL_DMACTRL);
0164         cdma->running = false;
0165     }
0166
0167     mutex_unlock(&cdma->lock);
0168 }
0169
0170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
0171                  bool stop)
0172 {
0173 #if HOST1X_HW >= 6
0174     host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
0175 #else
0176     u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
0177     if (stop)
0178         cmdproc_stop |= BIT(ch->id);
0179     else
0180         cmdproc_stop &= ~BIT(ch->id);
0181     host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
0182 #endif
0183 }
0184
0185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
0186 {
0187 #if HOST1X_HW >= 6
0188     host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
0189 #else
0190     host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
0191 #endif
0192 }
0193
0194 /*
0195  * Stops both channel's command processor and CDMA immediately.
0196  * Also, tears down the channel and resets corresponding module.
0197  */
0198 static void cdma_freeze(struct host1x_cdma *cdma)
0199 {
0200     struct host1x *host = cdma_to_host1x(cdma);
0201     struct host1x_channel *ch = cdma_to_channel(cdma);
0202
0203     if (cdma->torndown && !cdma->running) {
0204         dev_warn(host->dev, "Already torn down\n");
0205         return;
0206     }
0207
0208     dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
0209
0210     cdma_hw_cmdproc_stop(host, ch, true);
0211
0212     dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
0213         __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
0214         host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
0215         cdma->last_pos);
0216
0217     host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
0218              HOST1X_CHANNEL_DMACTRL);
0219
0220     cdma_hw_teardown(host, ch);
0221
0222     cdma->running = false;
0223     cdma->torndown = true;
0224 }
0225
0226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
0227 {
0228     struct host1x *host1x = cdma_to_host1x(cdma);
0229     struct host1x_channel *ch = cdma_to_channel(cdma);
0230
0231     dev_dbg(host1x->dev,
0232         "resuming channel (id %u, DMAGET restart = 0x%x)\n",
0233         ch->id, getptr);
0234
0235     cdma_hw_cmdproc_stop(host1x, ch, false);
0236
0237     cdma->torndown = false;
0238     cdma_timeout_restart(cdma, getptr);
0239 }
0240
0241 static void timeout_release_mlock(struct host1x_cdma *cdma)
0242 {
0243 #if HOST1X_HW >= 8
0244     /* Tegra186 and Tegra194 require a more complicated MLOCK release
0245      * sequence. Furthermore, those chips by default don't enforce MLOCKs,
0246      * so it turns out that if we don't /actually/ need MLOCKs, we can just
0247      * ignore them.
0248      *
0249      * As such, for now just implement this on Tegra234 where things are
0250      * stricter but also easy to implement.
0251      */
0252     struct host1x_channel *ch = cdma_to_channel(cdma);
0253     struct host1x *host1x = cdma_to_host1x(cdma);
0254     u32 offset;
0255
0256     switch (ch->client->class) {
0257     case HOST1X_CLASS_VIC:
0258         offset = HOST1X_COMMON_VIC_MLOCK;
0259         break;
0260     case HOST1X_CLASS_NVDEC:
0261         offset = HOST1X_COMMON_NVDEC_MLOCK;
0262         break;
0263     default:
0264         WARN(1, "%s was not updated for class %u", __func__, ch->client->class);
0265         return;
0266     }
0267
0268     host1x_common_writel(host1x, 0x0, offset);
0269 #endif
0270 }
0271
0272 /*
0273  * If this timeout fires, it indicates the current sync_queue entry has
0274  * exceeded its TTL and the userctx should be timed out and remaining
0275  * submits already issued cleaned up (future submits return an error).
0276  */
0277 static void cdma_timeout_handler(struct work_struct *work)
0278 {
0279     u32 syncpt_val;
0280     struct host1x_cdma *cdma;
0281     struct host1x *host1x;
0282     struct host1x_channel *ch;
0283
0284     cdma = container_of(to_delayed_work(work), struct host1x_cdma,
0285                 timeout.wq);
0286     host1x = cdma_to_host1x(cdma);
0287     ch = cdma_to_channel(cdma);
0288
0289     host1x_debug_dump(cdma_to_host1x(cdma));
0290
0291     mutex_lock(&cdma->lock);
0292
0293     if (!cdma->timeout.client) {
0294         dev_dbg(host1x->dev,
0295             "cdma_timeout: expired, but has no clientid\n");
0296         mutex_unlock(&cdma->lock);
0297         return;
0298     }
0299
0300     /* stop processing to get a clean snapshot */
0301     cdma_hw_cmdproc_stop(host1x, ch, true);
0302
0303     syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
0304
0305     /* has buffer actually completed? */
0306     if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
0307         dev_dbg(host1x->dev,
0308             "cdma_timeout: expired, but buffer had completed\n");
0309         /* restore */
0310         cdma_hw_cmdproc_stop(host1x, ch, false);
0311         mutex_unlock(&cdma->lock);
0312         return;
0313     }
0314
0315     dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
0316          __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
0317          syncpt_val, cdma->timeout.syncpt_val);
0318
0319     /* stop HW, resetting channel/module */
0320     host1x_hw_cdma_freeze(host1x, cdma);
0321
0322     /* release any held MLOCK */
0323     timeout_release_mlock(cdma);
0324
0325     host1x_cdma_update_sync_queue(cdma, ch->dev);
0326     mutex_unlock(&cdma->lock);
0327 }
0328
0329 /*
0330  * Init timeout resources
0331  */
0332 static int cdma_timeout_init(struct host1x_cdma *cdma)
0333 {
0334     INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
0335     cdma->timeout.initialized = true;
0336
0337     return 0;
0338 }
0339
0340 /*
0341  * Clean up timeout resources
0342  */
0343 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
0344 {
0345     if (cdma->timeout.initialized)
0346         cancel_delayed_work(&cdma->timeout.wq);
0347
0348     cdma->timeout.initialized = false;
0349 }
0350
0351 static const struct host1x_cdma_ops host1x_cdma_ops = {
0352     .start = cdma_start,
0353     .stop = cdma_stop,
0354     .flush = cdma_flush,
0355
0356     .timeout_init = cdma_timeout_init,
0357     .timeout_destroy = cdma_timeout_destroy,
0358     .freeze = cdma_freeze,
0359     .resume = cdma_resume,
0360     .timeout_cpu_incr = cdma_timeout_cpu_incr,
0361 };
0362
0363 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
0364     .init = push_buffer_init,
0365 };