host1x/hw/channel_hw.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Tegra host1x Channel
0004  *
0005  * Copyright (c) 2010-2013, NVIDIA Corporation.
0006  */
0007
0008 #include <linux/host1x.h>
0009 #include <linux/iommu.h>
0010 #include <linux/slab.h>
0011
0012 #include <trace/events/host1x.h>
0013
0014 #include "../channel.h"
0015 #include "../dev.h"
0016 #include "../intr.h"
0017 #include "../job.h"
0018
0019 #define TRACE_MAX_LENGTH 128U
0020
0021 static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo,
0022                    u32 offset, u32 words)
0023 {
0024     struct device *dev = cdma_to_channel(cdma)->dev;
0025     void *mem = NULL;
0026
0027     if (host1x_debug_trace_cmdbuf)
0028         mem = host1x_bo_mmap(bo);
0029
0030     if (mem) {
0031         u32 i;
0032         /*
0033          * Write in batches of 128 as there seems to be a limit
0034          * of how much you can output to ftrace at once.
0035          */
0036         for (i = 0; i < words; i += TRACE_MAX_LENGTH) {
0037             u32 num_words = min(words - i, TRACE_MAX_LENGTH);
0038
0039             offset += i * sizeof(u32);
0040
0041             trace_host1x_cdma_push_gather(dev_name(dev), bo,
0042                               num_words, offset,
0043                               mem);
0044         }
0045
0046         host1x_bo_munmap(bo, mem);
0047     }
0048 }
0049
0050 static void submit_wait(struct host1x_job *job, u32 id, u32 threshold,
0051             u32 next_class)
0052 {
0053     struct host1x_cdma *cdma = &job->channel->cdma;
0054
0055 #if HOST1X_HW >= 6
0056     u32 stream_id;
0057
0058     /*
0059      * If a memory context has been set, use it. Otherwise
0060      * (if context isolation is disabled) use the engine's
0061      * firmware stream ID.
0062      */
0063     if (job->memory_context)
0064         stream_id = job->memory_context->stream_id;
0065     else
0066         stream_id = job->engine_fallback_streamid;
0067
0068     host1x_cdma_push_wide(cdma,
0069         host1x_opcode_setclass(
0070             HOST1X_CLASS_HOST1X,
0071             HOST1X_UCLASS_LOAD_SYNCPT_PAYLOAD_32,
0072             /* WAIT_SYNCPT_32 is at SYNCPT_PAYLOAD_32+2 */
0073             BIT(0) | BIT(2)
0074         ),
0075         threshold,
0076         id,
0077         HOST1X_OPCODE_NOP
0078     );
0079     host1x_cdma_push_wide(&job->channel->cdma,
0080         host1x_opcode_setclass(job->class, 0, 0),
0081         host1x_opcode_setpayload(stream_id),
0082         host1x_opcode_setstreamid(job->engine_streamid_offset / 4),
0083         HOST1X_OPCODE_NOP);
0084 #elif HOST1X_HW >= 2
0085     host1x_cdma_push_wide(cdma,
0086         host1x_opcode_setclass(
0087             HOST1X_CLASS_HOST1X,
0088             HOST1X_UCLASS_LOAD_SYNCPT_PAYLOAD_32,
0089             /* WAIT_SYNCPT_32 is at SYNCPT_PAYLOAD_32+2 */
0090             BIT(0) | BIT(2)
0091         ),
0092         threshold,
0093         id,
0094         host1x_opcode_setclass(next_class, 0, 0)
0095     );
0096 #else
0097     /* TODO add waitchk or use waitbases or other mitigation */
0098     host1x_cdma_push(cdma,
0099         host1x_opcode_setclass(
0100             HOST1X_CLASS_HOST1X,
0101             host1x_uclass_wait_syncpt_r(),
0102             BIT(0)
0103         ),
0104         host1x_class_host_wait_syncpt(id, threshold)
0105     );
0106     host1x_cdma_push(cdma,
0107         host1x_opcode_setclass(next_class, 0, 0),
0108         HOST1X_OPCODE_NOP
0109     );
0110 #endif
0111 }
0112
0113 static void submit_gathers(struct host1x_job *job, u32 job_syncpt_base)
0114 {
0115     struct host1x_cdma *cdma = &job->channel->cdma;
0116 #if HOST1X_HW < 6
0117     struct device *dev = job->channel->dev;
0118 #endif
0119     unsigned int i;
0120     u32 threshold;
0121
0122     for (i = 0; i < job->num_cmds; i++) {
0123         struct host1x_job_cmd *cmd = &job->cmds[i];
0124
0125         if (cmd->is_wait) {
0126             if (cmd->wait.relative)
0127                 threshold = job_syncpt_base + cmd->wait.threshold;
0128             else
0129                 threshold = cmd->wait.threshold;
0130
0131             submit_wait(job, cmd->wait.id, threshold, cmd->wait.next_class);
0132         } else {
0133             struct host1x_job_gather *g = &cmd->gather;
0134
0135             dma_addr_t addr = g->base + g->offset;
0136             u32 op2, op3;
0137
0138             op2 = lower_32_bits(addr);
0139             op3 = upper_32_bits(addr);
0140
0141             trace_write_gather(cdma, g->bo, g->offset, g->words);
0142
0143             if (op3 != 0) {
0144 #if HOST1X_HW >= 6
0145                 u32 op1 = host1x_opcode_gather_wide(g->words);
0146                 u32 op4 = HOST1X_OPCODE_NOP;
0147
0148                 host1x_cdma_push_wide(cdma, op1, op2, op3, op4);
0149 #else
0150                 dev_err(dev, "invalid gather for push buffer %pad\n",
0151                     &addr);
0152                 continue;
0153 #endif
0154             } else {
0155                 u32 op1 = host1x_opcode_gather(g->words);
0156
0157                 host1x_cdma_push(cdma, op1, op2);
0158             }
0159         }
0160     }
0161 }
0162
0163 static inline void synchronize_syncpt_base(struct host1x_job *job)
0164 {
0165     struct host1x_syncpt *sp = job->syncpt;
0166     unsigned int id;
0167     u32 value;
0168
0169     value = host1x_syncpt_read_max(sp);
0170     id = sp->base->id;
0171
0172     host1x_cdma_push(&job->channel->cdma,
0173              host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
0174                 HOST1X_UCLASS_LOAD_SYNCPT_BASE, 1),
0175              HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(id) |
0176              HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(value));
0177 }
0178
0179 static void host1x_channel_set_streamid(struct host1x_channel *channel)
0180 {
0181 #if HOST1X_HW >= 6
0182     u32 sid = 0x7f;
0183 #ifdef CONFIG_IOMMU_API
0184     struct iommu_fwspec *spec = dev_iommu_fwspec_get(channel->dev->parent);
0185     if (spec)
0186         sid = spec->ids[0] & 0xffff;
0187 #endif
0188
0189     host1x_ch_writel(channel, sid, HOST1X_CHANNEL_SMMU_STREAMID);
0190 #endif
0191 }
0192
0193 static void host1x_enable_gather_filter(struct host1x_channel *ch)
0194 {
0195 #if HOST1X_HW >= 6
0196     struct host1x *host = dev_get_drvdata(ch->dev->parent);
0197     u32 val;
0198
0199     if (!host->hv_regs)
0200         return;
0201
0202     val = host1x_hypervisor_readl(
0203         host, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
0204     val |= BIT(ch->id % 32);
0205     host1x_hypervisor_writel(
0206         host, val, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
0207 #elif HOST1X_HW >= 4
0208     host1x_ch_writel(ch,
0209              HOST1X_CHANNEL_CHANNELCTRL_KERNEL_FILTER_GBUFFER(1),
0210              HOST1X_CHANNEL_CHANNELCTRL);
0211 #endif
0212 }
0213
0214 static void channel_program_cdma(struct host1x_job *job)
0215 {
0216     struct host1x_cdma *cdma = &job->channel->cdma;
0217     struct host1x_syncpt *sp = job->syncpt;
0218
0219 #if HOST1X_HW >= 6
0220     u32 fence;
0221
0222     /* Enter engine class with invalid stream ID. */
0223     host1x_cdma_push_wide(cdma,
0224         host1x_opcode_acquire_mlock(job->class),
0225         host1x_opcode_setclass(job->class, 0, 0),
0226         host1x_opcode_setpayload(0),
0227         host1x_opcode_setstreamid(job->engine_streamid_offset / 4));
0228
0229     /* Before switching stream ID to real stream ID, ensure engine is idle. */
0230     fence = host1x_syncpt_incr_max(sp, 1);
0231     host1x_cdma_push(&job->channel->cdma,
0232         host1x_opcode_nonincr(HOST1X_UCLASS_INCR_SYNCPT, 1),
0233         HOST1X_UCLASS_INCR_SYNCPT_INDX_F(job->syncpt->id) |
0234             HOST1X_UCLASS_INCR_SYNCPT_COND_F(4));
0235     submit_wait(job, job->syncpt->id, fence, job->class);
0236
0237     /* Submit work. */
0238     job->syncpt_end = host1x_syncpt_incr_max(sp, job->syncpt_incrs);
0239     submit_gathers(job, job->syncpt_end - job->syncpt_incrs);
0240
0241     /* Before releasing MLOCK, ensure engine is idle again. */
0242     fence = host1x_syncpt_incr_max(sp, 1);
0243     host1x_cdma_push(&job->channel->cdma,
0244         host1x_opcode_nonincr(HOST1X_UCLASS_INCR_SYNCPT, 1),
0245         HOST1X_UCLASS_INCR_SYNCPT_INDX_F(job->syncpt->id) |
0246             HOST1X_UCLASS_INCR_SYNCPT_COND_F(4));
0247     submit_wait(job, job->syncpt->id, fence, job->class);
0248
0249     /* Release MLOCK. */
0250     host1x_cdma_push(cdma,
0251         HOST1X_OPCODE_NOP, host1x_opcode_release_mlock(job->class));
0252 #else
0253     if (job->serialize) {
0254         /*
0255          * Force serialization by inserting a host wait for the
0256          * previous job to finish before this one can commence.
0257          */
0258         host1x_cdma_push(cdma,
0259                  host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
0260                     host1x_uclass_wait_syncpt_r(), 1),
0261                  host1x_class_host_wait_syncpt(job->syncpt->id,
0262                     host1x_syncpt_read_max(sp)));
0263     }
0264
0265     /* Synchronize base register to allow using it for relative waiting */
0266     if (sp->base)
0267         synchronize_syncpt_base(job);
0268
0269     /* add a setclass for modules that require it */
0270     if (job->class)
0271         host1x_cdma_push(cdma,
0272                  host1x_opcode_setclass(job->class, 0, 0),
0273                  HOST1X_OPCODE_NOP);
0274
0275     job->syncpt_end = host1x_syncpt_incr_max(sp, job->syncpt_incrs);
0276
0277     submit_gathers(job, job->syncpt_end - job->syncpt_incrs);
0278 #endif
0279 }
0280
0281 static int channel_submit(struct host1x_job *job)
0282 {
0283     struct host1x_channel *ch = job->channel;
0284     struct host1x_syncpt *sp = job->syncpt;
0285     u32 prev_max = 0;
0286     u32 syncval;
0287     int err;
0288     struct host1x_waitlist *completed_waiter = NULL;
0289     struct host1x *host = dev_get_drvdata(ch->dev->parent);
0290
0291     trace_host1x_channel_submit(dev_name(ch->dev),
0292                     job->num_cmds, job->num_relocs,
0293                     job->syncpt->id, job->syncpt_incrs);
0294
0295     /* before error checks, return current max */
0296     prev_max = job->syncpt_end = host1x_syncpt_read_max(sp);
0297
0298     /* get submit lock */
0299     err = mutex_lock_interruptible(&ch->submitlock);
0300     if (err)
0301         goto error;
0302
0303     completed_waiter = kzalloc(sizeof(*completed_waiter), GFP_KERNEL);
0304     if (!completed_waiter) {
0305         mutex_unlock(&ch->submitlock);
0306         err = -ENOMEM;
0307         goto error;
0308     }
0309
0310     host1x_channel_set_streamid(ch);
0311     host1x_enable_gather_filter(ch);
0312     host1x_hw_syncpt_assign_to_channel(host, sp, ch);
0313
0314     /* begin a CDMA submit */
0315     err = host1x_cdma_begin(&ch->cdma, job);
0316     if (err) {
0317         mutex_unlock(&ch->submitlock);
0318         goto error;
0319     }
0320
0321     channel_program_cdma(job);
0322     syncval = host1x_syncpt_read_max(sp);
0323
0324     /* end CDMA submit & stash pinned hMems into sync queue */
0325     host1x_cdma_end(&ch->cdma, job);
0326
0327     trace_host1x_channel_submitted(dev_name(ch->dev), prev_max, syncval);
0328
0329     /* schedule a submit complete interrupt */
0330     err = host1x_intr_add_action(host, sp, syncval,
0331                      HOST1X_INTR_ACTION_SUBMIT_COMPLETE, ch,
0332                      completed_waiter, &job->waiter);
0333     completed_waiter = NULL;
0334     WARN(err, "Failed to set submit complete interrupt");
0335
0336     mutex_unlock(&ch->submitlock);
0337
0338     return 0;
0339
0340 error:
0341     kfree(completed_waiter);
0342     return err;
0343 }
0344
0345 static int host1x_channel_init(struct host1x_channel *ch, struct host1x *dev,
0346                    unsigned int index)
0347 {
0348 #if HOST1X_HW < 6
0349     ch->regs = dev->regs + index * 0x4000;
0350 #else
0351     ch->regs = dev->regs + index * 0x100;
0352 #endif
0353     return 0;
0354 }
0355
0356 static const struct host1x_channel_ops host1x_channel_ops = {
0357     .init = host1x_channel_init,
0358     .submit = channel_submit,
0359 };