drm/lima/lima_pp.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
0003
0004 #include <linux/interrupt.h>
0005 #include <linux/io.h>
0006 #include <linux/device.h>
0007 #include <linux/slab.h>
0008
0009 #include <drm/lima_drm.h>
0010
0011 #include "lima_device.h"
0012 #include "lima_pp.h"
0013 #include "lima_dlbu.h"
0014 #include "lima_bcast.h"
0015 #include "lima_vm.h"
0016 #include "lima_regs.h"
0017
0018 #define pp_write(reg, data) writel(data, ip->iomem + reg)
0019 #define pp_read(reg) readl(ip->iomem + reg)
0020
0021 static void lima_pp_handle_irq(struct lima_ip *ip, u32 state)
0022 {
0023     struct lima_device *dev = ip->dev;
0024     struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
0025
0026     if (state & LIMA_PP_IRQ_MASK_ERROR) {
0027         u32 status = pp_read(LIMA_PP_STATUS);
0028
0029         dev_err(dev->dev, "pp error irq state=%x status=%x\n",
0030             state, status);
0031
0032         pipe->error = true;
0033
0034         /* mask all interrupts before hard reset */
0035         pp_write(LIMA_PP_INT_MASK, 0);
0036     }
0037
0038     pp_write(LIMA_PP_INT_CLEAR, state);
0039 }
0040
0041 static irqreturn_t lima_pp_irq_handler(int irq, void *data)
0042 {
0043     struct lima_ip *ip = data;
0044     struct lima_device *dev = ip->dev;
0045     struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
0046     u32 state = pp_read(LIMA_PP_INT_STATUS);
0047
0048     /* for shared irq case */
0049     if (!state)
0050         return IRQ_NONE;
0051
0052     lima_pp_handle_irq(ip, state);
0053
0054     if (atomic_dec_and_test(&pipe->task))
0055         lima_sched_pipe_task_done(pipe);
0056
0057     return IRQ_HANDLED;
0058 }
0059
0060 static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data)
0061 {
0062     int i;
0063     irqreturn_t ret = IRQ_NONE;
0064     struct lima_ip *pp_bcast = data;
0065     struct lima_device *dev = pp_bcast->dev;
0066     struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
0067     struct drm_lima_m450_pp_frame *frame;
0068
0069     /* for shared irq case */
0070     if (!pipe->current_task)
0071         return IRQ_NONE;
0072
0073     frame = pipe->current_task->frame;
0074
0075     for (i = 0; i < frame->num_pp; i++) {
0076         struct lima_ip *ip = pipe->processor[i];
0077         u32 status, state;
0078
0079         if (pipe->done & (1 << i))
0080             continue;
0081
0082         /* status read first in case int state change in the middle
0083          * which may miss the interrupt handling
0084          */
0085         status = pp_read(LIMA_PP_STATUS);
0086         state = pp_read(LIMA_PP_INT_STATUS);
0087
0088         if (state) {
0089             lima_pp_handle_irq(ip, state);
0090             ret = IRQ_HANDLED;
0091         } else {
0092             if (status & LIMA_PP_STATUS_RENDERING_ACTIVE)
0093                 continue;
0094         }
0095
0096         pipe->done |= (1 << i);
0097         if (atomic_dec_and_test(&pipe->task))
0098             lima_sched_pipe_task_done(pipe);
0099     }
0100
0101     return ret;
0102 }
0103
0104 static void lima_pp_soft_reset_async(struct lima_ip *ip)
0105 {
0106     if (ip->data.async_reset)
0107         return;
0108
0109     pp_write(LIMA_PP_INT_MASK, 0);
0110     pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL);
0111     pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET);
0112     ip->data.async_reset = true;
0113 }
0114
0115 static int lima_pp_soft_reset_poll(struct lima_ip *ip)
0116 {
0117     return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) &&
0118         pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED;
0119 }
0120
0121 static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip)
0122 {
0123     struct lima_device *dev = ip->dev;
0124     int ret;
0125
0126     ret = lima_poll_timeout(ip, lima_pp_soft_reset_poll, 0, 100);
0127     if (ret) {
0128         dev_err(dev->dev, "pp %s reset time out\n", lima_ip_name(ip));
0129         return ret;
0130     }
0131
0132     pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
0133     pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
0134     return 0;
0135 }
0136
0137 static int lima_pp_soft_reset_async_wait(struct lima_ip *ip)
0138 {
0139     int i, err = 0;
0140
0141     if (!ip->data.async_reset)
0142         return 0;
0143
0144     if (ip->id == lima_ip_pp_bcast) {
0145         struct lima_device *dev = ip->dev;
0146         struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
0147         struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame;
0148
0149         for (i = 0; i < frame->num_pp; i++)
0150             err |= lima_pp_soft_reset_async_wait_one(pipe->processor[i]);
0151     } else
0152         err = lima_pp_soft_reset_async_wait_one(ip);
0153
0154     ip->data.async_reset = false;
0155     return err;
0156 }
0157
0158 static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb)
0159 {
0160     int i, j, n = 0;
0161
0162     for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++)
0163         writel(frame[i], ip->iomem + LIMA_PP_FRAME + i * 4);
0164
0165     for (i = 0; i < 3; i++) {
0166         for (j = 0; j < LIMA_PP_WB_REG_NUM; j++)
0167             writel(wb[n++], ip->iomem + LIMA_PP_WB(i) + j * 4);
0168     }
0169 }
0170
0171 static int lima_pp_hard_reset_poll(struct lima_ip *ip)
0172 {
0173     pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000);
0174     return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000;
0175 }
0176
0177 static int lima_pp_hard_reset(struct lima_ip *ip)
0178 {
0179     struct lima_device *dev = ip->dev;
0180     int ret;
0181
0182     pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000);
0183     pp_write(LIMA_PP_INT_MASK, 0);
0184     pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET);
0185     ret = lima_poll_timeout(ip, lima_pp_hard_reset_poll, 10, 100);
0186     if (ret) {
0187         dev_err(dev->dev, "pp hard reset timeout\n");
0188         return ret;
0189     }
0190
0191     pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0);
0192     pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
0193     pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
0194     return 0;
0195 }
0196
0197 static void lima_pp_print_version(struct lima_ip *ip)
0198 {
0199     u32 version, major, minor;
0200     char *name;
0201
0202     version = pp_read(LIMA_PP_VERSION);
0203     major = (version >> 8) & 0xFF;
0204     minor = version & 0xFF;
0205     switch (version >> 16) {
0206     case 0xC807:
0207         name = "mali200";
0208         break;
0209     case 0xCE07:
0210         name = "mali300";
0211         break;
0212     case 0xCD07:
0213         name = "mali400";
0214         break;
0215     case 0xCF07:
0216         name = "mali450";
0217         break;
0218     default:
0219         name = "unknown";
0220         break;
0221     }
0222     dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
0223          lima_ip_name(ip), name, major, minor);
0224 }
0225
0226 static int lima_pp_hw_init(struct lima_ip *ip)
0227 {
0228     ip->data.async_reset = false;
0229     lima_pp_soft_reset_async(ip);
0230     return lima_pp_soft_reset_async_wait(ip);
0231 }
0232
0233 int lima_pp_resume(struct lima_ip *ip)
0234 {
0235     return lima_pp_hw_init(ip);
0236 }
0237
0238 void lima_pp_suspend(struct lima_ip *ip)
0239 {
0240
0241 }
0242
0243 int lima_pp_init(struct lima_ip *ip)
0244 {
0245     struct lima_device *dev = ip->dev;
0246     int err;
0247
0248     lima_pp_print_version(ip);
0249
0250     err = lima_pp_hw_init(ip);
0251     if (err)
0252         return err;
0253
0254     err = devm_request_irq(dev->dev, ip->irq, lima_pp_irq_handler,
0255                    IRQF_SHARED, lima_ip_name(ip), ip);
0256     if (err) {
0257         dev_err(dev->dev, "pp %s fail to request irq\n",
0258             lima_ip_name(ip));
0259         return err;
0260     }
0261
0262     dev->pp_version = pp_read(LIMA_PP_VERSION);
0263
0264     return 0;
0265 }
0266
0267 void lima_pp_fini(struct lima_ip *ip)
0268 {
0269
0270 }
0271
0272 int lima_pp_bcast_resume(struct lima_ip *ip)
0273 {
0274     /* PP has been reset by individual PP resume */
0275     ip->data.async_reset = false;
0276     return 0;
0277 }
0278
0279 void lima_pp_bcast_suspend(struct lima_ip *ip)
0280 {
0281
0282 }
0283
0284 int lima_pp_bcast_init(struct lima_ip *ip)
0285 {
0286     struct lima_device *dev = ip->dev;
0287     int err;
0288
0289     err = devm_request_irq(dev->dev, ip->irq, lima_pp_bcast_irq_handler,
0290                    IRQF_SHARED, lima_ip_name(ip), ip);
0291     if (err) {
0292         dev_err(dev->dev, "pp %s fail to request irq\n",
0293             lima_ip_name(ip));
0294         return err;
0295     }
0296
0297     return 0;
0298 }
0299
0300 void lima_pp_bcast_fini(struct lima_ip *ip)
0301 {
0302
0303 }
0304
0305 static int lima_pp_task_validate(struct lima_sched_pipe *pipe,
0306                  struct lima_sched_task *task)
0307 {
0308     u32 num_pp;
0309
0310     if (pipe->bcast_processor) {
0311         struct drm_lima_m450_pp_frame *f = task->frame;
0312
0313         num_pp = f->num_pp;
0314
0315         if (f->_pad)
0316             return -EINVAL;
0317     } else {
0318         struct drm_lima_m400_pp_frame *f = task->frame;
0319
0320         num_pp = f->num_pp;
0321     }
0322
0323     if (num_pp == 0 || num_pp > pipe->num_processor)
0324         return -EINVAL;
0325
0326     return 0;
0327 }
0328
0329 static void lima_pp_task_run(struct lima_sched_pipe *pipe,
0330                  struct lima_sched_task *task)
0331 {
0332     if (pipe->bcast_processor) {
0333         struct drm_lima_m450_pp_frame *frame = task->frame;
0334         struct lima_device *dev = pipe->bcast_processor->dev;
0335         struct lima_ip *ip = pipe->bcast_processor;
0336         int i;
0337
0338         pipe->done = 0;
0339         atomic_set(&pipe->task, frame->num_pp);
0340
0341         if (frame->use_dlbu) {
0342             lima_dlbu_enable(dev, frame->num_pp);
0343
0344             frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU;
0345             lima_dlbu_set_reg(dev->ip + lima_ip_dlbu, frame->dlbu_regs);
0346         } else
0347             lima_dlbu_disable(dev);
0348
0349         lima_bcast_enable(dev, frame->num_pp);
0350
0351         lima_pp_soft_reset_async_wait(ip);
0352
0353         lima_pp_write_frame(ip, frame->frame, frame->wb);
0354
0355         for (i = 0; i < frame->num_pp; i++) {
0356             struct lima_ip *ip = pipe->processor[i];
0357
0358             pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]);
0359             if (!frame->use_dlbu)
0360                 pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]);
0361         }
0362
0363         pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
0364     } else {
0365         struct drm_lima_m400_pp_frame *frame = task->frame;
0366         int i;
0367
0368         atomic_set(&pipe->task, frame->num_pp);
0369
0370         for (i = 0; i < frame->num_pp; i++) {
0371             struct lima_ip *ip = pipe->processor[i];
0372
0373             frame->frame[LIMA_PP_FRAME >> 2] =
0374                 frame->plbu_array_address[i];
0375             frame->frame[LIMA_PP_STACK >> 2] =
0376                 frame->fragment_stack_address[i];
0377
0378             lima_pp_soft_reset_async_wait(ip);
0379
0380             lima_pp_write_frame(ip, frame->frame, frame->wb);
0381
0382             pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
0383         }
0384     }
0385 }
0386
0387 static void lima_pp_task_fini(struct lima_sched_pipe *pipe)
0388 {
0389     if (pipe->bcast_processor)
0390         lima_pp_soft_reset_async(pipe->bcast_processor);
0391     else {
0392         int i;
0393
0394         for (i = 0; i < pipe->num_processor; i++)
0395             lima_pp_soft_reset_async(pipe->processor[i]);
0396     }
0397 }
0398
0399 static void lima_pp_task_error(struct lima_sched_pipe *pipe)
0400 {
0401     int i;
0402
0403     for (i = 0; i < pipe->num_processor; i++) {
0404         struct lima_ip *ip = pipe->processor[i];
0405
0406         dev_err(ip->dev->dev, "pp task error %d int_state=%x status=%x\n",
0407             i, pp_read(LIMA_PP_INT_STATUS), pp_read(LIMA_PP_STATUS));
0408
0409         lima_pp_hard_reset(ip);
0410     }
0411 }
0412
0413 static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe)
0414 {
0415     if (atomic_dec_and_test(&pipe->task))
0416         lima_sched_pipe_task_done(pipe);
0417 }
0418
0419 static struct kmem_cache *lima_pp_task_slab;
0420 static int lima_pp_task_slab_refcnt;
0421
0422 int lima_pp_pipe_init(struct lima_device *dev)
0423 {
0424     int frame_size;
0425     struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
0426
0427     if (dev->id == lima_gpu_mali400)
0428         frame_size = sizeof(struct drm_lima_m400_pp_frame);
0429     else
0430         frame_size = sizeof(struct drm_lima_m450_pp_frame);
0431
0432     if (!lima_pp_task_slab) {
0433         lima_pp_task_slab = kmem_cache_create_usercopy(
0434             "lima_pp_task", sizeof(struct lima_sched_task) + frame_size,
0435             0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task),
0436             frame_size, NULL);
0437         if (!lima_pp_task_slab)
0438             return -ENOMEM;
0439     }
0440     lima_pp_task_slab_refcnt++;
0441
0442     pipe->frame_size = frame_size;
0443     pipe->task_slab = lima_pp_task_slab;
0444
0445     pipe->task_validate = lima_pp_task_validate;
0446     pipe->task_run = lima_pp_task_run;
0447     pipe->task_fini = lima_pp_task_fini;
0448     pipe->task_error = lima_pp_task_error;
0449     pipe->task_mmu_error = lima_pp_task_mmu_error;
0450
0451     return 0;
0452 }
0453
0454 void lima_pp_pipe_fini(struct lima_device *dev)
0455 {
0456     if (!--lima_pp_task_slab_refcnt) {
0457         kmem_cache_destroy(lima_pp_task_slab);
0458         lima_pp_task_slab = NULL;
0459     }
0460 }