media/ipu3/ipu3-css.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (C) 2018 Intel Corporation
0003
0004 #include <linux/device.h>
0005 #include <linux/iopoll.h>
0006 #include <linux/slab.h>
0007
0008 #include "ipu3.h"
0009 #include "ipu3-css.h"
0010 #include "ipu3-css-fw.h"
0011 #include "ipu3-css-params.h"
0012 #include "ipu3-dmamap.h"
0013 #include "ipu3-tables.h"
0014
0015 /* IRQ configuration */
0016 #define IMGU_IRQCTRL_IRQ_MASK   (IMGU_IRQCTRL_IRQ_SP1 | \
0017                  IMGU_IRQCTRL_IRQ_SP2 | \
0018                  IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
0019                  IMGU_IRQCTRL_IRQ_SW_PIN(1))
0020
0021 #define IPU3_CSS_FORMAT_BPP_DEN 50  /* Denominator */
0022
0023 /* Some sane limits for resolutions */
0024 #define IPU3_CSS_MIN_RES    32
0025 #define IPU3_CSS_MAX_H      3136
0026 #define IPU3_CSS_MAX_W      4224
0027
0028 /* minimal envelope size(GDC in - out) should be 4 */
0029 #define MIN_ENVELOPE            4
0030
0031 /*
0032  * pre-allocated buffer size for CSS ABI, auxiliary frames
0033  * after BDS and before GDC. Those values should be tuned
0034  * to big enough to avoid buffer re-allocation when
0035  * streaming to lower streaming latency.
0036  */
0037 #define CSS_ABI_SIZE    136
0038 #define CSS_BDS_SIZE    (4480 * 3200 * 3)
0039 #define CSS_GDC_SIZE    (4224 * 3200 * 12 / 8)
0040
0041 #define IPU3_CSS_QUEUE_TO_FLAGS(q)  (1 << (q))
0042 #define IPU3_CSS_FORMAT_FL_IN       \
0043             IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
0044 #define IPU3_CSS_FORMAT_FL_OUT      \
0045             IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
0046 #define IPU3_CSS_FORMAT_FL_VF       \
0047             IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
0048
0049 /* Formats supported by IPU3 Camera Sub System */
0050 static const struct imgu_css_format imgu_css_formats[] = {
0051     {
0052         .pixelformat = V4L2_PIX_FMT_NV12,
0053         .colorspace = V4L2_COLORSPACE_SRGB,
0054         .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
0055         .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
0056         .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
0057         .chroma_decim = 4,
0058         .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
0059         .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
0060     }, {
0061         /* Each 32 bytes contains 25 10-bit pixels */
0062         .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
0063         .colorspace = V4L2_COLORSPACE_RAW,
0064         .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
0065         .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
0066         .bit_depth = 10,
0067         .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
0068         .flags = IPU3_CSS_FORMAT_FL_IN,
0069     }, {
0070         .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
0071         .colorspace = V4L2_COLORSPACE_RAW,
0072         .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
0073         .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
0074         .bit_depth = 10,
0075         .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
0076         .flags = IPU3_CSS_FORMAT_FL_IN,
0077     }, {
0078         .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
0079         .colorspace = V4L2_COLORSPACE_RAW,
0080         .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
0081         .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
0082         .bit_depth = 10,
0083         .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
0084         .flags = IPU3_CSS_FORMAT_FL_IN,
0085     }, {
0086         .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
0087         .colorspace = V4L2_COLORSPACE_RAW,
0088         .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
0089         .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
0090         .bit_depth = 10,
0091         .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
0092         .flags = IPU3_CSS_FORMAT_FL_IN,
0093     },
0094 };
0095
0096 static const struct {
0097     enum imgu_abi_queue_id qid;
0098     size_t ptr_ofs;
0099 } imgu_css_queues[IPU3_CSS_QUEUES] = {
0100     [IPU3_CSS_QUEUE_IN] = {
0101         IMGU_ABI_QUEUE_C_ID,
0102         offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
0103     },
0104     [IPU3_CSS_QUEUE_OUT] = {
0105         IMGU_ABI_QUEUE_D_ID,
0106         offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
0107     },
0108     [IPU3_CSS_QUEUE_VF] = {
0109         IMGU_ABI_QUEUE_E_ID,
0110         offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
0111     },
0112     [IPU3_CSS_QUEUE_STAT_3A] = {
0113         IMGU_ABI_QUEUE_F_ID,
0114         offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
0115     },
0116 };
0117
0118 /* Initialize queue based on given format, adjust format as needed */
0119 static int imgu_css_queue_init(struct imgu_css_queue *queue,
0120                    struct v4l2_pix_format_mplane *fmt, u32 flags)
0121 {
0122     struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
0123     unsigned int i;
0124     u32 sizeimage;
0125
0126     INIT_LIST_HEAD(&queue->bufs);
0127
0128     queue->css_fmt = NULL;  /* Disable */
0129     if (!fmt)
0130         return 0;
0131
0132     for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
0133         if (!(imgu_css_formats[i].flags & flags))
0134             continue;
0135         queue->css_fmt = &imgu_css_formats[i];
0136         if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
0137             break;
0138     }
0139     if (!queue->css_fmt)
0140         return -EINVAL; /* Could not find any suitable format */
0141
0142     queue->fmt.mpix = *fmt;
0143
0144     f->width = ALIGN(clamp_t(u32, f->width,
0145                  IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
0146     f->height = ALIGN(clamp_t(u32, f->height,
0147                   IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
0148     queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
0149     f->plane_fmt[0].bytesperline =
0150         imgu_bytesperline(f->width, queue->css_fmt->frame_format);
0151     sizeimage = f->height * f->plane_fmt[0].bytesperline;
0152     if (queue->css_fmt->chroma_decim)
0153         sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
0154
0155     f->plane_fmt[0].sizeimage = sizeimage;
0156     f->field = V4L2_FIELD_NONE;
0157     f->num_planes = 1;
0158     f->colorspace = queue->css_fmt->colorspace;
0159     f->flags = 0;
0160     f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
0161     f->quantization = V4L2_QUANTIZATION_DEFAULT;
0162     f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
0163     memset(f->reserved, 0, sizeof(f->reserved));
0164
0165     return 0;
0166 }
0167
0168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
0169 {
0170     return q->css_fmt;
0171 }
0172
0173 /******************* css hw *******************/
0174
0175 /* In the style of writesl() defined in include/asm-generic/io.h */
0176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
0177 {
0178     if (count >= 4) {
0179         const u32 *buf = mem;
0180
0181         count /= 4;
0182         do {
0183             writel(*buf++, addr);
0184             addr += 4;
0185         } while (--count);
0186     }
0187 }
0188
0189 /* Wait until register `reg', masked with `mask', becomes `cmp' */
0190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
0191 {
0192     u32 val;
0193
0194     return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
0195                   1000, 100 * 1000);
0196 }
0197
0198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
0199
0200 int imgu_css_set_powerup(struct device *dev, void __iomem *base,
0201              unsigned int freq)
0202 {
0203     u32 pm_ctrl, state, val;
0204
0205     dev_dbg(dev, "%s with freq %u\n", __func__, freq);
0206     /* Clear the CSS busy signal */
0207     readl(base + IMGU_REG_GP_BUSY);
0208     writel(0, base + IMGU_REG_GP_BUSY);
0209
0210     /* Wait for idle signal */
0211     if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
0212              IMGU_STATE_IDLE_STS)) {
0213         dev_err(dev, "failed to set CSS idle\n");
0214         goto fail;
0215     }
0216
0217     /* Reset the css */
0218     writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
0219            base + IMGU_REG_PM_CTRL);
0220
0221     usleep_range(200, 300);
0222
0223     /** Prepare CSS */
0224
0225     pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
0226     state = readl(base + IMGU_REG_STATE);
0227
0228     dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
0229         pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
0230
0231     /* Power up CSS using wrapper */
0232     if (state & IMGU_STATE_POWER_DOWN) {
0233         writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
0234                base + IMGU_REG_PM_CTRL);
0235         if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
0236                  IMGU_PM_CTRL_START, 0)) {
0237             dev_err(dev, "failed to power up CSS\n");
0238             goto fail;
0239         }
0240         usleep_range(2000, 3000);
0241     } else {
0242         writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
0243     }
0244
0245     /* Set the busy bit */
0246     writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
0247
0248     /* Set CSS clock frequency */
0249     pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
0250     val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
0251     writel(val, base + IMGU_REG_PM_CTRL);
0252     writel(0, base + IMGU_REG_GP_BUSY);
0253     if (imgu_hw_wait(base, IMGU_REG_STATE,
0254              IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
0255         dev_err(dev, "failed to pwrdn CSS\n");
0256         goto fail;
0257     }
0258     val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
0259     writel(val, base + IMGU_REG_SYSTEM_REQ);
0260     writel(1, base + IMGU_REG_GP_BUSY);
0261     writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
0262            base + IMGU_REG_PM_CTRL);
0263     if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
0264              IMGU_STATE_HALT_STS)) {
0265         dev_err(dev, "failed to halt CSS\n");
0266         goto fail;
0267     }
0268
0269     writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
0270            base + IMGU_REG_PM_CTRL);
0271     if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
0272         dev_err(dev, "failed to start CSS\n");
0273         goto fail;
0274     }
0275     writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
0276            base + IMGU_REG_PM_CTRL);
0277
0278     val = readl(base + IMGU_REG_PM_CTRL);   /* get pm_ctrl */
0279     val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
0280     val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
0281     writel(val, base + IMGU_REG_PM_CTRL);
0282
0283     return 0;
0284
0285 fail:
0286     imgu_css_set_powerdown(dev, base);
0287     return -EIO;
0288 }
0289
0290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
0291 {
0292     dev_dbg(dev, "%s\n", __func__);
0293     /* wait for cio idle signal */
0294     if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
0295              IMGU_CIO_GATE_BURST_MASK, 0))
0296         dev_warn(dev, "wait cio gate idle timeout");
0297
0298     /* wait for css idle signal */
0299     if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
0300              IMGU_STATE_IDLE_STS))
0301         dev_warn(dev, "wait css idle timeout\n");
0302
0303     /* do halt-halted handshake with css */
0304     writel(1, base + IMGU_REG_GP_HALT);
0305     if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
0306              IMGU_STATE_HALT_STS))
0307         dev_warn(dev, "failed to halt css");
0308
0309     /* de-assert the busy bit */
0310     writel(0, base + IMGU_REG_GP_BUSY);
0311 }
0312
0313 static void imgu_css_hw_enable_irq(struct imgu_css *css)
0314 {
0315     void __iomem *const base = css->base;
0316     u32 val, i;
0317
0318     /* Set up interrupts */
0319
0320     /*
0321      * Enable IRQ on the SP which signals that SP goes to idle
0322      * (aka ready state) and set trigger to pulse
0323      */
0324     val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
0325     writel(val, base + IMGU_REG_SP_CTRL(0));
0326     writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
0327
0328     /* Enable IRQs from the IMGU wrapper */
0329     writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
0330     /* Clear */
0331     writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
0332
0333     /* Enable IRQs from main IRQ controller */
0334     writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
0335     writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
0336     writel(IMGU_IRQCTRL_IRQ_MASK,
0337            base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
0338     writel(IMGU_IRQCTRL_IRQ_MASK,
0339            base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
0340     writel(IMGU_IRQCTRL_IRQ_MASK,
0341            base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
0342     writel(IMGU_IRQCTRL_IRQ_MASK,
0343            base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
0344     /* Wait for write complete */
0345     readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
0346
0347     /* Enable IRQs from SP0 and SP1 controllers */
0348     for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
0349         writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
0350         writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
0351         writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
0352         writel(IMGU_IRQCTRL_IRQ_MASK,
0353                base + IMGU_REG_IRQCTRL_ENABLE(i));
0354         writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
0355         writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
0356         /* Wait for write complete */
0357         readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
0358     }
0359 }
0360
0361 static int imgu_css_hw_init(struct imgu_css *css)
0362 {
0363     /* For checking that streaming monitor statuses are valid */
0364     static const struct {
0365         u32 reg;
0366         u32 mask;
0367         const char *name;
0368     } stream_monitors[] = {
0369         {
0370             IMGU_REG_GP_SP1_STRMON_STAT,
0371             IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
0372             "ISP0 to SP0"
0373         }, {
0374             IMGU_REG_GP_ISP_STRMON_STAT,
0375             IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
0376             "SP0 to ISP0"
0377         }, {
0378             IMGU_REG_GP_MOD_STRMON_STAT,
0379             IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
0380             "ISP0 to DMA0"
0381         }, {
0382             IMGU_REG_GP_ISP_STRMON_STAT,
0383             IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
0384             "DMA0 to ISP0"
0385         }, {
0386             IMGU_REG_GP_MOD_STRMON_STAT,
0387             IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
0388             "ISP0 to GDC0"
0389         }, {
0390             IMGU_REG_GP_MOD_STRMON_STAT,
0391             IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
0392             "GDC0 to ISP0"
0393         }, {
0394             IMGU_REG_GP_MOD_STRMON_STAT,
0395             IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
0396             "SP0 to DMA0"
0397         }, {
0398             IMGU_REG_GP_SP1_STRMON_STAT,
0399             IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
0400             "DMA0 to SP0"
0401         }, {
0402             IMGU_REG_GP_MOD_STRMON_STAT,
0403             IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
0404             "SP0 to GDC0"
0405         }, {
0406             IMGU_REG_GP_MOD_STRMON_STAT,
0407             IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
0408             "GDC0 to SP0"
0409         },
0410     };
0411
0412     struct device *dev = css->dev;
0413     void __iomem *const base = css->base;
0414     u32 val, i;
0415
0416     /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
0417     for (i = 0; i < IMGU_NUM_SP; i++) {
0418         struct imgu_fw_info *bi =
0419                     &css->fwp->binary_header[css->fw_sp[i]];
0420
0421         writel(css->binary[css->fw_sp[i]].daddr,
0422                base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
0423         writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
0424                IMGU_CTRL_ICACHE_INV,
0425                base + IMGU_REG_SP_CTRL(bi->type));
0426     }
0427     writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
0428     writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
0429            base + IMGU_REG_ISP_CTRL);
0430
0431     /* Check that IMGU hardware is ready */
0432
0433     if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
0434         dev_err(dev, "SP is not idle\n");
0435         return -EIO;
0436     }
0437     if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
0438         dev_err(dev, "ISP is not idle\n");
0439         return -EIO;
0440     }
0441
0442     for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
0443         val = readl(base + stream_monitors[i].reg);
0444         if (val & stream_monitors[i].mask) {
0445             dev_err(dev, "error: Stream monitor %s is valid\n",
0446                 stream_monitors[i].name);
0447             return -EIO;
0448         }
0449     }
0450
0451     /* Initialize GDC with default values */
0452
0453     for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
0454         u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
0455         u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
0456         u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
0457         u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
0458
0459         writel(val0 | (val1 << 16),
0460                base + IMGU_REG_GDC_LUT_BASE + i * 8);
0461         writel(val2 | (val3 << 16),
0462                base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
0463     }
0464
0465     return 0;
0466 }
0467
0468 /* Boot the given IPU3 CSS SP */
0469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
0470 {
0471     void __iomem *const base = css->base;
0472     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
0473     struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
0474         .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
0475             + bi->blob.data_source,
0476         .dmem_data_addr = bi->blob.data_target,
0477         .dmem_bss_addr = bi->blob.bss_target,
0478         .data_size = bi->blob.data_size,
0479         .bss_size = bi->blob.bss_size,
0480         .sp_id = sp,
0481     };
0482
0483     writes(&dmem_cfg, sizeof(dmem_cfg), base +
0484            IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
0485
0486     writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
0487
0488     writel(readl(base + IMGU_REG_SP_CTRL(sp))
0489         | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
0490
0491     if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
0492              + bi->info.sp.sw_state,
0493              ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
0494         return -EIO;
0495
0496     return 0;
0497 }
0498
0499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
0500 static int imgu_css_hw_start(struct imgu_css *css)
0501 {
0502     static const u32 event_mask =
0503         ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
0504         (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
0505         (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
0506         (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
0507         (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
0508         (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
0509         (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
0510         (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
0511         (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
0512         (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
0513         (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
0514         << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
0515
0516     void __iomem *const base = css->base;
0517     struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
0518     unsigned int i;
0519
0520     writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
0521
0522     /* Start bootloader */
0523
0524     writel(IMGU_ABI_BL_SWSTATE_BUSY,
0525            base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
0526     writel(IMGU_NUM_SP,
0527            base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
0528
0529     for (i = 0; i < IMGU_NUM_SP; i++) {
0530         int j = IMGU_NUM_SP - i - 1;    /* load sp1 first, then sp0 */
0531         struct imgu_fw_info *sp =
0532                     &css->fwp->binary_header[css->fw_sp[j]];
0533         struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
0534             .src_addr = css->binary[css->fw_sp[j]].daddr
0535                 + sp->blob.text_source,
0536             .size = sp->blob.text_size,
0537             .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
0538             .dst_addr = IMGU_SP_PMEM_BASE(j),
0539         };
0540
0541         writes(&dma_cmd, sizeof(dma_cmd),
0542                base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
0543                bl->info.bl.dma_cmd_list);
0544     }
0545
0546     writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
0547
0548     writel(readl(base + IMGU_REG_ISP_CTRL)
0549         | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
0550     if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
0551              + bl->info.bl.sw_state, ~0,
0552              IMGU_ABI_BL_SWSTATE_OK)) {
0553         dev_err(css->dev, "failed to start bootloader\n");
0554         return -EIO;
0555     }
0556
0557     /* Start ISP */
0558
0559     memset(css->xmem_sp_group_ptrs.vaddr, 0,
0560            sizeof(struct imgu_abi_sp_group));
0561
0562     bi = &css->fwp->binary_header[css->fw_sp[0]];
0563
0564     writel(css->xmem_sp_group_ptrs.daddr,
0565            base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
0566
0567     writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
0568            base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
0569     writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
0570
0571     if (imgu_css_hw_start_sp(css, 0))
0572         return -EIO;
0573
0574     writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
0575     writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
0576         bi->info.sp.host_sp_queues_initialized);
0577     writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
0578     writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
0579     writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
0580         + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
0581
0582     /* Enable all events for all queues */
0583
0584     for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
0585         writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
0586             + bi->info.sp.host_sp_com
0587             + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
0588     writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
0589         bi->info.sp.host_sp_queues_initialized);
0590
0591     /* Start SP1 */
0592
0593     bi = &css->fwp->binary_header[css->fw_sp[1]];
0594
0595     writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
0596            base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
0597
0598     if (imgu_css_hw_start_sp(css, 1))
0599         return -EIO;
0600
0601     writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
0602         + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
0603
0604     return 0;
0605 }
0606
0607 static void imgu_css_hw_stop(struct imgu_css *css)
0608 {
0609     void __iomem *const base = css->base;
0610     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
0611
0612     /* Stop fw */
0613     writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
0614            base + IMGU_REG_SP_DMEM_BASE(0) +
0615            bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
0616     if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
0617              IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
0618         dev_err(css->dev, "wait sp0 idle timeout.\n");
0619     if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
0620           IMGU_ABI_SP_SWSTATE_TERMINATED)
0621         dev_err(css->dev, "sp0 is not terminated.\n");
0622     if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
0623              IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
0624         dev_err(css->dev, "wait isp idle timeout\n");
0625 }
0626
0627 static void imgu_css_hw_cleanup(struct imgu_css *css)
0628 {
0629     void __iomem *const base = css->base;
0630
0631     /** Reset CSS **/
0632
0633     /* Clear the CSS busy signal */
0634     readl(base + IMGU_REG_GP_BUSY);
0635     writel(0, base + IMGU_REG_GP_BUSY);
0636
0637     /* Wait for idle signal */
0638     if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
0639              IMGU_STATE_IDLE_STS))
0640         dev_err(css->dev, "failed to shut down hw cleanly\n");
0641
0642     /* Reset the css */
0643     writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
0644            base + IMGU_REG_PM_CTRL);
0645
0646     usleep_range(200, 300);
0647 }
0648
0649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
0650 {
0651     struct imgu_device *imgu = dev_get_drvdata(css->dev);
0652     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
0653     unsigned int i;
0654
0655     imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
0656     imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
0657     imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
0658     imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
0659
0660     for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
0661         imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
0662 }
0663
0664 /*
0665  * This function initializes various stages of the
0666  * IPU3 CSS ISP pipeline
0667  */
0668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
0669 {
0670     static const int BYPC = 2;  /* Bytes per component */
0671     static const struct imgu_abi_buffer_sp buffer_sp_init = {
0672         .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
0673         .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
0674     };
0675
0676     struct imgu_abi_isp_iterator_config *cfg_iter;
0677     struct imgu_abi_isp_ref_config *cfg_ref;
0678     struct imgu_abi_isp_dvs_config *cfg_dvs;
0679     struct imgu_abi_isp_tnr3_config *cfg_tnr;
0680     struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
0681     struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
0682
0683     const int stage = 0;
0684     unsigned int i, j;
0685
0686     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
0687     struct imgu_css_queue *css_queue_in =
0688             &css_pipe->queue[IPU3_CSS_QUEUE_IN];
0689     struct imgu_css_queue *css_queue_out =
0690             &css_pipe->queue[IPU3_CSS_QUEUE_OUT];
0691     struct imgu_css_queue *css_queue_vf =
0692             &css_pipe->queue[IPU3_CSS_QUEUE_VF];
0693     const struct imgu_fw_info *bi =
0694             &css->fwp->binary_header[css_pipe->bindex];
0695     const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
0696
0697     struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
0698         bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
0699     struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
0700         bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
0701
0702     struct imgu_abi_isp_stage *isp_stage;
0703     struct imgu_abi_sp_stage *sp_stage;
0704     struct imgu_abi_sp_group *sp_group;
0705     struct imgu_abi_frames_sp *frames_sp;
0706     struct imgu_abi_frame_sp *frame_sp;
0707     struct imgu_abi_frame_sp_info *frame_sp_info;
0708
0709     const unsigned int bds_width_pad =
0710                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
0711                       2 * IPU3_UAPI_ISP_VEC_ELEMS);
0712
0713     const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
0714     enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
0715     void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
0716
0717     struct imgu_device *imgu = dev_get_drvdata(css->dev);
0718
0719     dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
0720
0721     /* Configure iterator */
0722
0723     cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0724                            &cofs->dmem.iterator,
0725                            sizeof(*cfg_iter), vaddr);
0726     if (!cfg_iter)
0727         goto bad_firmware;
0728
0729     frame_sp_info = &cfg_iter->input_info;
0730     frame_sp_info->res.width    = css_queue_in->fmt.mpix.width;
0731     frame_sp_info->res.height   = css_queue_in->fmt.mpix.height;
0732     frame_sp_info->padded_width = css_queue_in->width_pad;
0733     frame_sp_info->format       = css_queue_in->css_fmt->frame_format;
0734     frame_sp_info->raw_bit_depth    = css_queue_in->css_fmt->bit_depth;
0735     frame_sp_info->raw_bayer_order  = css_queue_in->css_fmt->bayer_order;
0736     frame_sp_info->raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0737
0738     frame_sp_info = &cfg_iter->internal_info;
0739     frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
0740     frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
0741     frame_sp_info->padded_width = bds_width_pad;
0742     frame_sp_info->format       = css_queue_out->css_fmt->frame_format;
0743     frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
0744     frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
0745     frame_sp_info->raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0746
0747     frame_sp_info = &cfg_iter->output_info;
0748     frame_sp_info->res.width    = css_queue_out->fmt.mpix.width;
0749     frame_sp_info->res.height   = css_queue_out->fmt.mpix.height;
0750     frame_sp_info->padded_width = css_queue_out->width_pad;
0751     frame_sp_info->format       = css_queue_out->css_fmt->frame_format;
0752     frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
0753     frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
0754     frame_sp_info->raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0755
0756     frame_sp_info = &cfg_iter->vf_info;
0757     frame_sp_info->res.width    = css_queue_vf->fmt.mpix.width;
0758     frame_sp_info->res.height   = css_queue_vf->fmt.mpix.height;
0759     frame_sp_info->padded_width = css_queue_vf->width_pad;
0760     frame_sp_info->format       = css_queue_vf->css_fmt->frame_format;
0761     frame_sp_info->raw_bit_depth    = css_queue_vf->css_fmt->bit_depth;
0762     frame_sp_info->raw_bayer_order  = css_queue_vf->css_fmt->bayer_order;
0763     frame_sp_info->raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0764
0765     cfg_iter->dvs_envelope.width =
0766                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
0767     cfg_iter->dvs_envelope.height =
0768                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
0769
0770     /* Configure reference (delay) frames */
0771
0772     cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0773                           &cofs->dmem.ref,
0774                           sizeof(*cfg_ref), vaddr);
0775     if (!cfg_ref)
0776         goto bad_firmware;
0777
0778     cfg_ref->port_b.crop = 0;
0779     cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
0780     cfg_ref->port_b.width =
0781         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
0782     cfg_ref->port_b.stride =
0783         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
0784     cfg_ref->width_a_over_b =
0785                 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
0786     cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
0787     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
0788         cfg_ref->ref_frame_addr_y[i] =
0789             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
0790         cfg_ref->ref_frame_addr_c[i] =
0791             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
0792             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
0793             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
0794     }
0795     for (; i < IMGU_ABI_FRAMES_REF; i++) {
0796         cfg_ref->ref_frame_addr_y[i] = 0;
0797         cfg_ref->ref_frame_addr_c[i] = 0;
0798     }
0799
0800     /* Configure DVS (digital video stabilization) */
0801
0802     cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0803                           &cofs->dmem.dvs, sizeof(*cfg_dvs),
0804                           vaddr);
0805     if (!cfg_dvs)
0806         goto bad_firmware;
0807
0808     cfg_dvs->num_horizontal_blocks =
0809             ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
0810                        IMGU_DVS_BLOCK_W), 2);
0811     cfg_dvs->num_vertical_blocks =
0812             DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
0813                      IMGU_DVS_BLOCK_H);
0814
0815     /* Configure TNR (temporal noise reduction) */
0816
0817     if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
0818         cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0819                               &cofs->dmem.tnr3,
0820                               sizeof(*cfg_tnr),
0821                               vaddr);
0822         if (!cfg_tnr)
0823             goto bad_firmware;
0824
0825         cfg_tnr->port_b.crop = 0;
0826         cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
0827         cfg_tnr->port_b.width =
0828             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
0829         cfg_tnr->port_b.stride =
0830             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
0831         cfg_tnr->width_a_over_b =
0832             IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
0833         cfg_tnr->frame_height =
0834             css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
0835         cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
0836         for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
0837             cfg_tnr->frame_addr[i] =
0838                 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
0839                     .mem[i].daddr;
0840         for (; i < IMGU_ABI_FRAMES_TNR; i++)
0841             cfg_tnr->frame_addr[i] = 0;
0842     }
0843
0844     /* Configure ref dmem state parameters */
0845
0846     cfg = IMGU_ABI_PARAM_CLASS_STATE;
0847     vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
0848
0849     cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0850                             &sofs->dmem.ref,
0851                             sizeof(*cfg_ref_state),
0852                             vaddr);
0853     if (!cfg_ref_state)
0854         goto bad_firmware;
0855
0856     cfg_ref_state->ref_in_buf_idx = 0;
0857     cfg_ref_state->ref_out_buf_idx = 1;
0858
0859     /* Configure tnr dmem state parameters */
0860     if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
0861         cfg_tnr_state =
0862             imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
0863                             &sofs->dmem.tnr3,
0864                             sizeof(*cfg_tnr_state),
0865                             vaddr);
0866         if (!cfg_tnr_state)
0867             goto bad_firmware;
0868
0869         cfg_tnr_state->in_bufidx = 0;
0870         cfg_tnr_state->out_bufidx = 1;
0871         cfg_tnr_state->bypass_filter = 0;
0872         cfg_tnr_state->total_frame_counter = 0;
0873         for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
0874             cfg_tnr_state->buffer_frame_counter[i] = 0;
0875     }
0876
0877     /* Configure ISP stage */
0878
0879     isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
0880     memset(isp_stage, 0, sizeof(*isp_stage));
0881     isp_stage->blob_info = bi->blob;
0882     isp_stage->binary_info = bi->info.isp.sp;
0883     strscpy(isp_stage->binary_name,
0884         (char *)css->fwp + bi->blob.prog_name_offset,
0885         sizeof(isp_stage->binary_name));
0886     isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
0887     for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
0888         for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
0889             isp_stage->mem_initializers.params[i][j].address =
0890                     css_pipe->binary_params_cs[i - 1][j].daddr;
0891
0892     /* Configure SP stage */
0893
0894     sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
0895     memset(sp_stage, 0, sizeof(*sp_stage));
0896
0897     frames_sp = &sp_stage->frames;
0898     frames_sp->in.buf_attr = buffer_sp_init;
0899     for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
0900         frames_sp->out[i].buf_attr = buffer_sp_init;
0901     frames_sp->out_vf.buf_attr = buffer_sp_init;
0902     frames_sp->s3a_buf = buffer_sp_init;
0903     frames_sp->dvs_buf = buffer_sp_init;
0904
0905     sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
0906     sp_stage->num = stage;
0907     sp_stage->isp_online = 0;
0908     sp_stage->isp_copy_vf = 0;
0909     sp_stage->isp_copy_output = 0;
0910
0911     sp_stage->enable.vf_output = css_pipe->vf_output_en;
0912
0913     frames_sp->effective_in_res.width =
0914                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
0915     frames_sp->effective_in_res.height =
0916                 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
0917
0918     frame_sp = &frames_sp->in;
0919     frame_sp->info.res.width    = css_queue_in->fmt.mpix.width;
0920     frame_sp->info.res.height   = css_queue_in->fmt.mpix.height;
0921     frame_sp->info.padded_width = css_queue_in->width_pad;
0922     frame_sp->info.format       = css_queue_in->css_fmt->frame_format;
0923     frame_sp->info.raw_bit_depth    = css_queue_in->css_fmt->bit_depth;
0924     frame_sp->info.raw_bayer_order  = css_queue_in->css_fmt->bayer_order;
0925     frame_sp->info.raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0926     frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
0927     frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
0928
0929     frame_sp = &frames_sp->out[0];
0930     frame_sp->info.res.width    = css_queue_out->fmt.mpix.width;
0931     frame_sp->info.res.height   = css_queue_out->fmt.mpix.height;
0932     frame_sp->info.padded_width = css_queue_out->width_pad;
0933     frame_sp->info.format       = css_queue_out->css_fmt->frame_format;
0934     frame_sp->info.raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
0935     frame_sp->info.raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
0936     frame_sp->info.raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0937     frame_sp->planes.nv.uv.offset   = css_queue_out->width_pad *
0938                       css_queue_out->fmt.mpix.height;
0939     frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
0940     frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
0941
0942     frame_sp = &frames_sp->out[1];
0943     frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
0944
0945     frame_sp_info = &frames_sp->internal_frame_info;
0946     frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
0947     frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
0948     frame_sp_info->padded_width = bds_width_pad;
0949     frame_sp_info->format       = css_queue_out->css_fmt->frame_format;
0950     frame_sp_info->raw_bit_depth    = css_queue_out->css_fmt->bit_depth;
0951     frame_sp_info->raw_bayer_order  = css_queue_out->css_fmt->bayer_order;
0952     frame_sp_info->raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0953
0954     frame_sp = &frames_sp->out_vf;
0955     frame_sp->info.res.width    = css_queue_vf->fmt.mpix.width;
0956     frame_sp->info.res.height   = css_queue_vf->fmt.mpix.height;
0957     frame_sp->info.padded_width = css_queue_vf->width_pad;
0958     frame_sp->info.format       = css_queue_vf->css_fmt->frame_format;
0959     frame_sp->info.raw_bit_depth    = css_queue_vf->css_fmt->bit_depth;
0960     frame_sp->info.raw_bayer_order  = css_queue_vf->css_fmt->bayer_order;
0961     frame_sp->info.raw_type     = IMGU_ABI_RAW_TYPE_BAYER;
0962     frame_sp->planes.yuv.u.offset   = css_queue_vf->width_pad *
0963                       css_queue_vf->fmt.mpix.height;
0964     frame_sp->planes.yuv.v.offset   = css_queue_vf->width_pad *
0965                       css_queue_vf->fmt.mpix.height * 5 / 4;
0966     frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
0967     frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
0968
0969     frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
0970     frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
0971
0972     frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
0973     frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
0974
0975     sp_stage->dvs_envelope.width =
0976                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
0977     sp_stage->dvs_envelope.height =
0978                 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
0979
0980     sp_stage->isp_pipe_version =
0981                 bi->info.isp.sp.pipeline.isp_pipe_version;
0982     sp_stage->isp_deci_log_factor =
0983             clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
0984                       IMGU_MAX_BQ_GRID_WIDTH),
0985                   fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
0986                       IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
0987     sp_stage->isp_vf_downscale_bits = 0;
0988     sp_stage->if_config_index = 255;
0989     sp_stage->sp_enable_xnr = 0;
0990     sp_stage->num_stripes = stripes;
0991     sp_stage->enable.s3a = 1;
0992     sp_stage->enable.dvs_stats = 0;
0993
0994     sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
0995     sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
0996     sp_stage->isp_stage_addr =
0997         css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
0998
0999     /* Configure SP group */
1000
1001     sp_group = css->xmem_sp_group_ptrs.vaddr;
1002     memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1003
1004     sp_group->pipe[pipe].num_stages = 1;
1005     sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1006     sp_group->pipe[pipe].thread_id = pipe;
1007     sp_group->pipe[pipe].pipe_num = pipe;
1008     sp_group->pipe[pipe].num_execs = -1;
1009     sp_group->pipe[pipe].pipe_qos_config = -1;
1010     sp_group->pipe[pipe].required_bds_factor = 0;
1011     sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1012     sp_group->pipe[pipe].inout_port_config =
1013                     IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1014                     IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1015     sp_group->pipe[pipe].scaler_pp_lut = 0;
1016     sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1017     sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1018     sp_group->pipe[pipe].sp_stage_addr[stage] =
1019             css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1020     sp_group->pipe[pipe].pipe_config =
1021             bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1022     sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1023
1024     /* Initialize parameter pools */
1025
1026     if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1027                    sizeof(struct imgu_abi_parameter_set_info)) ||
1028         imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1029                    sizeof(struct imgu_abi_acc_param)) ||
1030         imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1031                    sizeof(struct imgu_abi_gdc_warp_param) *
1032                    3 * cfg_dvs->num_horizontal_blocks / 2 *
1033                    cfg_dvs->num_vertical_blocks) ||
1034         imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1035                    imgu_css_fw_obgrid_size(
1036                    &css->fwp->binary_header[css_pipe->bindex])))
1037         goto out_of_memory;
1038
1039     for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1040         if (imgu_css_pool_init(imgu,
1041                        &css_pipe->pool.binary_params_p[i],
1042                        bi->info.isp.sp.mem_initializers.params
1043                        [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1044             goto out_of_memory;
1045
1046     return 0;
1047
1048 bad_firmware:
1049     imgu_css_pipeline_cleanup(css, pipe);
1050     return -EPROTO;
1051
1052 out_of_memory:
1053     imgu_css_pipeline_cleanup(css, pipe);
1054     return -ENOMEM;
1055 }
1056
1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1058 {
1059     static const unsigned int sp;
1060     void __iomem *const base = css->base;
1061     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1062     struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1063                         bi->info.sp.host_sp_queue;
1064
1065     return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1066                 readb(&q->host2sp_evtq_info.end);
1067 }
1068
1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1070 static int imgu_css_queue_data(struct imgu_css *css,
1071                    int queue, int thread, u32 data)
1072 {
1073     static const unsigned int sp;
1074     void __iomem *const base = css->base;
1075     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1076     struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1077                         bi->info.sp.host_sp_queue;
1078     u8 size, start, end, end2;
1079
1080     if (queue >= 0) {
1081         size = readb(&q->host2sp_bufq_info[thread][queue].size);
1082         start = readb(&q->host2sp_bufq_info[thread][queue].start);
1083         end = readb(&q->host2sp_bufq_info[thread][queue].end);
1084     } else {
1085         size = readb(&q->host2sp_evtq_info.size);
1086         start = readb(&q->host2sp_evtq_info.start);
1087         end = readb(&q->host2sp_evtq_info.end);
1088     }
1089
1090     if (size == 0)
1091         return -EIO;
1092
1093     end2 = (end + 1) % size;
1094     if (end2 == start)
1095         return -EBUSY;  /* Queue full */
1096
1097     if (queue >= 0) {
1098         writel(data, &q->host2sp_bufq[thread][queue][end]);
1099         writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1100     } else {
1101         writel(data, &q->host2sp_evtq[end]);
1102         writeb(end2, &q->host2sp_evtq_info.end);
1103     }
1104
1105     return 0;
1106 }
1107
1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */
1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1110 {
1111     static const unsigned int sp;
1112     void __iomem *const base = css->base;
1113     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1114     struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1115                         bi->info.sp.host_sp_queue;
1116     u8 size, start, end, start2;
1117
1118     if (queue >= 0) {
1119         size = readb(&q->sp2host_bufq_info[queue].size);
1120         start = readb(&q->sp2host_bufq_info[queue].start);
1121         end = readb(&q->sp2host_bufq_info[queue].end);
1122     } else {
1123         size = readb(&q->sp2host_evtq_info.size);
1124         start = readb(&q->sp2host_evtq_info.start);
1125         end = readb(&q->sp2host_evtq_info.end);
1126     }
1127
1128     if (size == 0)
1129         return -EIO;
1130
1131     if (end == start)
1132         return -EBUSY;  /* Queue empty */
1133
1134     start2 = (start + 1) % size;
1135
1136     if (queue >= 0) {
1137         *data = readl(&q->sp2host_bufq[queue][start]);
1138         writeb(start2, &q->sp2host_bufq_info[queue].start);
1139     } else {
1140         int r;
1141
1142         *data = readl(&q->sp2host_evtq[start]);
1143         writeb(start2, &q->sp2host_evtq_info.start);
1144
1145         /* Acknowledge events dequeued from event queue */
1146         r = imgu_css_queue_data(css, queue, 0,
1147                     IMGU_ABI_EVENT_EVENT_DEQUEUED);
1148         if (r < 0)
1149             return r;
1150     }
1151
1152     return 0;
1153 }
1154
1155 /* Free binary-specific resources */
1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1157 {
1158     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1159     unsigned int i, j;
1160
1161     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1162
1163     for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1164         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1165             imgu_dmamap_free(imgu,
1166                      &css_pipe->binary_params_cs[j][i]);
1167
1168     j = IPU3_CSS_AUX_FRAME_REF;
1169     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1170         imgu_dmamap_free(imgu,
1171                  &css_pipe->aux_frames[j].mem[i]);
1172
1173     j = IPU3_CSS_AUX_FRAME_TNR;
1174     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1175         imgu_dmamap_free(imgu,
1176                  &css_pipe->aux_frames[j].mem[i]);
1177 }
1178
1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1180 {
1181     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1182     unsigned int i, j;
1183
1184     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1185
1186     for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1187          j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1188         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1189             if (!imgu_dmamap_alloc(imgu,
1190                            &css_pipe->binary_params_cs[j - 1][i],
1191                            CSS_ABI_SIZE))
1192                 goto out_of_memory;
1193
1194     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1195         if (!imgu_dmamap_alloc(imgu,
1196                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].
1197                        mem[i], CSS_BDS_SIZE))
1198             goto out_of_memory;
1199
1200     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1201         if (!imgu_dmamap_alloc(imgu,
1202                        &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].
1203                        mem[i], CSS_GDC_SIZE))
1204             goto out_of_memory;
1205
1206     return 0;
1207
1208 out_of_memory:
1209     imgu_css_binary_cleanup(css, pipe);
1210     return -ENOMEM;
1211 }
1212
1213 /* allocate binary-specific resources */
1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1215 {
1216     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1217     struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1218     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1219     int i, j, size;
1220     static const int BYPC = 2;  /* Bytes per component */
1221     unsigned int w, h;
1222
1223     /* Allocate parameter memory blocks for this binary */
1224
1225     for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1226         for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1227             if (imgu_css_dma_buffer_resize(
1228                 imgu,
1229                 &css_pipe->binary_params_cs[j - 1][i],
1230                 bi->info.isp.sp.mem_initializers.params[j][i].size))
1231                 goto out_of_memory;
1232         }
1233
1234     /* Allocate internal frame buffers */
1235
1236     /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1237     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1238     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1239                     css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1240     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1241                 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1242                       IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1243     h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1244     w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1245           2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1246     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1247         css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1248     size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1249     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1250         if (imgu_css_dma_buffer_resize(
1251             imgu,
1252             &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1253             size))
1254             goto out_of_memory;
1255
1256     /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1257     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1258     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1259             roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1260                 bi->info.isp.sp.block.block_width *
1261                 IPU3_UAPI_ISP_VEC_ELEMS);
1262     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1263             roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1264                 bi->info.isp.sp.block.output_block_height);
1265
1266     w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1267     css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1268     h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1269     size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */
1270     for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1271         if (imgu_css_dma_buffer_resize(
1272             imgu,
1273             &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1274             size))
1275             goto out_of_memory;
1276
1277     return 0;
1278
1279 out_of_memory:
1280     imgu_css_binary_cleanup(css, pipe);
1281     return -ENOMEM;
1282 }
1283
1284 int imgu_css_start_streaming(struct imgu_css *css)
1285 {
1286     u32 data;
1287     int r, pipe;
1288
1289     if (css->streaming)
1290         return -EPROTO;
1291
1292     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1293         r = imgu_css_binary_setup(css, pipe);
1294         if (r < 0)
1295             return r;
1296     }
1297
1298     r = imgu_css_hw_init(css);
1299     if (r < 0)
1300         return r;
1301
1302     r = imgu_css_hw_start(css);
1303     if (r < 0)
1304         goto fail;
1305
1306     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1307         r = imgu_css_pipeline_init(css, pipe);
1308         if (r < 0)
1309             goto fail;
1310     }
1311
1312     css->streaming = true;
1313
1314     imgu_css_hw_enable_irq(css);
1315
1316     /* Initialize parameters to default */
1317     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1318         r = imgu_css_set_parameters(css, pipe, NULL);
1319         if (r < 0)
1320             goto fail;
1321     }
1322
1323     while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1324         ;
1325     if (r != -EBUSY)
1326         goto fail;
1327
1328     while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1329         ;
1330     if (r != -EBUSY)
1331         goto fail;
1332
1333     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1334         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1335                     IMGU_ABI_EVENT_START_STREAM |
1336                     pipe << 16);
1337         if (r < 0)
1338             goto fail;
1339     }
1340
1341     return 0;
1342
1343 fail:
1344     css->streaming = false;
1345     imgu_css_hw_cleanup(css);
1346     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1347         imgu_css_pipeline_cleanup(css, pipe);
1348         imgu_css_binary_cleanup(css, pipe);
1349     }
1350
1351     return r;
1352 }
1353
1354 void imgu_css_stop_streaming(struct imgu_css *css)
1355 {
1356     struct imgu_css_buffer *b, *b0;
1357     int q, r, pipe;
1358
1359     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1360         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1361                     IMGU_ABI_EVENT_STOP_STREAM);
1362         if (r < 0)
1363             dev_warn(css->dev, "failed on stop stream event\n");
1364     }
1365
1366     if (!css->streaming)
1367         return;
1368
1369     imgu_css_hw_stop(css);
1370
1371     imgu_css_hw_cleanup(css);
1372
1373     for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1374         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1375
1376         imgu_css_pipeline_cleanup(css, pipe);
1377
1378         spin_lock(&css_pipe->qlock);
1379         for (q = 0; q < IPU3_CSS_QUEUES; q++)
1380             list_for_each_entry_safe(b, b0,
1381                          &css_pipe->queue[q].bufs,
1382                          list) {
1383                 b->state = IPU3_CSS_BUFFER_FAILED;
1384                 list_del(&b->list);
1385             }
1386         spin_unlock(&css_pipe->qlock);
1387     }
1388
1389     css->streaming = false;
1390 }
1391
1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1393 {
1394     int q;
1395     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1396
1397     spin_lock(&css_pipe->qlock);
1398     for (q = 0; q < IPU3_CSS_QUEUES; q++)
1399         if (!list_empty(&css_pipe->queue[q].bufs))
1400             break;
1401     spin_unlock(&css_pipe->qlock);
1402     return (q == IPU3_CSS_QUEUES);
1403 }
1404
1405 bool imgu_css_queue_empty(struct imgu_css *css)
1406 {
1407     unsigned int pipe;
1408     bool ret = false;
1409
1410     for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1411         ret &= imgu_css_pipe_queue_empty(css, pipe);
1412
1413     return ret;
1414 }
1415
1416 bool imgu_css_is_streaming(struct imgu_css *css)
1417 {
1418     return css->streaming;
1419 }
1420
1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1422 {
1423     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1424     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1425     unsigned int p, q, i;
1426
1427     /* Allocate and map common structures with imgu hardware */
1428     for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1429         for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1430             if (!imgu_dmamap_alloc(imgu,
1431                            &css_pipe->
1432                            xmem_sp_stage_ptrs[p][i],
1433                            sizeof(struct imgu_abi_sp_stage)))
1434                 return -ENOMEM;
1435             if (!imgu_dmamap_alloc(imgu,
1436                            &css_pipe->
1437                            xmem_isp_stage_ptrs[p][i],
1438                            sizeof(struct imgu_abi_isp_stage)))
1439                 return -ENOMEM;
1440         }
1441
1442     if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1443                    ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1444                      IMGU_ABI_ISP_DDR_WORD_BYTES)))
1445         return -ENOMEM;
1446
1447     for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1448         unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1449
1450         for (i = 0; i < abi_buf_num; i++)
1451             if (!imgu_dmamap_alloc(imgu,
1452                            &css_pipe->abi_buffers[q][i],
1453                            sizeof(struct imgu_abi_buffer)))
1454                 return -ENOMEM;
1455     }
1456
1457     if (imgu_css_binary_preallocate(css, pipe)) {
1458         imgu_css_binary_cleanup(css, pipe);
1459         return -ENOMEM;
1460     }
1461
1462     return 0;
1463 }
1464
1465 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1466 {
1467     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1468     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1469     unsigned int p, q, i, abi_buf_num;
1470
1471     imgu_css_binary_cleanup(css, pipe);
1472
1473     for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1474         abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1475         for (i = 0; i < abi_buf_num; i++)
1476             imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1477     }
1478
1479     for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1480         for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1481             imgu_dmamap_free(imgu,
1482                      &css_pipe->xmem_sp_stage_ptrs[p][i]);
1483             imgu_dmamap_free(imgu,
1484                      &css_pipe->xmem_isp_stage_ptrs[p][i]);
1485         }
1486
1487     imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1488 }
1489
1490 void imgu_css_cleanup(struct imgu_css *css)
1491 {
1492     struct imgu_device *imgu = dev_get_drvdata(css->dev);
1493     unsigned int pipe;
1494
1495     imgu_css_stop_streaming(css);
1496     for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1497         imgu_css_pipe_cleanup(css, pipe);
1498     imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1499     imgu_css_fw_cleanup(css);
1500 }
1501
1502 int imgu_css_init(struct device *dev, struct imgu_css *css,
1503           void __iomem *base, int length)
1504 {
1505     struct imgu_device *imgu = dev_get_drvdata(dev);
1506     int r, q, pipe;
1507
1508     /* Initialize main data structure */
1509     css->dev = dev;
1510     css->base = base;
1511     css->iomem_length = length;
1512
1513     for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1514         struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1515
1516         css_pipe->vf_output_en = false;
1517         spin_lock_init(&css_pipe->qlock);
1518         css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1519         css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1520         for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1521             r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1522             if (r)
1523                 return r;
1524         }
1525         r = imgu_css_map_init(css, pipe);
1526         if (r) {
1527             imgu_css_cleanup(css);
1528             return r;
1529         }
1530     }
1531     if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1532                    sizeof(struct imgu_abi_sp_group)))
1533         return -ENOMEM;
1534
1535     r = imgu_css_fw_init(css);
1536     if (r)
1537         return r;
1538
1539     return 0;
1540 }
1541
1542 static u32 imgu_css_adjust(u32 res, u32 align)
1543 {
1544     u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1545
1546     return DIV_ROUND_CLOSEST(val, align) * align;
1547 }
1548
1549 /* Select a binary matching the required resolutions and formats */
1550 static int imgu_css_find_binary(struct imgu_css *css,
1551                 unsigned int pipe,
1552                 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1553                 struct v4l2_rect rects[IPU3_CSS_RECTS])
1554 {
1555     const int binary_nr = css->fwp->file_header.binary_nr;
1556     unsigned int binary_mode =
1557         (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1558         IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1559     const struct v4l2_pix_format_mplane *in =
1560                     &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1561     const struct v4l2_pix_format_mplane *out =
1562                     &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1563     const struct v4l2_pix_format_mplane *vf =
1564                     &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1565     u32 stripe_w = 0, stripe_h = 0;
1566     const char *name;
1567     int i, j;
1568
1569     if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1570         return -EINVAL;
1571
1572     /* Find out the strip size boundary */
1573     for (i = 0; i < binary_nr; i++) {
1574         struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1575
1576         u32 max_width = bi->info.isp.sp.output.max_width;
1577         u32 max_height = bi->info.isp.sp.output.max_height;
1578
1579         if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1580             stripe_w = stripe_w ?
1581                 min(stripe_w, max_width) : max_width;
1582             stripe_h = stripe_h ?
1583                 min(stripe_h, max_height) : max_height;
1584         }
1585     }
1586
1587     for (i = 0; i < binary_nr; i++) {
1588         struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1589         enum imgu_abi_frame_format q_fmt;
1590
1591         name = (void *)css->fwp + bi->blob.prog_name_offset;
1592
1593         /* Check that binary supports memory-to-memory processing */
1594         if (bi->info.isp.sp.input.source !=
1595             IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1596             continue;
1597
1598         /* Check that binary supports raw10 input */
1599         if (!bi->info.isp.sp.enable.input_feeder &&
1600             !bi->info.isp.sp.enable.input_raw)
1601             continue;
1602
1603         /* Check binary mode */
1604         if (bi->info.isp.sp.pipeline.mode != binary_mode)
1605             continue;
1606
1607         /* Since input is RGGB bayer, need to process colors */
1608         if (bi->info.isp.sp.enable.luma_only)
1609             continue;
1610
1611         if (in->width < bi->info.isp.sp.input.min_width ||
1612             in->width > bi->info.isp.sp.input.max_width ||
1613             in->height < bi->info.isp.sp.input.min_height ||
1614             in->height > bi->info.isp.sp.input.max_height)
1615             continue;
1616
1617         if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1618             if (bi->info.isp.num_output_pins <= 0)
1619                 continue;
1620
1621             q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1622             for (j = 0; j < bi->info.isp.num_output_formats; j++)
1623                 if (bi->info.isp.output_formats[j] == q_fmt)
1624                     break;
1625             if (j >= bi->info.isp.num_output_formats)
1626                 continue;
1627
1628             if (out->width < bi->info.isp.sp.output.min_width ||
1629                 out->width > bi->info.isp.sp.output.max_width ||
1630                 out->height < bi->info.isp.sp.output.min_height ||
1631                 out->height > bi->info.isp.sp.output.max_height)
1632                 continue;
1633
1634             if (out->width > bi->info.isp.sp.internal.max_width ||
1635                 out->height > bi->info.isp.sp.internal.max_height)
1636                 continue;
1637         }
1638
1639         if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1640             if (bi->info.isp.num_output_pins <= 1)
1641                 continue;
1642
1643             q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1644             for (j = 0; j < bi->info.isp.num_output_formats; j++)
1645                 if (bi->info.isp.output_formats[j] == q_fmt)
1646                     break;
1647             if (j >= bi->info.isp.num_output_formats)
1648                 continue;
1649
1650             if (vf->width < bi->info.isp.sp.output.min_width ||
1651                 vf->width > bi->info.isp.sp.output.max_width ||
1652                 vf->height < bi->info.isp.sp.output.min_height ||
1653                 vf->height > bi->info.isp.sp.output.max_height)
1654                 continue;
1655         }
1656
1657         /* All checks passed, select the binary */
1658         dev_dbg(css->dev, "using binary %s id = %u\n", name,
1659             bi->info.isp.sp.id);
1660         return i;
1661     }
1662
1663     /* Can not find suitable binary for these parameters */
1664     return -EINVAL;
1665 }
1666
1667 /*
1668  * Check that there is a binary matching requirements. Parameters may be
1669  * NULL indicating disabled input/output. Return negative if given
1670  * parameters can not be supported or on error, zero or positive indicating
1671  * found binary number. May modify the given parameters if not exact match
1672  * is found.
1673  */
1674 int imgu_css_fmt_try(struct imgu_css *css,
1675              struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1676              struct v4l2_rect *rects[IPU3_CSS_RECTS],
1677              unsigned int pipe)
1678 {
1679     static const u32 EFF_ALIGN_W = 2;
1680     static const u32 BDS_ALIGN_W = 4;
1681     static const u32 OUT_ALIGN_W = 8;
1682     static const u32 OUT_ALIGN_H = 4;
1683     static const u32 VF_ALIGN_W  = 2;
1684     static const char *qnames[IPU3_CSS_QUEUES] = {
1685         [IPU3_CSS_QUEUE_IN] = "in",
1686         [IPU3_CSS_QUEUE_PARAMS]    = "params",
1687         [IPU3_CSS_QUEUE_OUT] = "out",
1688         [IPU3_CSS_QUEUE_VF] = "vf",
1689         [IPU3_CSS_QUEUE_STAT_3A]   = "3a",
1690     };
1691     static const char *rnames[IPU3_CSS_RECTS] = {
1692         [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1693         [IPU3_CSS_RECT_BDS]       = "bayer-domain scaled resolution",
1694         [IPU3_CSS_RECT_ENVELOPE]  = "DVS envelope size",
1695         [IPU3_CSS_RECT_GDC]  = "GDC output res",
1696     };
1697     struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1698     struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1699     struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1700     struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1701     struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1702     struct imgu_css_queue *q;
1703     struct v4l2_pix_format_mplane *in, *out, *vf;
1704     int i, s, ret;
1705
1706     q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1707     if (!q)
1708         return -ENOMEM;
1709
1710     in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1711     out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1712     vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1713
1714     /* Adjust all formats, get statistics buffer sizes and formats */
1715     for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1716         if (fmts[i])
1717             dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1718                 qnames[i], fmts[i]->width, fmts[i]->height,
1719                 fmts[i]->pixelformat);
1720         else
1721             dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1722                 qnames[i]);
1723         if (imgu_css_queue_init(&q[i], fmts[i],
1724                     IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1725             dev_notice(css->dev, "can not initialize queue %s\n",
1726                    qnames[i]);
1727             ret = -EINVAL;
1728             goto out;
1729         }
1730     }
1731     for (i = 0; i < IPU3_CSS_RECTS; i++) {
1732         if (rects[i]) {
1733             dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1734                 rnames[i], rects[i]->width, rects[i]->height);
1735             r[i].width  = rects[i]->width;
1736             r[i].height = rects[i]->height;
1737         } else {
1738             dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1739                 rnames[i]);
1740         }
1741         /* For now, force known good resolutions */
1742         r[i].left = 0;
1743         r[i].top  = 0;
1744     }
1745
1746     /* Always require one input and vf only if out is also enabled */
1747     if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1748         !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1749         dev_warn(css->dev, "required queues are disabled\n");
1750         ret = -EINVAL;
1751         goto out;
1752     }
1753
1754     if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1755         out->width = in->width;
1756         out->height = in->height;
1757     }
1758     if (eff->width <= 0 || eff->height <= 0) {
1759         eff->width = in->width;
1760         eff->height = in->height;
1761     }
1762     if (bds->width <= 0 || bds->height <= 0) {
1763         bds->width = out->width;
1764         bds->height = out->height;
1765     }
1766     if (gdc->width <= 0 || gdc->height <= 0) {
1767         gdc->width = out->width;
1768         gdc->height = out->height;
1769     }
1770
1771     in->width   = imgu_css_adjust(in->width, 1);
1772     in->height  = imgu_css_adjust(in->height, 1);
1773     eff->width  = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1774     eff->height = imgu_css_adjust(eff->height, 1);
1775     bds->width  = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1776     bds->height = imgu_css_adjust(bds->height, 1);
1777     gdc->width  = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1778     gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1779     out->width  = imgu_css_adjust(out->width, OUT_ALIGN_W);
1780     out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1781     vf->width   = imgu_css_adjust(vf->width, VF_ALIGN_W);
1782     vf->height  = imgu_css_adjust(vf->height, 1);
1783
1784     s = (bds->width - gdc->width) / 2;
1785     env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1786     s = (bds->height - gdc->height) / 2;
1787     env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1788
1789     ret = imgu_css_find_binary(css, pipe, q, r);
1790     if (ret < 0) {
1791         dev_err(css->dev, "failed to find suitable binary\n");
1792         ret = -EINVAL;
1793         goto out;
1794     }
1795     css->pipes[pipe].bindex = ret;
1796
1797     dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1798         css->pipes[pipe].bindex, pipe);
1799
1800     /* Final adjustment and set back the queried formats */
1801     for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1802         if (fmts[i]) {
1803             if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1804                         IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1805                 dev_err(css->dev,
1806                     "final resolution adjustment failed\n");
1807                 ret = -EINVAL;
1808                 goto out;
1809             }
1810             *fmts[i] = q[i].fmt.mpix;
1811         }
1812     }
1813
1814     for (i = 0; i < IPU3_CSS_RECTS; i++)
1815         if (rects[i])
1816             *rects[i] = r[i];
1817
1818     dev_dbg(css->dev,
1819         "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1820          in->width, in->height, eff->width, eff->height,
1821          bds->width, bds->height, gdc->width, gdc->height,
1822          out->width, out->height, vf->width, vf->height);
1823
1824     ret = 0;
1825 out:
1826     kfree(q);
1827     return ret;
1828 }
1829
1830 int imgu_css_fmt_set(struct imgu_css *css,
1831              struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1832              struct v4l2_rect *rects[IPU3_CSS_RECTS],
1833              unsigned int pipe)
1834 {
1835     struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1836     struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1837     int i, r;
1838     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1839
1840     for (i = 0; i < IPU3_CSS_RECTS; i++) {
1841         if (rects[i])
1842             rect_data[i] = *rects[i];
1843         else
1844             memset(&rect_data[i], 0, sizeof(rect_data[i]));
1845         all_rects[i] = &rect_data[i];
1846     }
1847     r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1848     if (r < 0)
1849         return r;
1850
1851     for (i = 0; i < IPU3_CSS_QUEUES; i++)
1852         if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1853                     IPU3_CSS_QUEUE_TO_FLAGS(i)))
1854             return -EINVAL;
1855     for (i = 0; i < IPU3_CSS_RECTS; i++) {
1856         css_pipe->rect[i] = rect_data[i];
1857         if (rects[i])
1858             *rects[i] = rect_data[i];
1859     }
1860
1861     return 0;
1862 }
1863
1864 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1865 {
1866     switch (fmt->dataformat) {
1867     case V4L2_META_FMT_IPU3_PARAMS:
1868         fmt->buffersize = sizeof(struct ipu3_uapi_params);
1869
1870         /*
1871          * Sanity check for the parameter struct size. This must
1872          * not change!
1873          */
1874         BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1875
1876         break;
1877     case V4L2_META_FMT_IPU3_STAT_3A:
1878         fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1879         break;
1880     default:
1881         return -EINVAL;
1882     }
1883
1884     return 0;
1885 }
1886
1887 /*
1888  * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1889  * called for the buffer. May be called from interrupt context.
1890  * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1891  * code on error conditions.
1892  */
1893 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1894                struct imgu_css_buffer *b)
1895 {
1896     struct imgu_abi_buffer *abi_buf;
1897     struct imgu_addr_t *buf_addr;
1898     u32 data;
1899     int r;
1900     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1901
1902     if (!css->streaming)
1903         return -EPROTO; /* CSS or buffer in wrong state */
1904
1905     if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1906         return -EINVAL;
1907
1908     b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1909                       pipe);
1910
1911     if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1912         return -EIO;
1913     abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1914
1915     /* Fill struct abi_buffer for firmware */
1916     memset(abi_buf, 0, sizeof(*abi_buf));
1917
1918     buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1919     *(imgu_addr_t *)buf_addr = b->daddr;
1920
1921     if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1922         abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1923
1924     if (b->queue == IPU3_CSS_QUEUE_OUT)
1925         abi_buf->payload.frame.padded_width =
1926                 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1927
1928     if (b->queue == IPU3_CSS_QUEUE_VF)
1929         abi_buf->payload.frame.padded_width =
1930                     css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1931
1932     spin_lock(&css_pipe->qlock);
1933     list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1934     spin_unlock(&css_pipe->qlock);
1935     b->state = IPU3_CSS_BUFFER_QUEUED;
1936
1937     data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1938     r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1939                 pipe, data);
1940     if (r < 0)
1941         goto queueing_failed;
1942
1943     data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1944                           imgu_css_queues[b->queue].qid);
1945     r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1946     if (r < 0)
1947         goto queueing_failed;
1948
1949     dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1950         b, b->queue, pipe);
1951
1952     return 0;
1953
1954 queueing_failed:
1955     b->state = (r == -EBUSY || r == -EAGAIN) ?
1956         IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1957     list_del(&b->list);
1958
1959     return r;
1960 }
1961
1962 /*
1963  * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1964  * should be called again, or -EBUSY which means that there are no more
1965  * buffers available. May be called from interrupt context.
1966  */
1967 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1968 {
1969     static const unsigned char evtype_to_queue[] = {
1970         [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1971         [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1972         [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1973         [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1974     };
1975     struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1976     u32 event, daddr;
1977     int evtype, pipe, pipeid, queue, qid, r;
1978     struct imgu_css_pipe *css_pipe;
1979
1980     if (!css->streaming)
1981         return ERR_PTR(-EPROTO);
1982
1983     r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1984     if (r < 0)
1985         return ERR_PTR(r);
1986
1987     evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1988           IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1989
1990     switch (evtype) {
1991     case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1992     case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1993     case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1994     case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1995         pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1996             IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1997         pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1998             IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1999         queue = evtype_to_queue[evtype];
2000         qid = imgu_css_queues[queue].qid;
2001
2002         if (pipe >= IMGU_MAX_PIPE_NUM) {
2003             dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2004             return ERR_PTR(-EIO);
2005         }
2006
2007         if (qid >= IMGU_ABI_QUEUE_NUM) {
2008             dev_err(css->dev, "Invalid qid: %i\n", qid);
2009             return ERR_PTR(-EIO);
2010         }
2011         css_pipe = &css->pipes[pipe];
2012         dev_dbg(css->dev,
2013             "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2014             event, queue, pipe, pipeid);
2015
2016         r = imgu_css_dequeue_data(css, qid, &daddr);
2017         if (r < 0) {
2018             dev_err(css->dev, "failed to dequeue buffer\n");
2019             /* Force real error, not -EBUSY */
2020             return ERR_PTR(-EIO);
2021         }
2022
2023         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2024                     IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2025         if (r < 0) {
2026             dev_err(css->dev, "failed to queue event\n");
2027             return ERR_PTR(-EIO);
2028         }
2029
2030         spin_lock(&css_pipe->qlock);
2031         if (list_empty(&css_pipe->queue[queue].bufs)) {
2032             spin_unlock(&css_pipe->qlock);
2033             dev_err(css->dev, "event on empty queue\n");
2034             return ERR_PTR(-EIO);
2035         }
2036         b = list_first_entry(&css_pipe->queue[queue].bufs,
2037                      struct imgu_css_buffer, list);
2038         if (queue != b->queue ||
2039             daddr != css_pipe->abi_buffers
2040             [b->queue][b->queue_pos].daddr) {
2041             spin_unlock(&css_pipe->qlock);
2042             dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2043             return ERR_PTR(-EIO);
2044         }
2045
2046         dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2047         b->pipe = pipe;
2048         b->state = IPU3_CSS_BUFFER_DONE;
2049         list_del(&b->list);
2050         spin_unlock(&css_pipe->qlock);
2051         break;
2052     case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2053         pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2054             IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2055         if (pipe >= IMGU_MAX_PIPE_NUM) {
2056             dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2057             return ERR_PTR(-EIO);
2058         }
2059
2060         css_pipe = &css->pipes[pipe];
2061         dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2062             event, pipe);
2063         break;
2064     case IMGU_ABI_EVTTYPE_TIMER:
2065         r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2066         if (r < 0)
2067             return ERR_PTR(r);
2068
2069         if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2070             IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2071             dev_dbg(css->dev, "event: timer\n");
2072         else
2073             dev_warn(css->dev, "half of timer event missing\n");
2074         break;
2075     case IMGU_ABI_EVTTYPE_FW_WARNING:
2076         dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2077         break;
2078     case IMGU_ABI_EVTTYPE_FW_ASSERT:
2079         dev_err(css->dev,
2080             "event: firmware assert 0x%x module_id %i line_no %i\n",
2081             event,
2082             (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2083             IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2084             swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2085                    IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2086         break;
2087     default:
2088         dev_warn(css->dev, "received unknown event 0x%x\n", event);
2089     }
2090
2091     return b;
2092 }
2093
2094 /*
2095  * Get a new set of parameters from pool and initialize them based on
2096  * the parameters params, gdc, and obgrid. Any of these may be NULL,
2097  * in which case the previously set parameters are used.
2098  * If parameters haven't been set previously, initialize from scratch.
2099  *
2100  * Return index to css->parameter_set_info which has the newly created
2101  * parameters or negative value on error.
2102  */
2103 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2104                 struct ipu3_uapi_params *set_params)
2105 {
2106     static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2107     struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2108     const int stage = 0;
2109     const struct imgu_fw_info *bi;
2110     int obgrid_size;
2111     unsigned int stripes, i;
2112     struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2113
2114     /* Destination buffers which are filled here */
2115     struct imgu_abi_parameter_set_info *param_set;
2116     struct imgu_abi_acc_param *acc = NULL;
2117     struct imgu_abi_gdc_warp_param *gdc = NULL;
2118     struct ipu3_uapi_obgrid_param *obgrid = NULL;
2119     const struct imgu_css_map *map;
2120     void *vmem0 = NULL;
2121     void *dmem0 = NULL;
2122
2123     enum imgu_abi_memories m;
2124     int r = -EBUSY;
2125
2126     if (!css->streaming)
2127         return -EPROTO;
2128
2129     dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2130
2131     bi = &css->fwp->binary_header[css_pipe->bindex];
2132     obgrid_size = imgu_css_fw_obgrid_size(bi);
2133     stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2134
2135     imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2136     param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2137                        0)->vaddr;
2138
2139     /* Get a new acc only if new parameters given, or none yet */
2140     map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2141     if (set_params || !map->vaddr) {
2142         imgu_css_pool_get(&css_pipe->pool.acc);
2143         map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2144         acc = map->vaddr;
2145     }
2146
2147     /* Get new VMEM0 only if needed, or none yet */
2148     m = IMGU_ABI_MEM_ISP_VMEM0;
2149     map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2150     if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2151                        set_params->use.tnr3_vmem_params ||
2152                        set_params->use.xnr3_vmem_params))) {
2153         imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2154         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2155         vmem0 = map->vaddr;
2156     }
2157
2158     /* Get new DMEM0 only if needed, or none yet */
2159     m = IMGU_ABI_MEM_ISP_DMEM0;
2160     map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2161     if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2162                        set_params->use.xnr3_dmem_params))) {
2163         imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2164         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2165         dmem0 = map->vaddr;
2166     }
2167
2168     /* Configure acc parameter cluster */
2169     if (acc) {
2170         /* get acc_old */
2171         map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2172         /* user acc */
2173         r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2174             set_params ? &set_params->acc_param : NULL);
2175         if (r < 0)
2176             goto fail;
2177     }
2178
2179     /* Configure late binding parameters */
2180     if (vmem0) {
2181         m = IMGU_ABI_MEM_ISP_VMEM0;
2182         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2183         r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2184                        map->vaddr, set_params);
2185         if (r < 0)
2186             goto fail;
2187     }
2188
2189     if (dmem0) {
2190         m = IMGU_ABI_MEM_ISP_DMEM0;
2191         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2192         r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2193                        map->vaddr, set_params);
2194         if (r < 0)
2195             goto fail;
2196     }
2197
2198     /* Get a new gdc only if a new gdc is given, or none yet */
2199     if (bi->info.isp.sp.enable.dvs_6axis) {
2200         unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2201         unsigned int g = IPU3_CSS_RECT_GDC;
2202         unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2203
2204         map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2205         if (!map->vaddr) {
2206             imgu_css_pool_get(&css_pipe->pool.gdc);
2207             map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2208             gdc = map->vaddr;
2209             imgu_css_cfg_gdc_table(map->vaddr,
2210                 css_pipe->aux_frames[a].bytesperline /
2211                 css_pipe->aux_frames[a].bytesperpixel,
2212                 css_pipe->aux_frames[a].height,
2213                 css_pipe->rect[g].width,
2214                 css_pipe->rect[g].height,
2215                 css_pipe->rect[e].width,
2216                 css_pipe->rect[e].height);
2217         }
2218     }
2219
2220     /* Get a new obgrid only if a new obgrid is given, or none yet */
2221     map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2222     if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2223         imgu_css_pool_get(&css_pipe->pool.obgrid);
2224         map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2225         obgrid = map->vaddr;
2226
2227         /* Configure optical black level grid (obgrid) */
2228         if (set_params && set_params->use.obgrid_param)
2229             for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2230                 obgrid[i] = set_params->obgrid_param;
2231         else
2232             memset(obgrid, 0, obgrid_size);
2233     }
2234
2235     /* Configure parameter set info, queued to `queue_id' */
2236
2237     memset(param_set, 0, sizeof(*param_set));
2238     map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2239     param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2240
2241     map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2242     param_set->mem_map.dvs_6axis_params_y = map->daddr;
2243
2244     for (i = 0; i < stripes; i++) {
2245         map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2246         param_set->mem_map.obgrid_tbl[i] =
2247             map->daddr + (obgrid_size / stripes) * i;
2248     }
2249
2250     for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2251         map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2252         param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2253     }
2254
2255     /* Then queue the new parameter buffer */
2256     map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2257     r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2258     if (r < 0)
2259         goto fail;
2260
2261     r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2262                 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2263                                    queue_id));
2264     if (r < 0)
2265         goto fail_no_put;
2266
2267     /* Finally dequeue all old parameter buffers */
2268
2269     do {
2270         u32 daddr;
2271
2272         r = imgu_css_dequeue_data(css, queue_id, &daddr);
2273         if (r == -EBUSY)
2274             break;
2275         if (r)
2276             goto fail_no_put;
2277         r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2278                     IMGU_ABI_EVENT_BUFFER_DEQUEUED
2279                     (queue_id));
2280         if (r < 0) {
2281             dev_err(css->dev, "failed to queue parameter event\n");
2282             goto fail_no_put;
2283         }
2284     } while (1);
2285
2286     return 0;
2287
2288 fail:
2289     /*
2290      * A failure, most likely the parameter queue was full.
2291      * Return error but continue streaming. User can try submitting new
2292      * parameters again later.
2293      */
2294
2295     imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2296     if (acc)
2297         imgu_css_pool_put(&css_pipe->pool.acc);
2298     if (gdc)
2299         imgu_css_pool_put(&css_pipe->pool.gdc);
2300     if (obgrid)
2301         imgu_css_pool_put(&css_pipe->pool.obgrid);
2302     if (vmem0)
2303         imgu_css_pool_put(
2304             &css_pipe->pool.binary_params_p
2305             [IMGU_ABI_MEM_ISP_VMEM0]);
2306     if (dmem0)
2307         imgu_css_pool_put(
2308             &css_pipe->pool.binary_params_p
2309             [IMGU_ABI_MEM_ISP_DMEM0]);
2310
2311 fail_no_put:
2312     return r;
2313 }
2314
2315 int imgu_css_irq_ack(struct imgu_css *css)
2316 {
2317     static const int NUM_SWIRQS = 3;
2318     struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2319     void __iomem *const base = css->base;
2320     u32 irq_status[IMGU_IRQCTRL_NUM];
2321     int i;
2322
2323     u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2324
2325     writel(imgu_status, base + IMGU_REG_INT_STATUS);
2326     for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2327         irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2328
2329     for (i = 0; i < NUM_SWIRQS; i++) {
2330         if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2331             /* SP SW interrupt */
2332             u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333                     bi->info.sp.output);
2334             u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2335                     bi->info.sp.output + 4 + 4 * i);
2336
2337             dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2338                 __func__, i, cnt, val);
2339         }
2340     }
2341
2342     for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2343         if (irq_status[i]) {
2344             writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2345             /* Wait for write to complete */
2346             readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2347         }
2348
2349     dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2350         __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2351         irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2352
2353     if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2354         return -ENOMSG;
2355
2356     return 0;
2357 }