drm/vmwgfx/vmwgfx_blit.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /**************************************************************************
0003  *
0004  * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
0005  * All Rights Reserved.
0006  *
0007  * Permission is hereby granted, free of charge, to any person obtaining a
0008  * copy of this software and associated documentation files (the
0009  * "Software"), to deal in the Software without restriction, including
0010  * without limitation the rights to use, copy, modify, merge, publish,
0011  * distribute, sub license, and/or sell copies of the Software, and to
0012  * permit persons to whom the Software is furnished to do so, subject to
0013  * the following conditions:
0014  *
0015  * The above copyright notice and this permission notice (including the
0016  * next paragraph) shall be included in all copies or substantial portions
0017  * of the Software.
0018  *
0019  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0020  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0021  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
0022  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
0023  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
0024  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
0025  * USE OR OTHER DEALINGS IN THE SOFTWARE.
0026  *
0027  **************************************************************************/
0028
0029 #include "vmwgfx_drv.h"
0030 #include <linux/highmem.h>
0031
0032 /*
0033  * Template that implements find_first_diff() for a generic
0034  * unsigned integer type. @size and return value are in bytes.
0035  */
0036 #define VMW_FIND_FIRST_DIFF(_type)           \
0037 static size_t vmw_find_first_diff_ ## _type      \
0038     (const _type * dst, const _type * src, size_t size)\
0039 {                            \
0040     size_t i;                    \
0041                              \
0042     for (i = 0; i < size; i += sizeof(_type)) {  \
0043         if (*dst++ != *src++)            \
0044             break;               \
0045     }                        \
0046                              \
0047     return i;                    \
0048 }
0049
0050
0051 /*
0052  * Template that implements find_last_diff() for a generic
0053  * unsigned integer type. Pointers point to the item following the
0054  * *end* of the area to be examined. @size and return value are in
0055  * bytes.
0056  */
0057 #define VMW_FIND_LAST_DIFF(_type)                   \
0058 static ssize_t vmw_find_last_diff_ ## _type(                \
0059     const _type * dst, const _type * src, size_t size)      \
0060 {                                   \
0061     while (size) {                          \
0062         if (*--dst != *--src)                   \
0063             break;                      \
0064                                     \
0065         size -= sizeof(_type);                  \
0066     }                               \
0067     return size;                            \
0068 }
0069
0070
0071 /*
0072  * Instantiate find diff functions for relevant unsigned integer sizes,
0073  * assuming that wider integers are faster (including aligning) up to the
0074  * architecture native width, which is assumed to be 32 bit unless
0075  * CONFIG_64BIT is defined.
0076  */
0077 VMW_FIND_FIRST_DIFF(u8);
0078 VMW_FIND_LAST_DIFF(u8);
0079
0080 VMW_FIND_FIRST_DIFF(u16);
0081 VMW_FIND_LAST_DIFF(u16);
0082
0083 VMW_FIND_FIRST_DIFF(u32);
0084 VMW_FIND_LAST_DIFF(u32);
0085
0086 #ifdef CONFIG_64BIT
0087 VMW_FIND_FIRST_DIFF(u64);
0088 VMW_FIND_LAST_DIFF(u64);
0089 #endif
0090
0091
0092 /* We use size aligned copies. This computes (addr - align(addr)) */
0093 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
0094
0095
0096 /*
0097  * Template to compute find_first_diff() for a certain integer type
0098  * including a head copy for alignment, and adjustment of parameters
0099  * for tail find or increased resolution find using an unsigned integer find
0100  * of smaller width. If finding is complete, and resolution is sufficient,
0101  * the macro executes a return statement. Otherwise it falls through.
0102  */
0103 #define VMW_TRY_FIND_FIRST_DIFF(_type)                  \
0104 do {                                    \
0105     unsigned int spill = SPILL(dst, _type);             \
0106     size_t diff_offs;                       \
0107                                     \
0108     if (spill && spill == SPILL(src, _type) &&          \
0109         sizeof(_type) - spill <= size) {                \
0110         spill = sizeof(_type) - spill;              \
0111         diff_offs = vmw_find_first_diff_u8(dst, src, spill);    \
0112         if (diff_offs < spill)                  \
0113             return round_down(offset + diff_offs, granularity); \
0114                                     \
0115         dst += spill;                       \
0116         src += spill;                       \
0117         size -= spill;                      \
0118         offset += spill;                    \
0119         spill = 0;                      \
0120     }                               \
0121     if (!spill && !SPILL(src, _type)) {             \
0122         size_t to_copy = size &  ~(sizeof(_type) - 1);      \
0123                                     \
0124         diff_offs = vmw_find_first_diff_ ## _type       \
0125             ((_type *) dst, (_type *) src, to_copy);    \
0126         if (diff_offs >= size || granularity == sizeof(_type))  \
0127             return (offset + diff_offs);            \
0128                                     \
0129         dst += diff_offs;                   \
0130         src += diff_offs;                   \
0131         size -= diff_offs;                  \
0132         offset += diff_offs;                    \
0133     }                               \
0134 } while (0)                             \
0135
0136
0137 /**
0138  * vmw_find_first_diff - find the first difference between dst and src
0139  *
0140  * @dst: The destination address
0141  * @src: The source address
0142  * @size: Number of bytes to compare
0143  * @granularity: The granularity needed for the return value in bytes.
0144  * return: The offset from find start where the first difference was
0145  * encountered in bytes. If no difference was found, the function returns
0146  * a value >= @size.
0147  */
0148 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
0149                   size_t granularity)
0150 {
0151     size_t offset = 0;
0152
0153     /*
0154      * Try finding with large integers if alignment allows, or we can
0155      * fix it. Fall through if we need better resolution or alignment
0156      * was bad.
0157      */
0158 #ifdef CONFIG_64BIT
0159     VMW_TRY_FIND_FIRST_DIFF(u64);
0160 #endif
0161     VMW_TRY_FIND_FIRST_DIFF(u32);
0162     VMW_TRY_FIND_FIRST_DIFF(u16);
0163
0164     return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
0165               granularity);
0166 }
0167
0168
0169 /*
0170  * Template to compute find_last_diff() for a certain integer type
0171  * including a tail copy for alignment, and adjustment of parameters
0172  * for head find or increased resolution find using an unsigned integer find
0173  * of smaller width. If finding is complete, and resolution is sufficient,
0174  * the macro executes a return statement. Otherwise it falls through.
0175  */
0176 #define VMW_TRY_FIND_LAST_DIFF(_type)                   \
0177 do {                                    \
0178     unsigned int spill = SPILL(dst, _type);             \
0179     ssize_t location;                       \
0180     ssize_t diff_offs;                      \
0181                                     \
0182     if (spill && spill <= size && spill == SPILL(src, _type)) { \
0183         diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
0184         if (diff_offs) {                    \
0185             location = size - spill + diff_offs - 1;    \
0186             return round_down(location, granularity);   \
0187         }                           \
0188                                     \
0189         dst -= spill;                       \
0190         src -= spill;                       \
0191         size -= spill;                      \
0192         spill = 0;                      \
0193     }                               \
0194     if (!spill && !SPILL(src, _type)) {             \
0195         size_t to_copy = round_down(size, sizeof(_type));   \
0196                                     \
0197         diff_offs = vmw_find_last_diff_ ## _type        \
0198             ((_type *) dst, (_type *) src, to_copy);    \
0199         location = size - to_copy + diff_offs - sizeof(_type);  \
0200         if (location < 0 || granularity == sizeof(_type))   \
0201             return location;                \
0202                                     \
0203         dst -= to_copy - diff_offs;             \
0204         src -= to_copy - diff_offs;             \
0205         size -= to_copy - diff_offs;                \
0206     }                               \
0207 } while (0)
0208
0209
0210 /**
0211  * vmw_find_last_diff - find the last difference between dst and src
0212  *
0213  * @dst: The destination address
0214  * @src: The source address
0215  * @size: Number of bytes to compare
0216  * @granularity: The granularity needed for the return value in bytes.
0217  * return: The offset from find start where the last difference was
0218  * encountered in bytes, or a negative value if no difference was found.
0219  */
0220 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
0221                   size_t granularity)
0222 {
0223     dst += size;
0224     src += size;
0225
0226 #ifdef CONFIG_64BIT
0227     VMW_TRY_FIND_LAST_DIFF(u64);
0228 #endif
0229     VMW_TRY_FIND_LAST_DIFF(u32);
0230     VMW_TRY_FIND_LAST_DIFF(u16);
0231
0232     return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
0233               granularity);
0234 }
0235
0236
0237 /**
0238  * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
0239  * struct vmw_diff_cpy.
0240  *
0241  * @diff: The struct vmw_diff_cpy closure argument (unused).
0242  * @dest: The copy destination.
0243  * @src: The copy source.
0244  * @n: Number of bytes to copy.
0245  */
0246 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
0247 {
0248     memcpy(dest, src, n);
0249 }
0250
0251
0252 /**
0253  * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
0254  *
0255  * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
0256  * @diff_offs: The offset from @diff->line_offset where the difference was
0257  * found.
0258  */
0259 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
0260 {
0261     size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
0262     struct drm_rect *rect = &diff->rect;
0263
0264     rect->x1 = min_t(int, rect->x1, offs);
0265     rect->x2 = max_t(int, rect->x2, offs + 1);
0266     rect->y1 = min_t(int, rect->y1, diff->line);
0267     rect->y2 = max_t(int, rect->y2, diff->line + 1);
0268 }
0269
0270 /**
0271  * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
0272  *
0273  * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
0274  * @dest: The copy destination.
0275  * @src: The copy source.
0276  * @n: Number of bytes to copy.
0277  *
0278  * In order to correctly track the modified content, the field @diff->line must
0279  * be pre-loaded with the current line number, the field @diff->line_offset must
0280  * be pre-loaded with the line offset in bytes where the copy starts, and
0281  * finally the field @diff->cpp need to be preloaded with the number of bytes
0282  * per unit in the horizontal direction of the area we're examining.
0283  * Typically bytes per pixel.
0284  * This is needed to know the needed granularity of the difference computing
0285  * operations. A higher cpp generally leads to faster execution at the cost of
0286  * bounding box width precision.
0287  */
0288 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
0289              size_t n)
0290 {
0291     ssize_t csize, byte_len;
0292
0293     if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
0294         return;
0295
0296     /* TODO: Possibly use a single vmw_find_first_diff per line? */
0297     csize = vmw_find_first_diff(dest, src, n, diff->cpp);
0298     if (csize < n) {
0299         vmw_adjust_rect(diff, csize);
0300         byte_len = diff->cpp;
0301
0302         /*
0303          * Starting from where first difference was found, find
0304          * location of last difference, and then copy.
0305          */
0306         diff->line_offset += csize;
0307         dest += csize;
0308         src += csize;
0309         n -= csize;
0310         csize = vmw_find_last_diff(dest, src, n, diff->cpp);
0311         if (csize >= 0) {
0312             byte_len += csize;
0313             vmw_adjust_rect(diff, csize);
0314         }
0315         memcpy(dest, src, byte_len);
0316     }
0317     diff->line_offset += n;
0318 }
0319
0320 /**
0321  * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
0322  *
0323  * @mapped_dst: Already mapped destination page index in @dst_pages.
0324  * @dst_addr: Kernel virtual address of mapped destination page.
0325  * @dst_pages: Array of destination bo pages.
0326  * @dst_num_pages: Number of destination bo pages.
0327  * @dst_prot: Destination bo page protection.
0328  * @mapped_src: Already mapped source page index in @dst_pages.
0329  * @src_addr: Kernel virtual address of mapped source page.
0330  * @src_pages: Array of source bo pages.
0331  * @src_num_pages: Number of source bo pages.
0332  * @src_prot: Source bo page protection.
0333  * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
0334  */
0335 struct vmw_bo_blit_line_data {
0336     u32 mapped_dst;
0337     u8 *dst_addr;
0338     struct page **dst_pages;
0339     u32 dst_num_pages;
0340     pgprot_t dst_prot;
0341     u32 mapped_src;
0342     u8 *src_addr;
0343     struct page **src_pages;
0344     u32 src_num_pages;
0345     pgprot_t src_prot;
0346     struct vmw_diff_cpy *diff;
0347 };
0348
0349 /**
0350  * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
0351  *
0352  * @d: Blit data as described above.
0353  * @dst_offset: Destination copy start offset from start of bo.
0354  * @src_offset: Source copy start offset from start of bo.
0355  * @bytes_to_copy: Number of bytes to copy in this line.
0356  */
0357 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
0358                 u32 dst_offset,
0359                 u32 src_offset,
0360                 u32 bytes_to_copy)
0361 {
0362     struct vmw_diff_cpy *diff = d->diff;
0363
0364     while (bytes_to_copy) {
0365         u32 copy_size = bytes_to_copy;
0366         u32 dst_page = dst_offset >> PAGE_SHIFT;
0367         u32 src_page = src_offset >> PAGE_SHIFT;
0368         u32 dst_page_offset = dst_offset & ~PAGE_MASK;
0369         u32 src_page_offset = src_offset & ~PAGE_MASK;
0370         bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
0371         bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
0372                          unmap_dst);
0373
0374         copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
0375         copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
0376
0377         if (unmap_src) {
0378             kunmap_atomic(d->src_addr);
0379             d->src_addr = NULL;
0380         }
0381
0382         if (unmap_dst) {
0383             kunmap_atomic(d->dst_addr);
0384             d->dst_addr = NULL;
0385         }
0386
0387         if (!d->dst_addr) {
0388             if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
0389                 return -EINVAL;
0390
0391             d->dst_addr =
0392                 kmap_atomic_prot(d->dst_pages[dst_page],
0393                          d->dst_prot);
0394             if (!d->dst_addr)
0395                 return -ENOMEM;
0396
0397             d->mapped_dst = dst_page;
0398         }
0399
0400         if (!d->src_addr) {
0401             if (WARN_ON_ONCE(src_page >= d->src_num_pages))
0402                 return -EINVAL;
0403
0404             d->src_addr =
0405                 kmap_atomic_prot(d->src_pages[src_page],
0406                          d->src_prot);
0407             if (!d->src_addr)
0408                 return -ENOMEM;
0409
0410             d->mapped_src = src_page;
0411         }
0412         diff->do_cpy(diff, d->dst_addr + dst_page_offset,
0413                  d->src_addr + src_page_offset, copy_size);
0414
0415         bytes_to_copy -= copy_size;
0416         dst_offset += copy_size;
0417         src_offset += copy_size;
0418     }
0419
0420     return 0;
0421 }
0422
0423 /**
0424  * vmw_bo_cpu_blit - in-kernel cpu blit.
0425  *
0426  * @dst: Destination buffer object.
0427  * @dst_offset: Destination offset of blit start in bytes.
0428  * @dst_stride: Destination stride in bytes.
0429  * @src: Source buffer object.
0430  * @src_offset: Source offset of blit start in bytes.
0431  * @src_stride: Source stride in bytes.
0432  * @w: Width of blit.
0433  * @h: Height of blit.
0434  * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
0435  * return: Zero on success. Negative error value on failure. Will print out
0436  * kernel warnings on caller bugs.
0437  *
0438  * Performs a CPU blit from one buffer object to another avoiding a full
0439  * bo vmap which may exhaust- or fragment vmalloc space.
0440  * On supported architectures (x86), we're using kmap_atomic which avoids
0441  * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
0442  * reference already set-up mappings.
0443  *
0444  * Neither of the buffer objects may be placed in PCI memory
0445  * (Fixed memory in TTM terminology) when using this function.
0446  */
0447 int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
0448             u32 dst_offset, u32 dst_stride,
0449             struct ttm_buffer_object *src,
0450             u32 src_offset, u32 src_stride,
0451             u32 w, u32 h,
0452             struct vmw_diff_cpy *diff)
0453 {
0454     struct ttm_operation_ctx ctx = {
0455         .interruptible = false,
0456         .no_wait_gpu = false
0457     };
0458     u32 j, initial_line = dst_offset / dst_stride;
0459     struct vmw_bo_blit_line_data d;
0460     int ret = 0;
0461
0462     /* Buffer objects need to be either pinned or reserved: */
0463     if (!(dst->pin_count))
0464         dma_resv_assert_held(dst->base.resv);
0465     if (!(src->pin_count))
0466         dma_resv_assert_held(src->base.resv);
0467
0468     if (!ttm_tt_is_populated(dst->ttm)) {
0469         ret = dst->bdev->funcs->ttm_tt_populate(dst->bdev, dst->ttm, &ctx);
0470         if (ret)
0471             return ret;
0472     }
0473
0474     if (!ttm_tt_is_populated(src->ttm)) {
0475         ret = src->bdev->funcs->ttm_tt_populate(src->bdev, src->ttm, &ctx);
0476         if (ret)
0477             return ret;
0478     }
0479
0480     d.mapped_dst = 0;
0481     d.mapped_src = 0;
0482     d.dst_addr = NULL;
0483     d.src_addr = NULL;
0484     d.dst_pages = dst->ttm->pages;
0485     d.src_pages = src->ttm->pages;
0486     d.dst_num_pages = dst->resource->num_pages;
0487     d.src_num_pages = src->resource->num_pages;
0488     d.dst_prot = ttm_io_prot(dst, dst->resource, PAGE_KERNEL);
0489     d.src_prot = ttm_io_prot(src, src->resource, PAGE_KERNEL);
0490     d.diff = diff;
0491
0492     for (j = 0; j < h; ++j) {
0493         diff->line = j + initial_line;
0494         diff->line_offset = dst_offset % dst_stride;
0495         ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w);
0496         if (ret)
0497             goto out;
0498
0499         dst_offset += dst_stride;
0500         src_offset += src_stride;
0501     }
0502 out:
0503     if (d.src_addr)
0504         kunmap_atomic(d.src_addr);
0505     if (d.dst_addr)
0506         kunmap_atomic(d.dst_addr);
0507
0508     return ret;
0509 }