drm/i915/i915_memcpy.c

0001 /*
0002  * Copyright © 2016 Intel Corporation
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice (including the next
0012  * paragraph) shall be included in all copies or substantial portions of the
0013  * Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
0019  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
0020  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
0021  * IN THE SOFTWARE.
0022  *
0023  */
0024
0025 #include <linux/kernel.h>
0026 #include <asm/fpu/api.h>
0027
0028 #include "i915_memcpy.h"
0029
0030 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
0031 #define CI_BUG_ON(expr) BUG_ON(expr)
0032 #else
0033 #define CI_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
0034 #endif
0035
0036 static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
0037
0038 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
0039 {
0040     kernel_fpu_begin();
0041
0042     while (len >= 4) {
0043         asm("movntdqa   (%0), %%xmm0\n"
0044             "movntdqa 16(%0), %%xmm1\n"
0045             "movntdqa 32(%0), %%xmm2\n"
0046             "movntdqa 48(%0), %%xmm3\n"
0047             "movaps %%xmm0,   (%1)\n"
0048             "movaps %%xmm1, 16(%1)\n"
0049             "movaps %%xmm2, 32(%1)\n"
0050             "movaps %%xmm3, 48(%1)\n"
0051             :: "r" (src), "r" (dst) : "memory");
0052         src += 64;
0053         dst += 64;
0054         len -= 4;
0055     }
0056     while (len--) {
0057         asm("movntdqa (%0), %%xmm0\n"
0058             "movaps %%xmm0, (%1)\n"
0059             :: "r" (src), "r" (dst) : "memory");
0060         src += 16;
0061         dst += 16;
0062     }
0063
0064     kernel_fpu_end();
0065 }
0066
0067 static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len)
0068 {
0069     kernel_fpu_begin();
0070
0071     while (len >= 4) {
0072         asm("movntdqa   (%0), %%xmm0\n"
0073             "movntdqa 16(%0), %%xmm1\n"
0074             "movntdqa 32(%0), %%xmm2\n"
0075             "movntdqa 48(%0), %%xmm3\n"
0076             "movups %%xmm0,   (%1)\n"
0077             "movups %%xmm1, 16(%1)\n"
0078             "movups %%xmm2, 32(%1)\n"
0079             "movups %%xmm3, 48(%1)\n"
0080             :: "r" (src), "r" (dst) : "memory");
0081         src += 64;
0082         dst += 64;
0083         len -= 4;
0084     }
0085     while (len--) {
0086         asm("movntdqa (%0), %%xmm0\n"
0087             "movups %%xmm0, (%1)\n"
0088             :: "r" (src), "r" (dst) : "memory");
0089         src += 16;
0090         dst += 16;
0091     }
0092
0093     kernel_fpu_end();
0094 }
0095
0096 /**
0097  * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
0098  * @dst: destination pointer
0099  * @src: source pointer
0100  * @len: how many bytes to copy
0101  *
0102  * i915_memcpy_from_wc copies @len bytes from @src to @dst using
0103  * non-temporal instructions where available. Note that all arguments
0104  * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
0105  * of 16.
0106  *
0107  * To test whether accelerated reads from WC are supported, use
0108  * i915_memcpy_from_wc(NULL, NULL, 0);
0109  *
0110  * Returns true if the copy was successful, false if the preconditions
0111  * are not met.
0112  */
0113 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
0114 {
0115     if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
0116         return false;
0117
0118     if (static_branch_likely(&has_movntdqa)) {
0119         if (likely(len))
0120             __memcpy_ntdqa(dst, src, len >> 4);
0121         return true;
0122     }
0123
0124     return false;
0125 }
0126
0127 /**
0128  * i915_unaligned_memcpy_from_wc: perform a mostly accelerated read from WC
0129  * @dst: destination pointer
0130  * @src: source pointer
0131  * @len: how many bytes to copy
0132  *
0133  * Like i915_memcpy_from_wc(), the unaligned variant copies @len bytes from
0134  * @src to @dst using * non-temporal instructions where available, but
0135  * accepts that its arguments may not be aligned, but are valid for the
0136  * potential 16-byte read past the end.
0137  */
0138 void i915_unaligned_memcpy_from_wc(void *dst, const void *src, unsigned long len)
0139 {
0140     unsigned long addr;
0141
0142     CI_BUG_ON(!i915_has_memcpy_from_wc());
0143
0144     addr = (unsigned long)src;
0145     if (!IS_ALIGNED(addr, 16)) {
0146         unsigned long x = min(ALIGN(addr, 16) - addr, len);
0147
0148         memcpy(dst, src, x);
0149
0150         len -= x;
0151         dst += x;
0152         src += x;
0153     }
0154
0155     if (likely(len))
0156         __memcpy_ntdqu(dst, src, DIV_ROUND_UP(len, 16));
0157 }
0158
0159 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
0160 {
0161     /*
0162      * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
0163      * emulation. So don't enable movntdqa in hypervisor guest.
0164      */
0165     if (static_cpu_has(X86_FEATURE_XMM4_1) &&
0166         !boot_cpu_has(X86_FEATURE_HYPERVISOR))
0167         static_branch_enable(&has_movntdqa);
0168 }