include/asm/word-at-a-time.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Copyright (C) 2013 ARM Ltd.
0004  */
0005 #ifndef __ASM_WORD_AT_A_TIME_H
0006 #define __ASM_WORD_AT_A_TIME_H
0007
0008 #include <linux/uaccess.h>
0009
0010 #ifndef __AARCH64EB__
0011
0012 #include <linux/kernel.h>
0013
0014 struct word_at_a_time {
0015     const unsigned long one_bits, high_bits;
0016 };
0017
0018 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
0019
0020 static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
0021                      const struct word_at_a_time *c)
0022 {
0023     unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
0024     *bits = mask;
0025     return mask;
0026 }
0027
0028 #define prep_zero_mask(a, bits, c) (bits)
0029
0030 static inline unsigned long create_zero_mask(unsigned long bits)
0031 {
0032     bits = (bits - 1) & ~bits;
0033     return bits >> 7;
0034 }
0035
0036 static inline unsigned long find_zero(unsigned long mask)
0037 {
0038     return fls64(mask) >> 3;
0039 }
0040
0041 #define zero_bytemask(mask) (mask)
0042
0043 #else   /* __AARCH64EB__ */
0044 #include <asm-generic/word-at-a-time.h>
0045 #endif
0046
0047 /*
0048  * Load an unaligned word from kernel space.
0049  *
0050  * In the (very unlikely) case of the word being a page-crosser
0051  * and the next page not being mapped, take the exception and
0052  * return zeroes in the non-existing part.
0053  */
0054 static inline unsigned long load_unaligned_zeropad(const void *addr)
0055 {
0056     unsigned long ret;
0057
0058     __uaccess_enable_tco_async();
0059
0060     /* Load word from unaligned pointer addr */
0061     asm(
0062     "1: ldr %0, %2\n"
0063     "2:\n"
0064     _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(1b, 2b, %0, %1)
0065     : "=&r" (ret)
0066     : "r" (addr), "Q" (*(unsigned long *)addr));
0067
0068     __uaccess_disable_tco_async();
0069
0070     return ret;
0071 }
0072
0073 #endif /* __ASM_WORD_AT_A_TIME_H */