powerpc/primitives/word-at-a-time.h

0001 #ifndef _ASM_WORD_AT_A_TIME_H
0002 #define _ASM_WORD_AT_A_TIME_H
0003
0004 /*
0005  * Word-at-a-time interfaces for PowerPC.
0006  */
0007
0008 #include <linux/kernel.h>
0009 #include <asm/asm-compat.h>
0010 #include <asm/extable.h>
0011
0012 #ifdef __BIG_ENDIAN__
0013
0014 struct word_at_a_time {
0015     const unsigned long high_bits, low_bits;
0016 };
0017
0018 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0xfe) + 1, REPEAT_BYTE(0x7f) }
0019
0020 /* Bit set in the bytes that have a zero */
0021 static inline long prep_zero_mask(unsigned long val, unsigned long rhs, const struct word_at_a_time *c)
0022 {
0023     unsigned long mask = (val & c->low_bits) + c->low_bits;
0024     return ~(mask | rhs);
0025 }
0026
0027 #define create_zero_mask(mask) (mask)
0028
0029 static inline long find_zero(unsigned long mask)
0030 {
0031     long leading_zero_bits;
0032
0033     asm (PPC_CNTLZL "%0,%1" : "=r" (leading_zero_bits) : "r" (mask));
0034     return leading_zero_bits >> 3;
0035 }
0036
0037 static inline bool has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c)
0038 {
0039     unsigned long rhs = val | c->low_bits;
0040     *data = rhs;
0041     return (val + c->high_bits) & ~rhs;
0042 }
0043
0044 static inline unsigned long zero_bytemask(unsigned long mask)
0045 {
0046     return ~1ul << __fls(mask);
0047 }
0048
0049 #else
0050
0051 #ifdef CONFIG_64BIT
0052
0053 /* unused */
0054 struct word_at_a_time {
0055 };
0056
0057 #define WORD_AT_A_TIME_CONSTANTS { }
0058
0059 /* This will give us 0xff for a NULL char and 0x00 elsewhere */
0060 static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
0061 {
0062     unsigned long ret;
0063     unsigned long zero = 0;
0064
0065     asm("cmpb %0,%1,%2" : "=r" (ret) : "r" (a), "r" (zero));
0066     *bits = ret;
0067
0068     return ret;
0069 }
0070
0071 static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
0072 {
0073     return bits;
0074 }
0075
0076 /* Alan Modra's little-endian strlen tail for 64-bit */
0077 static inline unsigned long create_zero_mask(unsigned long bits)
0078 {
0079     unsigned long leading_zero_bits;
0080     long trailing_zero_bit_mask;
0081
0082     asm("addi   %1,%2,-1\n\t"
0083         "andc   %1,%1,%2\n\t"
0084         "popcntd    %0,%1"
0085         : "=r" (leading_zero_bits), "=&r" (trailing_zero_bit_mask)
0086         : "b" (bits));
0087
0088     return leading_zero_bits;
0089 }
0090
0091 static inline unsigned long find_zero(unsigned long mask)
0092 {
0093     return mask >> 3;
0094 }
0095
0096 /* This assumes that we never ask for an all 1s bitmask */
0097 static inline unsigned long zero_bytemask(unsigned long mask)
0098 {
0099     return (1UL << mask) - 1;
0100 }
0101
0102 #else   /* 32-bit case */
0103
0104 struct word_at_a_time {
0105     const unsigned long one_bits, high_bits;
0106 };
0107
0108 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
0109
0110 /*
0111  * This is largely generic for little-endian machines, but the
0112  * optimal byte mask counting is probably going to be something
0113  * that is architecture-specific. If you have a reliably fast
0114  * bit count instruction, that might be better than the multiply
0115  * and shift, for example.
0116  */
0117
0118 /* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
0119 static inline long count_masked_bytes(long mask)
0120 {
0121     /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
0122     long a = (0x0ff0001+mask) >> 23;
0123     /* Fix the 1 for 00 case */
0124     return a & mask;
0125 }
0126
0127 static inline unsigned long create_zero_mask(unsigned long bits)
0128 {
0129     bits = (bits - 1) & ~bits;
0130     return bits >> 7;
0131 }
0132
0133 static inline unsigned long find_zero(unsigned long mask)
0134 {
0135     return count_masked_bytes(mask);
0136 }
0137
0138 /* Return nonzero if it has a zero */
0139 static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
0140 {
0141     unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
0142     *bits = mask;
0143     return mask;
0144 }
0145
0146 static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
0147 {
0148     return bits;
0149 }
0150
0151 /* The mask we created is directly usable as a bytemask */
0152 #define zero_bytemask(mask) (mask)
0153
0154 #endif /* CONFIG_64BIT */
0155
0156 #endif /* __BIG_ENDIAN__ */
0157
0158 /*
0159  * We use load_unaligned_zero() in a selftest, which builds a userspace
0160  * program. Some linker scripts seem to discard the .fixup section, so allow
0161  * the test code to use a different section name.
0162  */
0163 #ifndef FIXUP_SECTION
0164 #define FIXUP_SECTION ".fixup"
0165 #endif
0166
0167 static inline unsigned long load_unaligned_zeropad(const void *addr)
0168 {
0169     unsigned long ret, offset, tmp;
0170
0171     asm(
0172     "1: " PPC_LL "%[ret], 0(%[addr])\n"
0173     "2:\n"
0174     ".section " FIXUP_SECTION ",\"ax\"\n"
0175     "3: "
0176 #ifdef __powerpc64__
0177     "clrrdi     %[tmp], %[addr], 3\n\t"
0178     "clrlsldi   %[offset], %[addr], 61, 3\n\t"
0179     "ld     %[ret], 0(%[tmp])\n\t"
0180 #ifdef __BIG_ENDIAN__
0181     "sld        %[ret], %[ret], %[offset]\n\t"
0182 #else
0183     "srd        %[ret], %[ret], %[offset]\n\t"
0184 #endif
0185 #else
0186     "clrrwi     %[tmp], %[addr], 2\n\t"
0187     "clrlslwi   %[offset], %[addr], 30, 3\n\t"
0188     "lwz        %[ret], 0(%[tmp])\n\t"
0189 #ifdef __BIG_ENDIAN__
0190     "slw        %[ret], %[ret], %[offset]\n\t"
0191 #else
0192     "srw        %[ret], %[ret], %[offset]\n\t"
0193 #endif
0194 #endif
0195     "b  2b\n"
0196     ".previous\n"
0197     EX_TABLE(1b, 3b)
0198     : [tmp] "=&b" (tmp), [offset] "=&r" (offset), [ret] "=&r" (ret)
0199     : [addr] "b" (addr), "m" (*(unsigned long *)addr));
0200
0201     return ret;
0202 }
0203
0204 #undef FIXUP_SECTION
0205
0206 #endif /* _ASM_WORD_AT_A_TIME_H */