x86/lib/hweight.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #include <linux/linkage.h>
0003 #include <asm/export.h>
0004
0005 #include <asm/asm.h>
0006
0007 /*
0008  * unsigned int __sw_hweight32(unsigned int w)
0009  * %rdi: w
0010  */
0011 SYM_FUNC_START(__sw_hweight32)
0012
0013 #ifdef CONFIG_X86_64
0014     movl %edi, %eax             # w
0015 #endif
0016     __ASM_SIZE(push,) %__ASM_REG(dx)
0017     movl %eax, %edx             # w -> t
0018     shrl %edx               # t >>= 1
0019     andl $0x55555555, %edx          # t &= 0x55555555
0020     subl %edx, %eax             # w -= t
0021
0022     movl %eax, %edx             # w -> t
0023     shrl $2, %eax               # w_tmp >>= 2
0024     andl $0x33333333, %edx          # t &= 0x33333333
0025     andl $0x33333333, %eax          # w_tmp &= 0x33333333
0026     addl %edx, %eax             # w = w_tmp + t
0027
0028     movl %eax, %edx             # w -> t
0029     shrl $4, %edx               # t >>= 4
0030     addl %edx, %eax             # w_tmp += t
0031     andl  $0x0f0f0f0f, %eax         # w_tmp &= 0x0f0f0f0f
0032     imull $0x01010101, %eax, %eax       # w_tmp *= 0x01010101
0033     shrl $24, %eax              # w = w_tmp >> 24
0034     __ASM_SIZE(pop,) %__ASM_REG(dx)
0035     RET
0036 SYM_FUNC_END(__sw_hweight32)
0037 EXPORT_SYMBOL(__sw_hweight32)
0038
0039 SYM_FUNC_START(__sw_hweight64)
0040 #ifdef CONFIG_X86_64
0041     pushq   %rdi
0042     pushq   %rdx
0043
0044     movq    %rdi, %rdx                      # w -> t
0045     movabsq $0x5555555555555555, %rax
0046     shrq    %rdx                            # t >>= 1
0047     andq    %rdx, %rax                      # t &= 0x5555555555555555
0048     movabsq $0x3333333333333333, %rdx
0049     subq    %rax, %rdi                      # w -= t
0050
0051     movq    %rdi, %rax                      # w -> t
0052     shrq    $2, %rdi                        # w_tmp >>= 2
0053     andq    %rdx, %rax                      # t     &= 0x3333333333333333
0054     andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
0055     addq    %rdx, %rax                      # w = w_tmp + t
0056
0057     movq    %rax, %rdx                      # w -> t
0058     shrq    $4, %rdx                        # t >>= 4
0059     addq    %rdx, %rax                      # w_tmp += t
0060     movabsq $0x0f0f0f0f0f0f0f0f, %rdx
0061     andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
0062     movabsq $0x0101010101010101, %rdx
0063     imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
0064     shrq    $56, %rax                       # w = w_tmp >> 56
0065
0066     popq    %rdx
0067     popq    %rdi
0068     RET
0069 #else /* CONFIG_X86_32 */
0070     /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
0071     pushl   %ecx
0072
0073     call    __sw_hweight32
0074     movl    %eax, %ecx                      # stash away result
0075     movl    %edx, %eax                      # second part of input
0076     call    __sw_hweight32
0077     addl    %ecx, %eax                      # result
0078
0079     popl    %ecx
0080     RET
0081 #endif
0082 SYM_FUNC_END(__sw_hweight64)
0083 EXPORT_SYMBOL(__sw_hweight64)