Back to home page

LXR

 
 

    


0001 /*
0002  * Handle unaligned accesses by emulation.
0003  *
0004  * This file is subject to the terms and conditions of the GNU General Public
0005  * License.  See the file "COPYING" in the main directory of this archive
0006  * for more details.
0007  *
0008  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
0009  * Copyright (C) 1999 Silicon Graphics, Inc.
0010  * Copyright (C) 2014 Imagination Technologies Ltd.
0011  *
0012  * This file contains exception handler for address error exception with the
0013  * special capability to execute faulting instructions in software.  The
0014  * handler does not try to handle the case when the program counter points
0015  * to an address not aligned to a word boundary.
0016  *
0017  * Putting data to unaligned addresses is a bad practice even on Intel where
0018  * only the performance is affected.  Much worse is that such code is non-
0019  * portable.  Due to several programs that die on MIPS due to alignment
0020  * problems I decided to implement this handler anyway though I originally
0021  * didn't intend to do this at all for user code.
0022  *
0023  * For now I enable fixing of address errors by default to make life easier.
0024  * I however intend to disable this somewhen in the future when the alignment
0025  * problems with user programs have been fixed.  For programmers this is the
0026  * right way to go.
0027  *
0028  * Fixing address errors is a per process option.  The option is inherited
0029  * across fork(2) and execve(2) calls.  If you really want to use the
0030  * option in your user programs - I discourage the use of the software
0031  * emulation strongly - use the following code in your userland stuff:
0032  *
0033  * #include <sys/sysmips.h>
0034  *
0035  * ...
0036  * sysmips(MIPS_FIXADE, x);
0037  * ...
0038  *
0039  * The argument x is 0 for disabling software emulation, enabled otherwise.
0040  *
0041  * Below a little program to play around with this feature.
0042  *
0043  * #include <stdio.h>
0044  * #include <sys/sysmips.h>
0045  *
0046  * struct foo {
0047  *     unsigned char bar[8];
0048  * };
0049  *
0050  * main(int argc, char *argv[])
0051  * {
0052  *     struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
0053  *     unsigned int *p = (unsigned int *) (x.bar + 3);
0054  *     int i;
0055  *
0056  *     if (argc > 1)
0057  *         sysmips(MIPS_FIXADE, atoi(argv[1]));
0058  *
0059  *     printf("*p = %08lx\n", *p);
0060  *
0061  *     *p = 0xdeadface;
0062  *
0063  *     for(i = 0; i <= 7; i++)
0064  *     printf("%02x ", x.bar[i]);
0065  *     printf("\n");
0066  * }
0067  *
0068  * Coprocessor loads are not supported; I think this case is unimportant
0069  * in the practice.
0070  *
0071  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
0072  *   exception for the R6000.
0073  *   A store crossing a page boundary might be executed only partially.
0074  *   Undo the partial store in this case.
0075  */
0076 #include <linux/context_tracking.h>
0077 #include <linux/mm.h>
0078 #include <linux/signal.h>
0079 #include <linux/smp.h>
0080 #include <linux/sched.h>
0081 #include <linux/debugfs.h>
0082 #include <linux/perf_event.h>
0083 
0084 #include <asm/asm.h>
0085 #include <asm/branch.h>
0086 #include <asm/byteorder.h>
0087 #include <asm/cop2.h>
0088 #include <asm/debug.h>
0089 #include <asm/fpu.h>
0090 #include <asm/fpu_emulator.h>
0091 #include <asm/inst.h>
0092 #include <linux/uaccess.h>
0093 
0094 #define STR(x)  __STR(x)
0095 #define __STR(x)  #x
0096 
0097 enum {
0098     UNALIGNED_ACTION_QUIET,
0099     UNALIGNED_ACTION_SIGNAL,
0100     UNALIGNED_ACTION_SHOW,
0101 };
0102 #ifdef CONFIG_DEBUG_FS
0103 static u32 unaligned_instructions;
0104 static u32 unaligned_action;
0105 #else
0106 #define unaligned_action UNALIGNED_ACTION_QUIET
0107 #endif
0108 extern void show_registers(struct pt_regs *regs);
0109 
0110 #ifdef __BIG_ENDIAN
0111 #define     _LoadHW(addr, value, res, type)  \
0112 do {                                                        \
0113         __asm__ __volatile__ (".set\tnoat\n"        \
0114             "1:\t"type##_lb("%0", "0(%2)")"\n"  \
0115             "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
0116             "sll\t%0, 0x8\n\t"                  \
0117             "or\t%0, $1\n\t"                    \
0118             "li\t%1, 0\n"                       \
0119             "3:\t.set\tat\n\t"                  \
0120             ".insn\n\t"                         \
0121             ".section\t.fixup,\"ax\"\n\t"       \
0122             "4:\tli\t%1, %3\n\t"                \
0123             "j\t3b\n\t"                         \
0124             ".previous\n\t"                     \
0125             ".section\t__ex_table,\"a\"\n\t"    \
0126             STR(PTR)"\t1b, 4b\n\t"              \
0127             STR(PTR)"\t2b, 4b\n\t"              \
0128             ".previous"                         \
0129             : "=&r" (value), "=r" (res)         \
0130             : "r" (addr), "i" (-EFAULT));       \
0131 } while(0)
0132 
0133 #ifndef CONFIG_CPU_MIPSR6
0134 #define     _LoadW(addr, value, res, type)   \
0135 do {                                                        \
0136         __asm__ __volatile__ (                      \
0137             "1:\t"type##_lwl("%0", "(%2)")"\n"   \
0138             "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
0139             "li\t%1, 0\n"                       \
0140             "3:\n\t"                            \
0141             ".insn\n\t"                         \
0142             ".section\t.fixup,\"ax\"\n\t"       \
0143             "4:\tli\t%1, %3\n\t"                \
0144             "j\t3b\n\t"                         \
0145             ".previous\n\t"                     \
0146             ".section\t__ex_table,\"a\"\n\t"    \
0147             STR(PTR)"\t1b, 4b\n\t"              \
0148             STR(PTR)"\t2b, 4b\n\t"              \
0149             ".previous"                         \
0150             : "=&r" (value), "=r" (res)         \
0151             : "r" (addr), "i" (-EFAULT));       \
0152 } while(0)
0153 
0154 #else
0155 /* MIPSR6 has no lwl instruction */
0156 #define     _LoadW(addr, value, res, type) \
0157 do {                                                        \
0158         __asm__ __volatile__ (              \
0159             ".set\tpush\n"              \
0160             ".set\tnoat\n\t"            \
0161             "1:"type##_lb("%0", "0(%2)")"\n\t"  \
0162             "2:"type##_lbu("$1", "1(%2)")"\n\t" \
0163             "sll\t%0, 0x8\n\t"          \
0164             "or\t%0, $1\n\t"            \
0165             "3:"type##_lbu("$1", "2(%2)")"\n\t" \
0166             "sll\t%0, 0x8\n\t"          \
0167             "or\t%0, $1\n\t"            \
0168             "4:"type##_lbu("$1", "3(%2)")"\n\t" \
0169             "sll\t%0, 0x8\n\t"          \
0170             "or\t%0, $1\n\t"            \
0171             "li\t%1, 0\n"               \
0172             ".set\tpop\n"               \
0173             "10:\n\t"               \
0174             ".insn\n\t"             \
0175             ".section\t.fixup,\"ax\"\n\t"       \
0176             "11:\tli\t%1, %3\n\t"           \
0177             "j\t10b\n\t"                \
0178             ".previous\n\t"             \
0179             ".section\t__ex_table,\"a\"\n\t"    \
0180             STR(PTR)"\t1b, 11b\n\t"         \
0181             STR(PTR)"\t2b, 11b\n\t"         \
0182             STR(PTR)"\t3b, 11b\n\t"         \
0183             STR(PTR)"\t4b, 11b\n\t"         \
0184             ".previous"             \
0185             : "=&r" (value), "=r" (res)     \
0186             : "r" (addr), "i" (-EFAULT));       \
0187 } while(0)
0188 
0189 #endif /* CONFIG_CPU_MIPSR6 */
0190 
0191 #define     _LoadHWU(addr, value, res, type) \
0192 do {                                                        \
0193         __asm__ __volatile__ (                      \
0194             ".set\tnoat\n"                      \
0195             "1:\t"type##_lbu("%0", "0(%2)")"\n" \
0196             "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
0197             "sll\t%0, 0x8\n\t"                  \
0198             "or\t%0, $1\n\t"                    \
0199             "li\t%1, 0\n"                       \
0200             "3:\n\t"                            \
0201             ".insn\n\t"                         \
0202             ".set\tat\n\t"                      \
0203             ".section\t.fixup,\"ax\"\n\t"       \
0204             "4:\tli\t%1, %3\n\t"                \
0205             "j\t3b\n\t"                         \
0206             ".previous\n\t"                     \
0207             ".section\t__ex_table,\"a\"\n\t"    \
0208             STR(PTR)"\t1b, 4b\n\t"              \
0209             STR(PTR)"\t2b, 4b\n\t"              \
0210             ".previous"                         \
0211             : "=&r" (value), "=r" (res)         \
0212             : "r" (addr), "i" (-EFAULT));       \
0213 } while(0)
0214 
0215 #ifndef CONFIG_CPU_MIPSR6
0216 #define     _LoadWU(addr, value, res, type)  \
0217 do {                                                        \
0218         __asm__ __volatile__ (                      \
0219             "1:\t"type##_lwl("%0", "(%2)")"\n"  \
0220             "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
0221             "dsll\t%0, %0, 32\n\t"              \
0222             "dsrl\t%0, %0, 32\n\t"              \
0223             "li\t%1, 0\n"                       \
0224             "3:\n\t"                            \
0225             ".insn\n\t"                         \
0226             "\t.section\t.fixup,\"ax\"\n\t"     \
0227             "4:\tli\t%1, %3\n\t"                \
0228             "j\t3b\n\t"                         \
0229             ".previous\n\t"                     \
0230             ".section\t__ex_table,\"a\"\n\t"    \
0231             STR(PTR)"\t1b, 4b\n\t"              \
0232             STR(PTR)"\t2b, 4b\n\t"              \
0233             ".previous"                         \
0234             : "=&r" (value), "=r" (res)         \
0235             : "r" (addr), "i" (-EFAULT));       \
0236 } while(0)
0237 
0238 #define     _LoadDW(addr, value, res)  \
0239 do {                                                        \
0240         __asm__ __volatile__ (                      \
0241             "1:\tldl\t%0, (%2)\n"               \
0242             "2:\tldr\t%0, 7(%2)\n\t"            \
0243             "li\t%1, 0\n"                       \
0244             "3:\n\t"                            \
0245             ".insn\n\t"                         \
0246             "\t.section\t.fixup,\"ax\"\n\t"     \
0247             "4:\tli\t%1, %3\n\t"                \
0248             "j\t3b\n\t"                         \
0249             ".previous\n\t"                     \
0250             ".section\t__ex_table,\"a\"\n\t"    \
0251             STR(PTR)"\t1b, 4b\n\t"              \
0252             STR(PTR)"\t2b, 4b\n\t"              \
0253             ".previous"                         \
0254             : "=&r" (value), "=r" (res)         \
0255             : "r" (addr), "i" (-EFAULT));       \
0256 } while(0)
0257 
0258 #else
0259 /* MIPSR6 has not lwl and ldl instructions */
0260 #define     _LoadWU(addr, value, res, type) \
0261 do {                                                        \
0262         __asm__ __volatile__ (              \
0263             ".set\tpush\n\t"            \
0264             ".set\tnoat\n\t"            \
0265             "1:"type##_lbu("%0", "0(%2)")"\n\t" \
0266             "2:"type##_lbu("$1", "1(%2)")"\n\t" \
0267             "sll\t%0, 0x8\n\t"          \
0268             "or\t%0, $1\n\t"            \
0269             "3:"type##_lbu("$1", "2(%2)")"\n\t" \
0270             "sll\t%0, 0x8\n\t"          \
0271             "or\t%0, $1\n\t"            \
0272             "4:"type##_lbu("$1", "3(%2)")"\n\t" \
0273             "sll\t%0, 0x8\n\t"          \
0274             "or\t%0, $1\n\t"            \
0275             "li\t%1, 0\n"               \
0276             ".set\tpop\n"               \
0277             "10:\n\t"               \
0278             ".insn\n\t"             \
0279             ".section\t.fixup,\"ax\"\n\t"       \
0280             "11:\tli\t%1, %3\n\t"           \
0281             "j\t10b\n\t"                \
0282             ".previous\n\t"             \
0283             ".section\t__ex_table,\"a\"\n\t"    \
0284             STR(PTR)"\t1b, 11b\n\t"         \
0285             STR(PTR)"\t2b, 11b\n\t"         \
0286             STR(PTR)"\t3b, 11b\n\t"         \
0287             STR(PTR)"\t4b, 11b\n\t"         \
0288             ".previous"             \
0289             : "=&r" (value), "=r" (res)     \
0290             : "r" (addr), "i" (-EFAULT));       \
0291 } while(0)
0292 
0293 #define     _LoadDW(addr, value, res)  \
0294 do {                                                        \
0295         __asm__ __volatile__ (              \
0296             ".set\tpush\n\t"            \
0297             ".set\tnoat\n\t"            \
0298             "1:lb\t%0, 0(%2)\n\t"           \
0299             "2:lbu\t $1, 1(%2)\n\t"         \
0300             "dsll\t%0, 0x8\n\t"         \
0301             "or\t%0, $1\n\t"            \
0302             "3:lbu\t$1, 2(%2)\n\t"          \
0303             "dsll\t%0, 0x8\n\t"         \
0304             "or\t%0, $1\n\t"            \
0305             "4:lbu\t$1, 3(%2)\n\t"          \
0306             "dsll\t%0, 0x8\n\t"         \
0307             "or\t%0, $1\n\t"            \
0308             "5:lbu\t$1, 4(%2)\n\t"          \
0309             "dsll\t%0, 0x8\n\t"         \
0310             "or\t%0, $1\n\t"            \
0311             "6:lbu\t$1, 5(%2)\n\t"          \
0312             "dsll\t%0, 0x8\n\t"         \
0313             "or\t%0, $1\n\t"            \
0314             "7:lbu\t$1, 6(%2)\n\t"          \
0315             "dsll\t%0, 0x8\n\t"         \
0316             "or\t%0, $1\n\t"            \
0317             "8:lbu\t$1, 7(%2)\n\t"          \
0318             "dsll\t%0, 0x8\n\t"         \
0319             "or\t%0, $1\n\t"            \
0320             "li\t%1, 0\n"               \
0321             ".set\tpop\n\t"             \
0322             "10:\n\t"               \
0323             ".insn\n\t"             \
0324             ".section\t.fixup,\"ax\"\n\t"       \
0325             "11:\tli\t%1, %3\n\t"           \
0326             "j\t10b\n\t"                \
0327             ".previous\n\t"             \
0328             ".section\t__ex_table,\"a\"\n\t"    \
0329             STR(PTR)"\t1b, 11b\n\t"         \
0330             STR(PTR)"\t2b, 11b\n\t"         \
0331             STR(PTR)"\t3b, 11b\n\t"         \
0332             STR(PTR)"\t4b, 11b\n\t"         \
0333             STR(PTR)"\t5b, 11b\n\t"         \
0334             STR(PTR)"\t6b, 11b\n\t"         \
0335             STR(PTR)"\t7b, 11b\n\t"         \
0336             STR(PTR)"\t8b, 11b\n\t"         \
0337             ".previous"             \
0338             : "=&r" (value), "=r" (res)     \
0339             : "r" (addr), "i" (-EFAULT));       \
0340 } while(0)
0341 
0342 #endif /* CONFIG_CPU_MIPSR6 */
0343 
0344 
0345 #define     _StoreHW(addr, value, res, type) \
0346 do {                                                        \
0347         __asm__ __volatile__ (                      \
0348             ".set\tnoat\n"                      \
0349             "1:\t"type##_sb("%1", "1(%2)")"\n"  \
0350             "srl\t$1, %1, 0x8\n"                \
0351             "2:\t"type##_sb("$1", "0(%2)")"\n"  \
0352             ".set\tat\n\t"                      \
0353             "li\t%0, 0\n"                       \
0354             "3:\n\t"                            \
0355             ".insn\n\t"                         \
0356             ".section\t.fixup,\"ax\"\n\t"       \
0357             "4:\tli\t%0, %3\n\t"                \
0358             "j\t3b\n\t"                         \
0359             ".previous\n\t"                     \
0360             ".section\t__ex_table,\"a\"\n\t"    \
0361             STR(PTR)"\t1b, 4b\n\t"              \
0362             STR(PTR)"\t2b, 4b\n\t"              \
0363             ".previous"                         \
0364             : "=r" (res)                        \
0365             : "r" (value), "r" (addr), "i" (-EFAULT));\
0366 } while(0)
0367 
0368 #ifndef CONFIG_CPU_MIPSR6
0369 #define     _StoreW(addr, value, res, type)  \
0370 do {                                                        \
0371         __asm__ __volatile__ (                      \
0372             "1:\t"type##_swl("%1", "(%2)")"\n"  \
0373             "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
0374             "li\t%0, 0\n"                       \
0375             "3:\n\t"                            \
0376             ".insn\n\t"                         \
0377             ".section\t.fixup,\"ax\"\n\t"       \
0378             "4:\tli\t%0, %3\n\t"                \
0379             "j\t3b\n\t"                         \
0380             ".previous\n\t"                     \
0381             ".section\t__ex_table,\"a\"\n\t"    \
0382             STR(PTR)"\t1b, 4b\n\t"              \
0383             STR(PTR)"\t2b, 4b\n\t"              \
0384             ".previous"                         \
0385         : "=r" (res)                                \
0386         : "r" (value), "r" (addr), "i" (-EFAULT));  \
0387 } while(0)
0388 
0389 #define     _StoreDW(addr, value, res) \
0390 do {                                                        \
0391         __asm__ __volatile__ (                      \
0392             "1:\tsdl\t%1,(%2)\n"                \
0393             "2:\tsdr\t%1, 7(%2)\n\t"            \
0394             "li\t%0, 0\n"                       \
0395             "3:\n\t"                            \
0396             ".insn\n\t"                         \
0397             ".section\t.fixup,\"ax\"\n\t"       \
0398             "4:\tli\t%0, %3\n\t"                \
0399             "j\t3b\n\t"                         \
0400             ".previous\n\t"                     \
0401             ".section\t__ex_table,\"a\"\n\t"    \
0402             STR(PTR)"\t1b, 4b\n\t"              \
0403             STR(PTR)"\t2b, 4b\n\t"              \
0404             ".previous"                         \
0405         : "=r" (res)                                \
0406         : "r" (value), "r" (addr), "i" (-EFAULT));  \
0407 } while(0)
0408 
0409 #else
0410 /* MIPSR6 has no swl and sdl instructions */
0411 #define     _StoreW(addr, value, res, type)  \
0412 do {                                                        \
0413         __asm__ __volatile__ (                      \
0414             ".set\tpush\n\t"            \
0415             ".set\tnoat\n\t"            \
0416             "1:"type##_sb("%1", "3(%2)")"\n\t"  \
0417             "srl\t$1, %1, 0x8\n\t"          \
0418             "2:"type##_sb("$1", "2(%2)")"\n\t"  \
0419             "srl\t$1, $1,  0x8\n\t"         \
0420             "3:"type##_sb("$1", "1(%2)")"\n\t"  \
0421             "srl\t$1, $1, 0x8\n\t"          \
0422             "4:"type##_sb("$1", "0(%2)")"\n\t"  \
0423             ".set\tpop\n\t"             \
0424             "li\t%0, 0\n"               \
0425             "10:\n\t"               \
0426             ".insn\n\t"             \
0427             ".section\t.fixup,\"ax\"\n\t"       \
0428             "11:\tli\t%0, %3\n\t"           \
0429             "j\t10b\n\t"                \
0430             ".previous\n\t"             \
0431             ".section\t__ex_table,\"a\"\n\t"    \
0432             STR(PTR)"\t1b, 11b\n\t"         \
0433             STR(PTR)"\t2b, 11b\n\t"         \
0434             STR(PTR)"\t3b, 11b\n\t"         \
0435             STR(PTR)"\t4b, 11b\n\t"         \
0436             ".previous"             \
0437         : "=&r" (res)                       \
0438         : "r" (value), "r" (addr), "i" (-EFAULT)    \
0439         : "memory");                                \
0440 } while(0)
0441 
0442 #define     _StoreDW(addr, value, res) \
0443 do {                                                        \
0444         __asm__ __volatile__ (                      \
0445             ".set\tpush\n\t"            \
0446             ".set\tnoat\n\t"            \
0447             "1:sb\t%1, 7(%2)\n\t"           \
0448             "dsrl\t$1, %1, 0x8\n\t"         \
0449             "2:sb\t$1, 6(%2)\n\t"           \
0450             "dsrl\t$1, $1, 0x8\n\t"         \
0451             "3:sb\t$1, 5(%2)\n\t"           \
0452             "dsrl\t$1, $1, 0x8\n\t"         \
0453             "4:sb\t$1, 4(%2)\n\t"           \
0454             "dsrl\t$1, $1, 0x8\n\t"         \
0455             "5:sb\t$1, 3(%2)\n\t"           \
0456             "dsrl\t$1, $1, 0x8\n\t"         \
0457             "6:sb\t$1, 2(%2)\n\t"           \
0458             "dsrl\t$1, $1, 0x8\n\t"         \
0459             "7:sb\t$1, 1(%2)\n\t"           \
0460             "dsrl\t$1, $1, 0x8\n\t"         \
0461             "8:sb\t$1, 0(%2)\n\t"           \
0462             "dsrl\t$1, $1, 0x8\n\t"         \
0463             ".set\tpop\n\t"             \
0464             "li\t%0, 0\n"               \
0465             "10:\n\t"               \
0466             ".insn\n\t"             \
0467             ".section\t.fixup,\"ax\"\n\t"       \
0468             "11:\tli\t%0, %3\n\t"           \
0469             "j\t10b\n\t"                \
0470             ".previous\n\t"             \
0471             ".section\t__ex_table,\"a\"\n\t"    \
0472             STR(PTR)"\t1b, 11b\n\t"         \
0473             STR(PTR)"\t2b, 11b\n\t"         \
0474             STR(PTR)"\t3b, 11b\n\t"         \
0475             STR(PTR)"\t4b, 11b\n\t"         \
0476             STR(PTR)"\t5b, 11b\n\t"         \
0477             STR(PTR)"\t6b, 11b\n\t"         \
0478             STR(PTR)"\t7b, 11b\n\t"         \
0479             STR(PTR)"\t8b, 11b\n\t"         \
0480             ".previous"             \
0481         : "=&r" (res)                       \
0482         : "r" (value), "r" (addr), "i" (-EFAULT)    \
0483         : "memory");                                \
0484 } while(0)
0485 
0486 #endif /* CONFIG_CPU_MIPSR6 */
0487 
0488 #else /* __BIG_ENDIAN */
0489 
0490 #define     _LoadHW(addr, value, res, type)  \
0491 do {                                                        \
0492         __asm__ __volatile__ (".set\tnoat\n"        \
0493             "1:\t"type##_lb("%0", "1(%2)")"\n"  \
0494             "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
0495             "sll\t%0, 0x8\n\t"                  \
0496             "or\t%0, $1\n\t"                    \
0497             "li\t%1, 0\n"                       \
0498             "3:\t.set\tat\n\t"                  \
0499             ".insn\n\t"                         \
0500             ".section\t.fixup,\"ax\"\n\t"       \
0501             "4:\tli\t%1, %3\n\t"                \
0502             "j\t3b\n\t"                         \
0503             ".previous\n\t"                     \
0504             ".section\t__ex_table,\"a\"\n\t"    \
0505             STR(PTR)"\t1b, 4b\n\t"              \
0506             STR(PTR)"\t2b, 4b\n\t"              \
0507             ".previous"                         \
0508             : "=&r" (value), "=r" (res)         \
0509             : "r" (addr), "i" (-EFAULT));       \
0510 } while(0)
0511 
0512 #ifndef CONFIG_CPU_MIPSR6
0513 #define     _LoadW(addr, value, res, type)   \
0514 do {                                                        \
0515         __asm__ __volatile__ (                      \
0516             "1:\t"type##_lwl("%0", "3(%2)")"\n" \
0517             "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
0518             "li\t%1, 0\n"                       \
0519             "3:\n\t"                            \
0520             ".insn\n\t"                         \
0521             ".section\t.fixup,\"ax\"\n\t"       \
0522             "4:\tli\t%1, %3\n\t"                \
0523             "j\t3b\n\t"                         \
0524             ".previous\n\t"                     \
0525             ".section\t__ex_table,\"a\"\n\t"    \
0526             STR(PTR)"\t1b, 4b\n\t"              \
0527             STR(PTR)"\t2b, 4b\n\t"              \
0528             ".previous"                         \
0529             : "=&r" (value), "=r" (res)         \
0530             : "r" (addr), "i" (-EFAULT));       \
0531 } while(0)
0532 
0533 #else
0534 /* MIPSR6 has no lwl instruction */
0535 #define     _LoadW(addr, value, res, type) \
0536 do {                                                        \
0537         __asm__ __volatile__ (              \
0538             ".set\tpush\n"              \
0539             ".set\tnoat\n\t"            \
0540             "1:"type##_lb("%0", "3(%2)")"\n\t"  \
0541             "2:"type##_lbu("$1", "2(%2)")"\n\t" \
0542             "sll\t%0, 0x8\n\t"          \
0543             "or\t%0, $1\n\t"            \
0544             "3:"type##_lbu("$1", "1(%2)")"\n\t" \
0545             "sll\t%0, 0x8\n\t"          \
0546             "or\t%0, $1\n\t"            \
0547             "4:"type##_lbu("$1", "0(%2)")"\n\t" \
0548             "sll\t%0, 0x8\n\t"          \
0549             "or\t%0, $1\n\t"            \
0550             "li\t%1, 0\n"               \
0551             ".set\tpop\n"               \
0552             "10:\n\t"               \
0553             ".insn\n\t"             \
0554             ".section\t.fixup,\"ax\"\n\t"       \
0555             "11:\tli\t%1, %3\n\t"           \
0556             "j\t10b\n\t"                \
0557             ".previous\n\t"             \
0558             ".section\t__ex_table,\"a\"\n\t"    \
0559             STR(PTR)"\t1b, 11b\n\t"         \
0560             STR(PTR)"\t2b, 11b\n\t"         \
0561             STR(PTR)"\t3b, 11b\n\t"         \
0562             STR(PTR)"\t4b, 11b\n\t"         \
0563             ".previous"             \
0564             : "=&r" (value), "=r" (res)     \
0565             : "r" (addr), "i" (-EFAULT));       \
0566 } while(0)
0567 
0568 #endif /* CONFIG_CPU_MIPSR6 */
0569 
0570 
0571 #define     _LoadHWU(addr, value, res, type) \
0572 do {                                                        \
0573         __asm__ __volatile__ (                      \
0574             ".set\tnoat\n"                      \
0575             "1:\t"type##_lbu("%0", "1(%2)")"\n" \
0576             "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
0577             "sll\t%0, 0x8\n\t"                  \
0578             "or\t%0, $1\n\t"                    \
0579             "li\t%1, 0\n"                       \
0580             "3:\n\t"                            \
0581             ".insn\n\t"                         \
0582             ".set\tat\n\t"                      \
0583             ".section\t.fixup,\"ax\"\n\t"       \
0584             "4:\tli\t%1, %3\n\t"                \
0585             "j\t3b\n\t"                         \
0586             ".previous\n\t"                     \
0587             ".section\t__ex_table,\"a\"\n\t"    \
0588             STR(PTR)"\t1b, 4b\n\t"              \
0589             STR(PTR)"\t2b, 4b\n\t"              \
0590             ".previous"                         \
0591             : "=&r" (value), "=r" (res)         \
0592             : "r" (addr), "i" (-EFAULT));       \
0593 } while(0)
0594 
0595 #ifndef CONFIG_CPU_MIPSR6
0596 #define     _LoadWU(addr, value, res, type)  \
0597 do {                                                        \
0598         __asm__ __volatile__ (                      \
0599             "1:\t"type##_lwl("%0", "3(%2)")"\n" \
0600             "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
0601             "dsll\t%0, %0, 32\n\t"              \
0602             "dsrl\t%0, %0, 32\n\t"              \
0603             "li\t%1, 0\n"                       \
0604             "3:\n\t"                            \
0605             ".insn\n\t"                         \
0606             "\t.section\t.fixup,\"ax\"\n\t"     \
0607             "4:\tli\t%1, %3\n\t"                \
0608             "j\t3b\n\t"                         \
0609             ".previous\n\t"                     \
0610             ".section\t__ex_table,\"a\"\n\t"    \
0611             STR(PTR)"\t1b, 4b\n\t"              \
0612             STR(PTR)"\t2b, 4b\n\t"              \
0613             ".previous"                         \
0614             : "=&r" (value), "=r" (res)         \
0615             : "r" (addr), "i" (-EFAULT));       \
0616 } while(0)
0617 
0618 #define     _LoadDW(addr, value, res)  \
0619 do {                                                        \
0620         __asm__ __volatile__ (                      \
0621             "1:\tldl\t%0, 7(%2)\n"              \
0622             "2:\tldr\t%0, (%2)\n\t"             \
0623             "li\t%1, 0\n"                       \
0624             "3:\n\t"                            \
0625             ".insn\n\t"                         \
0626             "\t.section\t.fixup,\"ax\"\n\t"     \
0627             "4:\tli\t%1, %3\n\t"                \
0628             "j\t3b\n\t"                         \
0629             ".previous\n\t"                     \
0630             ".section\t__ex_table,\"a\"\n\t"    \
0631             STR(PTR)"\t1b, 4b\n\t"              \
0632             STR(PTR)"\t2b, 4b\n\t"              \
0633             ".previous"                         \
0634             : "=&r" (value), "=r" (res)         \
0635             : "r" (addr), "i" (-EFAULT));       \
0636 } while(0)
0637 
0638 #else
0639 /* MIPSR6 has not lwl and ldl instructions */
0640 #define     _LoadWU(addr, value, res, type) \
0641 do {                                                        \
0642         __asm__ __volatile__ (              \
0643             ".set\tpush\n\t"            \
0644             ".set\tnoat\n\t"            \
0645             "1:"type##_lbu("%0", "3(%2)")"\n\t" \
0646             "2:"type##_lbu("$1", "2(%2)")"\n\t" \
0647             "sll\t%0, 0x8\n\t"          \
0648             "or\t%0, $1\n\t"            \
0649             "3:"type##_lbu("$1", "1(%2)")"\n\t" \
0650             "sll\t%0, 0x8\n\t"          \
0651             "or\t%0, $1\n\t"            \
0652             "4:"type##_lbu("$1", "0(%2)")"\n\t" \
0653             "sll\t%0, 0x8\n\t"          \
0654             "or\t%0, $1\n\t"            \
0655             "li\t%1, 0\n"               \
0656             ".set\tpop\n"               \
0657             "10:\n\t"               \
0658             ".insn\n\t"             \
0659             ".section\t.fixup,\"ax\"\n\t"       \
0660             "11:\tli\t%1, %3\n\t"           \
0661             "j\t10b\n\t"                \
0662             ".previous\n\t"             \
0663             ".section\t__ex_table,\"a\"\n\t"    \
0664             STR(PTR)"\t1b, 11b\n\t"         \
0665             STR(PTR)"\t2b, 11b\n\t"         \
0666             STR(PTR)"\t3b, 11b\n\t"         \
0667             STR(PTR)"\t4b, 11b\n\t"         \
0668             ".previous"             \
0669             : "=&r" (value), "=r" (res)     \
0670             : "r" (addr), "i" (-EFAULT));       \
0671 } while(0)
0672 
0673 #define     _LoadDW(addr, value, res)  \
0674 do {                                                        \
0675         __asm__ __volatile__ (              \
0676             ".set\tpush\n\t"            \
0677             ".set\tnoat\n\t"            \
0678             "1:lb\t%0, 7(%2)\n\t"           \
0679             "2:lbu\t$1, 6(%2)\n\t"          \
0680             "dsll\t%0, 0x8\n\t"         \
0681             "or\t%0, $1\n\t"            \
0682             "3:lbu\t$1, 5(%2)\n\t"          \
0683             "dsll\t%0, 0x8\n\t"         \
0684             "or\t%0, $1\n\t"            \
0685             "4:lbu\t$1, 4(%2)\n\t"          \
0686             "dsll\t%0, 0x8\n\t"         \
0687             "or\t%0, $1\n\t"            \
0688             "5:lbu\t$1, 3(%2)\n\t"          \
0689             "dsll\t%0, 0x8\n\t"         \
0690             "or\t%0, $1\n\t"            \
0691             "6:lbu\t$1, 2(%2)\n\t"          \
0692             "dsll\t%0, 0x8\n\t"         \
0693             "or\t%0, $1\n\t"            \
0694             "7:lbu\t$1, 1(%2)\n\t"          \
0695             "dsll\t%0, 0x8\n\t"         \
0696             "or\t%0, $1\n\t"            \
0697             "8:lbu\t$1, 0(%2)\n\t"          \
0698             "dsll\t%0, 0x8\n\t"         \
0699             "or\t%0, $1\n\t"            \
0700             "li\t%1, 0\n"               \
0701             ".set\tpop\n\t"             \
0702             "10:\n\t"               \
0703             ".insn\n\t"             \
0704             ".section\t.fixup,\"ax\"\n\t"       \
0705             "11:\tli\t%1, %3\n\t"           \
0706             "j\t10b\n\t"                \
0707             ".previous\n\t"             \
0708             ".section\t__ex_table,\"a\"\n\t"    \
0709             STR(PTR)"\t1b, 11b\n\t"         \
0710             STR(PTR)"\t2b, 11b\n\t"         \
0711             STR(PTR)"\t3b, 11b\n\t"         \
0712             STR(PTR)"\t4b, 11b\n\t"         \
0713             STR(PTR)"\t5b, 11b\n\t"         \
0714             STR(PTR)"\t6b, 11b\n\t"         \
0715             STR(PTR)"\t7b, 11b\n\t"         \
0716             STR(PTR)"\t8b, 11b\n\t"         \
0717             ".previous"             \
0718             : "=&r" (value), "=r" (res)     \
0719             : "r" (addr), "i" (-EFAULT));       \
0720 } while(0)
0721 #endif /* CONFIG_CPU_MIPSR6 */
0722 
0723 #define     _StoreHW(addr, value, res, type) \
0724 do {                                                        \
0725         __asm__ __volatile__ (                      \
0726             ".set\tnoat\n"                      \
0727             "1:\t"type##_sb("%1", "0(%2)")"\n"  \
0728             "srl\t$1,%1, 0x8\n"                 \
0729             "2:\t"type##_sb("$1", "1(%2)")"\n"  \
0730             ".set\tat\n\t"                      \
0731             "li\t%0, 0\n"                       \
0732             "3:\n\t"                            \
0733             ".insn\n\t"                         \
0734             ".section\t.fixup,\"ax\"\n\t"       \
0735             "4:\tli\t%0, %3\n\t"                \
0736             "j\t3b\n\t"                         \
0737             ".previous\n\t"                     \
0738             ".section\t__ex_table,\"a\"\n\t"    \
0739             STR(PTR)"\t1b, 4b\n\t"              \
0740             STR(PTR)"\t2b, 4b\n\t"              \
0741             ".previous"                         \
0742             : "=r" (res)                        \
0743             : "r" (value), "r" (addr), "i" (-EFAULT));\
0744 } while(0)
0745 
0746 #ifndef CONFIG_CPU_MIPSR6
0747 #define     _StoreW(addr, value, res, type)  \
0748 do {                                                        \
0749         __asm__ __volatile__ (                      \
0750             "1:\t"type##_swl("%1", "3(%2)")"\n" \
0751             "2:\t"type##_swr("%1", "(%2)")"\n\t"\
0752             "li\t%0, 0\n"                       \
0753             "3:\n\t"                            \
0754             ".insn\n\t"                         \
0755             ".section\t.fixup,\"ax\"\n\t"       \
0756             "4:\tli\t%0, %3\n\t"                \
0757             "j\t3b\n\t"                         \
0758             ".previous\n\t"                     \
0759             ".section\t__ex_table,\"a\"\n\t"    \
0760             STR(PTR)"\t1b, 4b\n\t"              \
0761             STR(PTR)"\t2b, 4b\n\t"              \
0762             ".previous"                         \
0763         : "=r" (res)                                \
0764         : "r" (value), "r" (addr), "i" (-EFAULT));  \
0765 } while(0)
0766 
0767 #define     _StoreDW(addr, value, res) \
0768 do {                                                        \
0769         __asm__ __volatile__ (                      \
0770             "1:\tsdl\t%1, 7(%2)\n"              \
0771             "2:\tsdr\t%1, (%2)\n\t"             \
0772             "li\t%0, 0\n"                       \
0773             "3:\n\t"                            \
0774             ".insn\n\t"                         \
0775             ".section\t.fixup,\"ax\"\n\t"       \
0776             "4:\tli\t%0, %3\n\t"                \
0777             "j\t3b\n\t"                         \
0778             ".previous\n\t"                     \
0779             ".section\t__ex_table,\"a\"\n\t"    \
0780             STR(PTR)"\t1b, 4b\n\t"              \
0781             STR(PTR)"\t2b, 4b\n\t"              \
0782             ".previous"                         \
0783         : "=r" (res)                                \
0784         : "r" (value), "r" (addr), "i" (-EFAULT));  \
0785 } while(0)
0786 
0787 #else
0788 /* MIPSR6 has no swl and sdl instructions */
0789 #define     _StoreW(addr, value, res, type)  \
0790 do {                                                        \
0791         __asm__ __volatile__ (                      \
0792             ".set\tpush\n\t"            \
0793             ".set\tnoat\n\t"            \
0794             "1:"type##_sb("%1", "0(%2)")"\n\t"  \
0795             "srl\t$1, %1, 0x8\n\t"          \
0796             "2:"type##_sb("$1", "1(%2)")"\n\t"  \
0797             "srl\t$1, $1,  0x8\n\t"         \
0798             "3:"type##_sb("$1", "2(%2)")"\n\t"  \
0799             "srl\t$1, $1, 0x8\n\t"          \
0800             "4:"type##_sb("$1", "3(%2)")"\n\t"  \
0801             ".set\tpop\n\t"             \
0802             "li\t%0, 0\n"               \
0803             "10:\n\t"               \
0804             ".insn\n\t"             \
0805             ".section\t.fixup,\"ax\"\n\t"       \
0806             "11:\tli\t%0, %3\n\t"           \
0807             "j\t10b\n\t"                \
0808             ".previous\n\t"             \
0809             ".section\t__ex_table,\"a\"\n\t"    \
0810             STR(PTR)"\t1b, 11b\n\t"         \
0811             STR(PTR)"\t2b, 11b\n\t"         \
0812             STR(PTR)"\t3b, 11b\n\t"         \
0813             STR(PTR)"\t4b, 11b\n\t"         \
0814             ".previous"             \
0815         : "=&r" (res)                       \
0816         : "r" (value), "r" (addr), "i" (-EFAULT)    \
0817         : "memory");                                \
0818 } while(0)
0819 
0820 #define     _StoreDW(addr, value, res) \
0821 do {                                                        \
0822         __asm__ __volatile__ (                      \
0823             ".set\tpush\n\t"            \
0824             ".set\tnoat\n\t"            \
0825             "1:sb\t%1, 0(%2)\n\t"           \
0826             "dsrl\t$1, %1, 0x8\n\t"         \
0827             "2:sb\t$1, 1(%2)\n\t"           \
0828             "dsrl\t$1, $1, 0x8\n\t"         \
0829             "3:sb\t$1, 2(%2)\n\t"           \
0830             "dsrl\t$1, $1, 0x8\n\t"         \
0831             "4:sb\t$1, 3(%2)\n\t"           \
0832             "dsrl\t$1, $1, 0x8\n\t"         \
0833             "5:sb\t$1, 4(%2)\n\t"           \
0834             "dsrl\t$1, $1, 0x8\n\t"         \
0835             "6:sb\t$1, 5(%2)\n\t"           \
0836             "dsrl\t$1, $1, 0x8\n\t"         \
0837             "7:sb\t$1, 6(%2)\n\t"           \
0838             "dsrl\t$1, $1, 0x8\n\t"         \
0839             "8:sb\t$1, 7(%2)\n\t"           \
0840             "dsrl\t$1, $1, 0x8\n\t"         \
0841             ".set\tpop\n\t"             \
0842             "li\t%0, 0\n"               \
0843             "10:\n\t"               \
0844             ".insn\n\t"             \
0845             ".section\t.fixup,\"ax\"\n\t"       \
0846             "11:\tli\t%0, %3\n\t"           \
0847             "j\t10b\n\t"                \
0848             ".previous\n\t"             \
0849             ".section\t__ex_table,\"a\"\n\t"    \
0850             STR(PTR)"\t1b, 11b\n\t"         \
0851             STR(PTR)"\t2b, 11b\n\t"         \
0852             STR(PTR)"\t3b, 11b\n\t"         \
0853             STR(PTR)"\t4b, 11b\n\t"         \
0854             STR(PTR)"\t5b, 11b\n\t"         \
0855             STR(PTR)"\t6b, 11b\n\t"         \
0856             STR(PTR)"\t7b, 11b\n\t"         \
0857             STR(PTR)"\t8b, 11b\n\t"         \
0858             ".previous"             \
0859         : "=&r" (res)                       \
0860         : "r" (value), "r" (addr), "i" (-EFAULT)    \
0861         : "memory");                                \
0862 } while(0)
0863 
0864 #endif /* CONFIG_CPU_MIPSR6 */
0865 #endif
0866 
0867 #define LoadHWU(addr, value, res)   _LoadHWU(addr, value, res, kernel)
0868 #define LoadHWUE(addr, value, res)  _LoadHWU(addr, value, res, user)
0869 #define LoadWU(addr, value, res)    _LoadWU(addr, value, res, kernel)
0870 #define LoadWUE(addr, value, res)   _LoadWU(addr, value, res, user)
0871 #define LoadHW(addr, value, res)    _LoadHW(addr, value, res, kernel)
0872 #define LoadHWE(addr, value, res)   _LoadHW(addr, value, res, user)
0873 #define LoadW(addr, value, res)     _LoadW(addr, value, res, kernel)
0874 #define LoadWE(addr, value, res)    _LoadW(addr, value, res, user)
0875 #define LoadDW(addr, value, res)    _LoadDW(addr, value, res)
0876 
0877 #define StoreHW(addr, value, res)   _StoreHW(addr, value, res, kernel)
0878 #define StoreHWE(addr, value, res)  _StoreHW(addr, value, res, user)
0879 #define StoreW(addr, value, res)    _StoreW(addr, value, res, kernel)
0880 #define StoreWE(addr, value, res)   _StoreW(addr, value, res, user)
0881 #define StoreDW(addr, value, res)   _StoreDW(addr, value, res)
0882 
0883 static void emulate_load_store_insn(struct pt_regs *regs,
0884     void __user *addr, unsigned int __user *pc)
0885 {
0886     union mips_instruction insn;
0887     unsigned long value;
0888     unsigned int res, preempted;
0889     unsigned long origpc;
0890     unsigned long orig31;
0891     void __user *fault_addr = NULL;
0892 #ifdef  CONFIG_EVA
0893     mm_segment_t seg;
0894 #endif
0895     union fpureg *fpr;
0896     enum msa_2b_fmt df;
0897     unsigned int wd;
0898     origpc = (unsigned long)pc;
0899     orig31 = regs->regs[31];
0900 
0901     perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
0902 
0903     /*
0904      * This load never faults.
0905      */
0906     __get_user(insn.word, pc);
0907 
0908     switch (insn.i_format.opcode) {
0909         /*
0910          * These are instructions that a compiler doesn't generate.  We
0911          * can assume therefore that the code is MIPS-aware and
0912          * really buggy.  Emulating these instructions would break the
0913          * semantics anyway.
0914          */
0915     case ll_op:
0916     case lld_op:
0917     case sc_op:
0918     case scd_op:
0919 
0920         /*
0921          * For these instructions the only way to create an address
0922          * error is an attempted access to kernel/supervisor address
0923          * space.
0924          */
0925     case ldl_op:
0926     case ldr_op:
0927     case lwl_op:
0928     case lwr_op:
0929     case sdl_op:
0930     case sdr_op:
0931     case swl_op:
0932     case swr_op:
0933     case lb_op:
0934     case lbu_op:
0935     case sb_op:
0936         goto sigbus;
0937 
0938         /*
0939          * The remaining opcodes are the ones that are really of
0940          * interest.
0941          */
0942 #ifdef CONFIG_EVA
0943     case spec3_op:
0944         /*
0945          * we can land here only from kernel accessing user memory,
0946          * so we need to "switch" the address limit to user space, so
0947          * address check can work properly.
0948          */
0949         seg = get_fs();
0950         set_fs(USER_DS);
0951         switch (insn.spec3_format.func) {
0952         case lhe_op:
0953             if (!access_ok(VERIFY_READ, addr, 2)) {
0954                 set_fs(seg);
0955                 goto sigbus;
0956             }
0957             LoadHWE(addr, value, res);
0958             if (res) {
0959                 set_fs(seg);
0960                 goto fault;
0961             }
0962             compute_return_epc(regs);
0963             regs->regs[insn.spec3_format.rt] = value;
0964             break;
0965         case lwe_op:
0966             if (!access_ok(VERIFY_READ, addr, 4)) {
0967                 set_fs(seg);
0968                 goto sigbus;
0969             }
0970                 LoadWE(addr, value, res);
0971             if (res) {
0972                 set_fs(seg);
0973                 goto fault;
0974             }
0975             compute_return_epc(regs);
0976             regs->regs[insn.spec3_format.rt] = value;
0977             break;
0978         case lhue_op:
0979             if (!access_ok(VERIFY_READ, addr, 2)) {
0980                 set_fs(seg);
0981                 goto sigbus;
0982             }
0983             LoadHWUE(addr, value, res);
0984             if (res) {
0985                 set_fs(seg);
0986                 goto fault;
0987             }
0988             compute_return_epc(regs);
0989             regs->regs[insn.spec3_format.rt] = value;
0990             break;
0991         case she_op:
0992             if (!access_ok(VERIFY_WRITE, addr, 2)) {
0993                 set_fs(seg);
0994                 goto sigbus;
0995             }
0996             compute_return_epc(regs);
0997             value = regs->regs[insn.spec3_format.rt];
0998             StoreHWE(addr, value, res);
0999             if (res) {
1000                 set_fs(seg);
1001                 goto fault;
1002             }
1003             break;
1004         case swe_op:
1005             if (!access_ok(VERIFY_WRITE, addr, 4)) {
1006                 set_fs(seg);
1007                 goto sigbus;
1008             }
1009             compute_return_epc(regs);
1010             value = regs->regs[insn.spec3_format.rt];
1011             StoreWE(addr, value, res);
1012             if (res) {
1013                 set_fs(seg);
1014                 goto fault;
1015             }
1016             break;
1017         default:
1018             set_fs(seg);
1019             goto sigill;
1020         }
1021         set_fs(seg);
1022         break;
1023 #endif
1024     case lh_op:
1025         if (!access_ok(VERIFY_READ, addr, 2))
1026             goto sigbus;
1027 
1028         if (IS_ENABLED(CONFIG_EVA)) {
1029             if (segment_eq(get_fs(), get_ds()))
1030                 LoadHW(addr, value, res);
1031             else
1032                 LoadHWE(addr, value, res);
1033         } else {
1034             LoadHW(addr, value, res);
1035         }
1036 
1037         if (res)
1038             goto fault;
1039         compute_return_epc(regs);
1040         regs->regs[insn.i_format.rt] = value;
1041         break;
1042 
1043     case lw_op:
1044         if (!access_ok(VERIFY_READ, addr, 4))
1045             goto sigbus;
1046 
1047         if (IS_ENABLED(CONFIG_EVA)) {
1048             if (segment_eq(get_fs(), get_ds()))
1049                 LoadW(addr, value, res);
1050             else
1051                 LoadWE(addr, value, res);
1052         } else {
1053             LoadW(addr, value, res);
1054         }
1055 
1056         if (res)
1057             goto fault;
1058         compute_return_epc(regs);
1059         regs->regs[insn.i_format.rt] = value;
1060         break;
1061 
1062     case lhu_op:
1063         if (!access_ok(VERIFY_READ, addr, 2))
1064             goto sigbus;
1065 
1066         if (IS_ENABLED(CONFIG_EVA)) {
1067             if (segment_eq(get_fs(), get_ds()))
1068                 LoadHWU(addr, value, res);
1069             else
1070                 LoadHWUE(addr, value, res);
1071         } else {
1072             LoadHWU(addr, value, res);
1073         }
1074 
1075         if (res)
1076             goto fault;
1077         compute_return_epc(regs);
1078         regs->regs[insn.i_format.rt] = value;
1079         break;
1080 
1081     case lwu_op:
1082 #ifdef CONFIG_64BIT
1083         /*
1084          * A 32-bit kernel might be running on a 64-bit processor.  But
1085          * if we're on a 32-bit processor and an i-cache incoherency
1086          * or race makes us see a 64-bit instruction here the sdl/sdr
1087          * would blow up, so for now we don't handle unaligned 64-bit
1088          * instructions on 32-bit kernels.
1089          */
1090         if (!access_ok(VERIFY_READ, addr, 4))
1091             goto sigbus;
1092 
1093         LoadWU(addr, value, res);
1094         if (res)
1095             goto fault;
1096         compute_return_epc(regs);
1097         regs->regs[insn.i_format.rt] = value;
1098         break;
1099 #endif /* CONFIG_64BIT */
1100 
1101         /* Cannot handle 64-bit instructions in 32-bit kernel */
1102         goto sigill;
1103 
1104     case ld_op:
1105 #ifdef CONFIG_64BIT
1106         /*
1107          * A 32-bit kernel might be running on a 64-bit processor.  But
1108          * if we're on a 32-bit processor and an i-cache incoherency
1109          * or race makes us see a 64-bit instruction here the sdl/sdr
1110          * would blow up, so for now we don't handle unaligned 64-bit
1111          * instructions on 32-bit kernels.
1112          */
1113         if (!access_ok(VERIFY_READ, addr, 8))
1114             goto sigbus;
1115 
1116         LoadDW(addr, value, res);
1117         if (res)
1118             goto fault;
1119         compute_return_epc(regs);
1120         regs->regs[insn.i_format.rt] = value;
1121         break;
1122 #endif /* CONFIG_64BIT */
1123 
1124         /* Cannot handle 64-bit instructions in 32-bit kernel */
1125         goto sigill;
1126 
1127     case sh_op:
1128         if (!access_ok(VERIFY_WRITE, addr, 2))
1129             goto sigbus;
1130 
1131         compute_return_epc(regs);
1132         value = regs->regs[insn.i_format.rt];
1133 
1134         if (IS_ENABLED(CONFIG_EVA)) {
1135             if (segment_eq(get_fs(), get_ds()))
1136                 StoreHW(addr, value, res);
1137             else
1138                 StoreHWE(addr, value, res);
1139         } else {
1140             StoreHW(addr, value, res);
1141         }
1142 
1143         if (res)
1144             goto fault;
1145         break;
1146 
1147     case sw_op:
1148         if (!access_ok(VERIFY_WRITE, addr, 4))
1149             goto sigbus;
1150 
1151         compute_return_epc(regs);
1152         value = regs->regs[insn.i_format.rt];
1153 
1154         if (IS_ENABLED(CONFIG_EVA)) {
1155             if (segment_eq(get_fs(), get_ds()))
1156                 StoreW(addr, value, res);
1157             else
1158                 StoreWE(addr, value, res);
1159         } else {
1160             StoreW(addr, value, res);
1161         }
1162 
1163         if (res)
1164             goto fault;
1165         break;
1166 
1167     case sd_op:
1168 #ifdef CONFIG_64BIT
1169         /*
1170          * A 32-bit kernel might be running on a 64-bit processor.  But
1171          * if we're on a 32-bit processor and an i-cache incoherency
1172          * or race makes us see a 64-bit instruction here the sdl/sdr
1173          * would blow up, so for now we don't handle unaligned 64-bit
1174          * instructions on 32-bit kernels.
1175          */
1176         if (!access_ok(VERIFY_WRITE, addr, 8))
1177             goto sigbus;
1178 
1179         compute_return_epc(regs);
1180         value = regs->regs[insn.i_format.rt];
1181         StoreDW(addr, value, res);
1182         if (res)
1183             goto fault;
1184         break;
1185 #endif /* CONFIG_64BIT */
1186 
1187         /* Cannot handle 64-bit instructions in 32-bit kernel */
1188         goto sigill;
1189 
1190     case lwc1_op:
1191     case ldc1_op:
1192     case swc1_op:
1193     case sdc1_op:
1194     case cop1x_op:
1195         die_if_kernel("Unaligned FP access in kernel code", regs);
1196         BUG_ON(!used_math());
1197 
1198         lose_fpu(1);    /* Save FPU state for the emulator. */
1199         res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1200                            &fault_addr);
1201         own_fpu(1); /* Restore FPU state. */
1202 
1203         /* Signal if something went wrong. */
1204         process_fpemu_return(res, fault_addr, 0);
1205 
1206         if (res == 0)
1207             break;
1208         return;
1209 
1210     case msa_op:
1211         if (!cpu_has_msa)
1212             goto sigill;
1213 
1214         /*
1215          * If we've reached this point then userland should have taken
1216          * the MSA disabled exception & initialised vector context at
1217          * some point in the past.
1218          */
1219         BUG_ON(!thread_msa_context_live());
1220 
1221         df = insn.msa_mi10_format.df;
1222         wd = insn.msa_mi10_format.wd;
1223         fpr = &current->thread.fpu.fpr[wd];
1224 
1225         switch (insn.msa_mi10_format.func) {
1226         case msa_ld_op:
1227             if (!access_ok(VERIFY_READ, addr, sizeof(*fpr)))
1228                 goto sigbus;
1229 
1230             do {
1231                 /*
1232                  * If we have live MSA context keep track of
1233                  * whether we get preempted in order to avoid
1234                  * the register context we load being clobbered
1235                  * by the live context as it's saved during
1236                  * preemption. If we don't have live context
1237                  * then it can't be saved to clobber the value
1238                  * we load.
1239                  */
1240                 preempted = test_thread_flag(TIF_USEDMSA);
1241 
1242                 res = __copy_from_user_inatomic(fpr, addr,
1243                                 sizeof(*fpr));
1244                 if (res)
1245                     goto fault;
1246 
1247                 /*
1248                  * Update the hardware register if it is in use
1249                  * by the task in this quantum, in order to
1250                  * avoid having to save & restore the whole
1251                  * vector context.
1252                  */
1253                 preempt_disable();
1254                 if (test_thread_flag(TIF_USEDMSA)) {
1255                     write_msa_wr(wd, fpr, df);
1256                     preempted = 0;
1257                 }
1258                 preempt_enable();
1259             } while (preempted);
1260             break;
1261 
1262         case msa_st_op:
1263             if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr)))
1264                 goto sigbus;
1265 
1266             /*
1267              * Update from the hardware register if it is in use by
1268              * the task in this quantum, in order to avoid having to
1269              * save & restore the whole vector context.
1270              */
1271             preempt_disable();
1272             if (test_thread_flag(TIF_USEDMSA))
1273                 read_msa_wr(wd, fpr, df);
1274             preempt_enable();
1275 
1276             res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1277             if (res)
1278                 goto fault;
1279             break;
1280 
1281         default:
1282             goto sigbus;
1283         }
1284 
1285         compute_return_epc(regs);
1286         break;
1287 
1288 #ifndef CONFIG_CPU_MIPSR6
1289     /*
1290      * COP2 is available to implementor for application specific use.
1291      * It's up to applications to register a notifier chain and do
1292      * whatever they have to do, including possible sending of signals.
1293      *
1294      * This instruction has been reallocated in Release 6
1295      */
1296     case lwc2_op:
1297         cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1298         break;
1299 
1300     case ldc2_op:
1301         cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1302         break;
1303 
1304     case swc2_op:
1305         cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1306         break;
1307 
1308     case sdc2_op:
1309         cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1310         break;
1311 #endif
1312     default:
1313         /*
1314          * Pheeee...  We encountered an yet unknown instruction or
1315          * cache coherence problem.  Die sucker, die ...
1316          */
1317         goto sigill;
1318     }
1319 
1320 #ifdef CONFIG_DEBUG_FS
1321     unaligned_instructions++;
1322 #endif
1323 
1324     return;
1325 
1326 fault:
1327     /* roll back jump/branch */
1328     regs->cp0_epc = origpc;
1329     regs->regs[31] = orig31;
1330     /* Did we have an exception handler installed? */
1331     if (fixup_exception(regs))
1332         return;
1333 
1334     die_if_kernel("Unhandled kernel unaligned access", regs);
1335     force_sig(SIGSEGV, current);
1336 
1337     return;
1338 
1339 sigbus:
1340     die_if_kernel("Unhandled kernel unaligned access", regs);
1341     force_sig(SIGBUS, current);
1342 
1343     return;
1344 
1345 sigill:
1346     die_if_kernel
1347         ("Unhandled kernel unaligned access or invalid instruction", regs);
1348     force_sig(SIGILL, current);
1349 }
1350 
1351 /* Recode table from 16-bit register notation to 32-bit GPR. */
1352 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1353 
1354 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1355 const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1356 
1357 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1358                      void __user *addr)
1359 {
1360     unsigned long value;
1361     unsigned int res;
1362     int i;
1363     unsigned int reg = 0, rvar;
1364     unsigned long orig31;
1365     u16 __user *pc16;
1366     u16 halfword;
1367     unsigned int word;
1368     unsigned long origpc, contpc;
1369     union mips_instruction insn;
1370     struct mm_decoded_insn mminsn;
1371     void __user *fault_addr = NULL;
1372 
1373     origpc = regs->cp0_epc;
1374     orig31 = regs->regs[31];
1375 
1376     mminsn.micro_mips_mode = 1;
1377 
1378     /*
1379      * This load never faults.
1380      */
1381     pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1382     __get_user(halfword, pc16);
1383     pc16++;
1384     contpc = regs->cp0_epc + 2;
1385     word = ((unsigned int)halfword << 16);
1386     mminsn.pc_inc = 2;
1387 
1388     if (!mm_insn_16bit(halfword)) {
1389         __get_user(halfword, pc16);
1390         pc16++;
1391         contpc = regs->cp0_epc + 4;
1392         mminsn.pc_inc = 4;
1393         word |= halfword;
1394     }
1395     mminsn.insn = word;
1396 
1397     if (get_user(halfword, pc16))
1398         goto fault;
1399     mminsn.next_pc_inc = 2;
1400     word = ((unsigned int)halfword << 16);
1401 
1402     if (!mm_insn_16bit(halfword)) {
1403         pc16++;
1404         if (get_user(halfword, pc16))
1405             goto fault;
1406         mminsn.next_pc_inc = 4;
1407         word |= halfword;
1408     }
1409     mminsn.next_insn = word;
1410 
1411     insn = (union mips_instruction)(mminsn.insn);
1412     if (mm_isBranchInstr(regs, mminsn, &contpc))
1413         insn = (union mips_instruction)(mminsn.next_insn);
1414 
1415     /*  Parse instruction to find what to do */
1416 
1417     switch (insn.mm_i_format.opcode) {
1418 
1419     case mm_pool32a_op:
1420         switch (insn.mm_x_format.func) {
1421         case mm_lwxs_op:
1422             reg = insn.mm_x_format.rd;
1423             goto loadW;
1424         }
1425 
1426         goto sigbus;
1427 
1428     case mm_pool32b_op:
1429         switch (insn.mm_m_format.func) {
1430         case mm_lwp_func:
1431             reg = insn.mm_m_format.rd;
1432             if (reg == 31)
1433                 goto sigbus;
1434 
1435             if (!access_ok(VERIFY_READ, addr, 8))
1436                 goto sigbus;
1437 
1438             LoadW(addr, value, res);
1439             if (res)
1440                 goto fault;
1441             regs->regs[reg] = value;
1442             addr += 4;
1443             LoadW(addr, value, res);
1444             if (res)
1445                 goto fault;
1446             regs->regs[reg + 1] = value;
1447             goto success;
1448 
1449         case mm_swp_func:
1450             reg = insn.mm_m_format.rd;
1451             if (reg == 31)
1452                 goto sigbus;
1453 
1454             if (!access_ok(VERIFY_WRITE, addr, 8))
1455                 goto sigbus;
1456 
1457             value = regs->regs[reg];
1458             StoreW(addr, value, res);
1459             if (res)
1460                 goto fault;
1461             addr += 4;
1462             value = regs->regs[reg + 1];
1463             StoreW(addr, value, res);
1464             if (res)
1465                 goto fault;
1466             goto success;
1467 
1468         case mm_ldp_func:
1469 #ifdef CONFIG_64BIT
1470             reg = insn.mm_m_format.rd;
1471             if (reg == 31)
1472                 goto sigbus;
1473 
1474             if (!access_ok(VERIFY_READ, addr, 16))
1475                 goto sigbus;
1476 
1477             LoadDW(addr, value, res);
1478             if (res)
1479                 goto fault;
1480             regs->regs[reg] = value;
1481             addr += 8;
1482             LoadDW(addr, value, res);
1483             if (res)
1484                 goto fault;
1485             regs->regs[reg + 1] = value;
1486             goto success;
1487 #endif /* CONFIG_64BIT */
1488 
1489             goto sigill;
1490 
1491         case mm_sdp_func:
1492 #ifdef CONFIG_64BIT
1493             reg = insn.mm_m_format.rd;
1494             if (reg == 31)
1495                 goto sigbus;
1496 
1497             if (!access_ok(VERIFY_WRITE, addr, 16))
1498                 goto sigbus;
1499 
1500             value = regs->regs[reg];
1501             StoreDW(addr, value, res);
1502             if (res)
1503                 goto fault;
1504             addr += 8;
1505             value = regs->regs[reg + 1];
1506             StoreDW(addr, value, res);
1507             if (res)
1508                 goto fault;
1509             goto success;
1510 #endif /* CONFIG_64BIT */
1511 
1512             goto sigill;
1513 
1514         case mm_lwm32_func:
1515             reg = insn.mm_m_format.rd;
1516             rvar = reg & 0xf;
1517             if ((rvar > 9) || !reg)
1518                 goto sigill;
1519             if (reg & 0x10) {
1520                 if (!access_ok
1521                     (VERIFY_READ, addr, 4 * (rvar + 1)))
1522                     goto sigbus;
1523             } else {
1524                 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1525                     goto sigbus;
1526             }
1527             if (rvar == 9)
1528                 rvar = 8;
1529             for (i = 16; rvar; rvar--, i++) {
1530                 LoadW(addr, value, res);
1531                 if (res)
1532                     goto fault;
1533                 addr += 4;
1534                 regs->regs[i] = value;
1535             }
1536             if ((reg & 0xf) == 9) {
1537                 LoadW(addr, value, res);
1538                 if (res)
1539                     goto fault;
1540                 addr += 4;
1541                 regs->regs[30] = value;
1542             }
1543             if (reg & 0x10) {
1544                 LoadW(addr, value, res);
1545                 if (res)
1546                     goto fault;
1547                 regs->regs[31] = value;
1548             }
1549             goto success;
1550 
1551         case mm_swm32_func:
1552             reg = insn.mm_m_format.rd;
1553             rvar = reg & 0xf;
1554             if ((rvar > 9) || !reg)
1555                 goto sigill;
1556             if (reg & 0x10) {
1557                 if (!access_ok
1558                     (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1559                     goto sigbus;
1560             } else {
1561                 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1562                     goto sigbus;
1563             }
1564             if (rvar == 9)
1565                 rvar = 8;
1566             for (i = 16; rvar; rvar--, i++) {
1567                 value = regs->regs[i];
1568                 StoreW(addr, value, res);
1569                 if (res)
1570                     goto fault;
1571                 addr += 4;
1572             }
1573             if ((reg & 0xf) == 9) {
1574                 value = regs->regs[30];
1575                 StoreW(addr, value, res);
1576                 if (res)
1577                     goto fault;
1578                 addr += 4;
1579             }
1580             if (reg & 0x10) {
1581                 value = regs->regs[31];
1582                 StoreW(addr, value, res);
1583                 if (res)
1584                     goto fault;
1585             }
1586             goto success;
1587 
1588         case mm_ldm_func:
1589 #ifdef CONFIG_64BIT
1590             reg = insn.mm_m_format.rd;
1591             rvar = reg & 0xf;
1592             if ((rvar > 9) || !reg)
1593                 goto sigill;
1594             if (reg & 0x10) {
1595                 if (!access_ok
1596                     (VERIFY_READ, addr, 8 * (rvar + 1)))
1597                     goto sigbus;
1598             } else {
1599                 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1600                     goto sigbus;
1601             }
1602             if (rvar == 9)
1603                 rvar = 8;
1604 
1605             for (i = 16; rvar; rvar--, i++) {
1606                 LoadDW(addr, value, res);
1607                 if (res)
1608                     goto fault;
1609                 addr += 4;
1610                 regs->regs[i] = value;
1611             }
1612             if ((reg & 0xf) == 9) {
1613                 LoadDW(addr, value, res);
1614                 if (res)
1615                     goto fault;
1616                 addr += 8;
1617                 regs->regs[30] = value;
1618             }
1619             if (reg & 0x10) {
1620                 LoadDW(addr, value, res);
1621                 if (res)
1622                     goto fault;
1623                 regs->regs[31] = value;
1624             }
1625             goto success;
1626 #endif /* CONFIG_64BIT */
1627 
1628             goto sigill;
1629 
1630         case mm_sdm_func:
1631 #ifdef CONFIG_64BIT
1632             reg = insn.mm_m_format.rd;
1633             rvar = reg & 0xf;
1634             if ((rvar > 9) || !reg)
1635                 goto sigill;
1636             if (reg & 0x10) {
1637                 if (!access_ok
1638                     (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1639                     goto sigbus;
1640             } else {
1641                 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1642                     goto sigbus;
1643             }
1644             if (rvar == 9)
1645                 rvar = 8;
1646 
1647             for (i = 16; rvar; rvar--, i++) {
1648                 value = regs->regs[i];
1649                 StoreDW(addr, value, res);
1650                 if (res)
1651                     goto fault;
1652                 addr += 8;
1653             }
1654             if ((reg & 0xf) == 9) {
1655                 value = regs->regs[30];
1656                 StoreDW(addr, value, res);
1657                 if (res)
1658                     goto fault;
1659                 addr += 8;
1660             }
1661             if (reg & 0x10) {
1662                 value = regs->regs[31];
1663                 StoreDW(addr, value, res);
1664                 if (res)
1665                     goto fault;
1666             }
1667             goto success;
1668 #endif /* CONFIG_64BIT */
1669 
1670             goto sigill;
1671 
1672             /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1673         }
1674 
1675         goto sigbus;
1676 
1677     case mm_pool32c_op:
1678         switch (insn.mm_m_format.func) {
1679         case mm_lwu_func:
1680             reg = insn.mm_m_format.rd;
1681             goto loadWU;
1682         }
1683 
1684         /*  LL,SC,LLD,SCD are not serviced */
1685         goto sigbus;
1686 
1687     case mm_pool32f_op:
1688         switch (insn.mm_x_format.func) {
1689         case mm_lwxc1_func:
1690         case mm_swxc1_func:
1691         case mm_ldxc1_func:
1692         case mm_sdxc1_func:
1693             goto fpu_emul;
1694         }
1695 
1696         goto sigbus;
1697 
1698     case mm_ldc132_op:
1699     case mm_sdc132_op:
1700     case mm_lwc132_op:
1701     case mm_swc132_op:
1702 fpu_emul:
1703         /* roll back jump/branch */
1704         regs->cp0_epc = origpc;
1705         regs->regs[31] = orig31;
1706 
1707         die_if_kernel("Unaligned FP access in kernel code", regs);
1708         BUG_ON(!used_math());
1709         BUG_ON(!is_fpu_owner());
1710 
1711         lose_fpu(1);    /* save the FPU state for the emulator */
1712         res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1713                            &fault_addr);
1714         own_fpu(1); /* restore FPU state */
1715 
1716         /* If something went wrong, signal */
1717         process_fpemu_return(res, fault_addr, 0);
1718 
1719         if (res == 0)
1720             goto success;
1721         return;
1722 
1723     case mm_lh32_op:
1724         reg = insn.mm_i_format.rt;
1725         goto loadHW;
1726 
1727     case mm_lhu32_op:
1728         reg = insn.mm_i_format.rt;
1729         goto loadHWU;
1730 
1731     case mm_lw32_op:
1732         reg = insn.mm_i_format.rt;
1733         goto loadW;
1734 
1735     case mm_sh32_op:
1736         reg = insn.mm_i_format.rt;
1737         goto storeHW;
1738 
1739     case mm_sw32_op:
1740         reg = insn.mm_i_format.rt;
1741         goto storeW;
1742 
1743     case mm_ld32_op:
1744         reg = insn.mm_i_format.rt;
1745         goto loadDW;
1746 
1747     case mm_sd32_op:
1748         reg = insn.mm_i_format.rt;
1749         goto storeDW;
1750 
1751     case mm_pool16c_op:
1752         switch (insn.mm16_m_format.func) {
1753         case mm_lwm16_op:
1754             reg = insn.mm16_m_format.rlist;
1755             rvar = reg + 1;
1756             if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1757                 goto sigbus;
1758 
1759             for (i = 16; rvar; rvar--, i++) {
1760                 LoadW(addr, value, res);
1761                 if (res)
1762                     goto fault;
1763                 addr += 4;
1764                 regs->regs[i] = value;
1765             }
1766             LoadW(addr, value, res);
1767             if (res)
1768                 goto fault;
1769             regs->regs[31] = value;
1770 
1771             goto success;
1772 
1773         case mm_swm16_op:
1774             reg = insn.mm16_m_format.rlist;
1775             rvar = reg + 1;
1776             if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1777                 goto sigbus;
1778 
1779             for (i = 16; rvar; rvar--, i++) {
1780                 value = regs->regs[i];
1781                 StoreW(addr, value, res);
1782                 if (res)
1783                     goto fault;
1784                 addr += 4;
1785             }
1786             value = regs->regs[31];
1787             StoreW(addr, value, res);
1788             if (res)
1789                 goto fault;
1790 
1791             goto success;
1792 
1793         }
1794 
1795         goto sigbus;
1796 
1797     case mm_lhu16_op:
1798         reg = reg16to32[insn.mm16_rb_format.rt];
1799         goto loadHWU;
1800 
1801     case mm_lw16_op:
1802         reg = reg16to32[insn.mm16_rb_format.rt];
1803         goto loadW;
1804 
1805     case mm_sh16_op:
1806         reg = reg16to32st[insn.mm16_rb_format.rt];
1807         goto storeHW;
1808 
1809     case mm_sw16_op:
1810         reg = reg16to32st[insn.mm16_rb_format.rt];
1811         goto storeW;
1812 
1813     case mm_lwsp16_op:
1814         reg = insn.mm16_r5_format.rt;
1815         goto loadW;
1816 
1817     case mm_swsp16_op:
1818         reg = insn.mm16_r5_format.rt;
1819         goto storeW;
1820 
1821     case mm_lwgp16_op:
1822         reg = reg16to32[insn.mm16_r3_format.rt];
1823         goto loadW;
1824 
1825     default:
1826         goto sigill;
1827     }
1828 
1829 loadHW:
1830     if (!access_ok(VERIFY_READ, addr, 2))
1831         goto sigbus;
1832 
1833     LoadHW(addr, value, res);
1834     if (res)
1835         goto fault;
1836     regs->regs[reg] = value;
1837     goto success;
1838 
1839 loadHWU:
1840     if (!access_ok(VERIFY_READ, addr, 2))
1841         goto sigbus;
1842 
1843     LoadHWU(addr, value, res);
1844     if (res)
1845         goto fault;
1846     regs->regs[reg] = value;
1847     goto success;
1848 
1849 loadW:
1850     if (!access_ok(VERIFY_READ, addr, 4))
1851         goto sigbus;
1852 
1853     LoadW(addr, value, res);
1854     if (res)
1855         goto fault;
1856     regs->regs[reg] = value;
1857     goto success;
1858 
1859 loadWU:
1860 #ifdef CONFIG_64BIT
1861     /*
1862      * A 32-bit kernel might be running on a 64-bit processor.  But
1863      * if we're on a 32-bit processor and an i-cache incoherency
1864      * or race makes us see a 64-bit instruction here the sdl/sdr
1865      * would blow up, so for now we don't handle unaligned 64-bit
1866      * instructions on 32-bit kernels.
1867      */
1868     if (!access_ok(VERIFY_READ, addr, 4))
1869         goto sigbus;
1870 
1871     LoadWU(addr, value, res);
1872     if (res)
1873         goto fault;
1874     regs->regs[reg] = value;
1875     goto success;
1876 #endif /* CONFIG_64BIT */
1877 
1878     /* Cannot handle 64-bit instructions in 32-bit kernel */
1879     goto sigill;
1880 
1881 loadDW:
1882 #ifdef CONFIG_64BIT
1883     /*
1884      * A 32-bit kernel might be running on a 64-bit processor.  But
1885      * if we're on a 32-bit processor and an i-cache incoherency
1886      * or race makes us see a 64-bit instruction here the sdl/sdr
1887      * would blow up, so for now we don't handle unaligned 64-bit
1888      * instructions on 32-bit kernels.
1889      */
1890     if (!access_ok(VERIFY_READ, addr, 8))
1891         goto sigbus;
1892 
1893     LoadDW(addr, value, res);
1894     if (res)
1895         goto fault;
1896     regs->regs[reg] = value;
1897     goto success;
1898 #endif /* CONFIG_64BIT */
1899 
1900     /* Cannot handle 64-bit instructions in 32-bit kernel */
1901     goto sigill;
1902 
1903 storeHW:
1904     if (!access_ok(VERIFY_WRITE, addr, 2))
1905         goto sigbus;
1906 
1907     value = regs->regs[reg];
1908     StoreHW(addr, value, res);
1909     if (res)
1910         goto fault;
1911     goto success;
1912 
1913 storeW:
1914     if (!access_ok(VERIFY_WRITE, addr, 4))
1915         goto sigbus;
1916 
1917     value = regs->regs[reg];
1918     StoreW(addr, value, res);
1919     if (res)
1920         goto fault;
1921     goto success;
1922 
1923 storeDW:
1924 #ifdef CONFIG_64BIT
1925     /*
1926      * A 32-bit kernel might be running on a 64-bit processor.  But
1927      * if we're on a 32-bit processor and an i-cache incoherency
1928      * or race makes us see a 64-bit instruction here the sdl/sdr
1929      * would blow up, so for now we don't handle unaligned 64-bit
1930      * instructions on 32-bit kernels.
1931      */
1932     if (!access_ok(VERIFY_WRITE, addr, 8))
1933         goto sigbus;
1934 
1935     value = regs->regs[reg];
1936     StoreDW(addr, value, res);
1937     if (res)
1938         goto fault;
1939     goto success;
1940 #endif /* CONFIG_64BIT */
1941 
1942     /* Cannot handle 64-bit instructions in 32-bit kernel */
1943     goto sigill;
1944 
1945 success:
1946     regs->cp0_epc = contpc; /* advance or branch */
1947 
1948 #ifdef CONFIG_DEBUG_FS
1949     unaligned_instructions++;
1950 #endif
1951     return;
1952 
1953 fault:
1954     /* roll back jump/branch */
1955     regs->cp0_epc = origpc;
1956     regs->regs[31] = orig31;
1957     /* Did we have an exception handler installed? */
1958     if (fixup_exception(regs))
1959         return;
1960 
1961     die_if_kernel("Unhandled kernel unaligned access", regs);
1962     force_sig(SIGSEGV, current);
1963 
1964     return;
1965 
1966 sigbus:
1967     die_if_kernel("Unhandled kernel unaligned access", regs);
1968     force_sig(SIGBUS, current);
1969 
1970     return;
1971 
1972 sigill:
1973     die_if_kernel
1974         ("Unhandled kernel unaligned access or invalid instruction", regs);
1975     force_sig(SIGILL, current);
1976 }
1977 
1978 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1979 {
1980     unsigned long value;
1981     unsigned int res;
1982     int reg;
1983     unsigned long orig31;
1984     u16 __user *pc16;
1985     unsigned long origpc;
1986     union mips16e_instruction mips16inst, oldinst;
1987 
1988     origpc = regs->cp0_epc;
1989     orig31 = regs->regs[31];
1990     pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1991     /*
1992      * This load never faults.
1993      */
1994     __get_user(mips16inst.full, pc16);
1995     oldinst = mips16inst;
1996 
1997     /* skip EXTEND instruction */
1998     if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1999         pc16++;
2000         __get_user(mips16inst.full, pc16);
2001     } else if (delay_slot(regs)) {
2002         /*  skip jump instructions */
2003         /*  JAL/JALX are 32 bits but have OPCODE in first short int */
2004         if (mips16inst.ri.opcode == MIPS16e_jal_op)
2005             pc16++;
2006         pc16++;
2007         if (get_user(mips16inst.full, pc16))
2008             goto sigbus;
2009     }
2010 
2011     switch (mips16inst.ri.opcode) {
2012     case MIPS16e_i64_op:    /* I64 or RI64 instruction */
2013         switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
2014         case MIPS16e_ldpc_func:
2015         case MIPS16e_ldsp_func:
2016             reg = reg16to32[mips16inst.ri64.ry];
2017             goto loadDW;
2018 
2019         case MIPS16e_sdsp_func:
2020             reg = reg16to32[mips16inst.ri64.ry];
2021             goto writeDW;
2022 
2023         case MIPS16e_sdrasp_func:
2024             reg = 29;   /* GPRSP */
2025             goto writeDW;
2026         }
2027 
2028         goto sigbus;
2029 
2030     case MIPS16e_swsp_op:
2031     case MIPS16e_lwpc_op:
2032     case MIPS16e_lwsp_op:
2033         reg = reg16to32[mips16inst.ri.rx];
2034         break;
2035 
2036     case MIPS16e_i8_op:
2037         if (mips16inst.i8.func != MIPS16e_swrasp_func)
2038             goto sigbus;
2039         reg = 29;   /* GPRSP */
2040         break;
2041 
2042     default:
2043         reg = reg16to32[mips16inst.rri.ry];
2044         break;
2045     }
2046 
2047     switch (mips16inst.ri.opcode) {
2048 
2049     case MIPS16e_lb_op:
2050     case MIPS16e_lbu_op:
2051     case MIPS16e_sb_op:
2052         goto sigbus;
2053 
2054     case MIPS16e_lh_op:
2055         if (!access_ok(VERIFY_READ, addr, 2))
2056             goto sigbus;
2057 
2058         LoadHW(addr, value, res);
2059         if (res)
2060             goto fault;
2061         MIPS16e_compute_return_epc(regs, &oldinst);
2062         regs->regs[reg] = value;
2063         break;
2064 
2065     case MIPS16e_lhu_op:
2066         if (!access_ok(VERIFY_READ, addr, 2))
2067             goto sigbus;
2068 
2069         LoadHWU(addr, value, res);
2070         if (res)
2071             goto fault;
2072         MIPS16e_compute_return_epc(regs, &oldinst);
2073         regs->regs[reg] = value;
2074         break;
2075 
2076     case MIPS16e_lw_op:
2077     case MIPS16e_lwpc_op:
2078     case MIPS16e_lwsp_op:
2079         if (!access_ok(VERIFY_READ, addr, 4))
2080             goto sigbus;
2081 
2082         LoadW(addr, value, res);
2083         if (res)
2084             goto fault;
2085         MIPS16e_compute_return_epc(regs, &oldinst);
2086         regs->regs[reg] = value;
2087         break;
2088 
2089     case MIPS16e_lwu_op:
2090 #ifdef CONFIG_64BIT
2091         /*
2092          * A 32-bit kernel might be running on a 64-bit processor.  But
2093          * if we're on a 32-bit processor and an i-cache incoherency
2094          * or race makes us see a 64-bit instruction here the sdl/sdr
2095          * would blow up, so for now we don't handle unaligned 64-bit
2096          * instructions on 32-bit kernels.
2097          */
2098         if (!access_ok(VERIFY_READ, addr, 4))
2099             goto sigbus;
2100 
2101         LoadWU(addr, value, res);
2102         if (res)
2103             goto fault;
2104         MIPS16e_compute_return_epc(regs, &oldinst);
2105         regs->regs[reg] = value;
2106         break;
2107 #endif /* CONFIG_64BIT */
2108 
2109         /* Cannot handle 64-bit instructions in 32-bit kernel */
2110         goto sigill;
2111 
2112     case MIPS16e_ld_op:
2113 loadDW:
2114 #ifdef CONFIG_64BIT
2115         /*
2116          * A 32-bit kernel might be running on a 64-bit processor.  But
2117          * if we're on a 32-bit processor and an i-cache incoherency
2118          * or race makes us see a 64-bit instruction here the sdl/sdr
2119          * would blow up, so for now we don't handle unaligned 64-bit
2120          * instructions on 32-bit kernels.
2121          */
2122         if (!access_ok(VERIFY_READ, addr, 8))
2123             goto sigbus;
2124 
2125         LoadDW(addr, value, res);
2126         if (res)
2127             goto fault;
2128         MIPS16e_compute_return_epc(regs, &oldinst);
2129         regs->regs[reg] = value;
2130         break;
2131 #endif /* CONFIG_64BIT */
2132 
2133         /* Cannot handle 64-bit instructions in 32-bit kernel */
2134         goto sigill;
2135 
2136     case MIPS16e_sh_op:
2137         if (!access_ok(VERIFY_WRITE, addr, 2))
2138             goto sigbus;
2139 
2140         MIPS16e_compute_return_epc(regs, &oldinst);
2141         value = regs->regs[reg];
2142         StoreHW(addr, value, res);
2143         if (res)
2144             goto fault;
2145         break;
2146 
2147     case MIPS16e_sw_op:
2148     case MIPS16e_swsp_op:
2149     case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */
2150         if (!access_ok(VERIFY_WRITE, addr, 4))
2151             goto sigbus;
2152 
2153         MIPS16e_compute_return_epc(regs, &oldinst);
2154         value = regs->regs[reg];
2155         StoreW(addr, value, res);
2156         if (res)
2157             goto fault;
2158         break;
2159 
2160     case MIPS16e_sd_op:
2161 writeDW:
2162 #ifdef CONFIG_64BIT
2163         /*
2164          * A 32-bit kernel might be running on a 64-bit processor.  But
2165          * if we're on a 32-bit processor and an i-cache incoherency
2166          * or race makes us see a 64-bit instruction here the sdl/sdr
2167          * would blow up, so for now we don't handle unaligned 64-bit
2168          * instructions on 32-bit kernels.
2169          */
2170         if (!access_ok(VERIFY_WRITE, addr, 8))
2171             goto sigbus;
2172 
2173         MIPS16e_compute_return_epc(regs, &oldinst);
2174         value = regs->regs[reg];
2175         StoreDW(addr, value, res);
2176         if (res)
2177             goto fault;
2178         break;
2179 #endif /* CONFIG_64BIT */
2180 
2181         /* Cannot handle 64-bit instructions in 32-bit kernel */
2182         goto sigill;
2183 
2184     default:
2185         /*
2186          * Pheeee...  We encountered an yet unknown instruction or
2187          * cache coherence problem.  Die sucker, die ...
2188          */
2189         goto sigill;
2190     }
2191 
2192 #ifdef CONFIG_DEBUG_FS
2193     unaligned_instructions++;
2194 #endif
2195 
2196     return;
2197 
2198 fault:
2199     /* roll back jump/branch */
2200     regs->cp0_epc = origpc;
2201     regs->regs[31] = orig31;
2202     /* Did we have an exception handler installed? */
2203     if (fixup_exception(regs))
2204         return;
2205 
2206     die_if_kernel("Unhandled kernel unaligned access", regs);
2207     force_sig(SIGSEGV, current);
2208 
2209     return;
2210 
2211 sigbus:
2212     die_if_kernel("Unhandled kernel unaligned access", regs);
2213     force_sig(SIGBUS, current);
2214 
2215     return;
2216 
2217 sigill:
2218     die_if_kernel
2219         ("Unhandled kernel unaligned access or invalid instruction", regs);
2220     force_sig(SIGILL, current);
2221 }
2222 
2223 asmlinkage void do_ade(struct pt_regs *regs)
2224 {
2225     enum ctx_state prev_state;
2226     unsigned int __user *pc;
2227     mm_segment_t seg;
2228 
2229     prev_state = exception_enter();
2230     perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2231             1, regs, regs->cp0_badvaddr);
2232     /*
2233      * Did we catch a fault trying to load an instruction?
2234      */
2235     if (regs->cp0_badvaddr == regs->cp0_epc)
2236         goto sigbus;
2237 
2238     if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2239         goto sigbus;
2240     if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2241         goto sigbus;
2242 
2243     /*
2244      * Do branch emulation only if we didn't forward the exception.
2245      * This is all so but ugly ...
2246      */
2247 
2248     /*
2249      * Are we running in microMIPS mode?
2250      */
2251     if (get_isa16_mode(regs->cp0_epc)) {
2252         /*
2253          * Did we catch a fault trying to load an instruction in
2254          * 16-bit mode?
2255          */
2256         if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2257             goto sigbus;
2258         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2259             show_registers(regs);
2260 
2261         if (cpu_has_mmips) {
2262             seg = get_fs();
2263             if (!user_mode(regs))
2264                 set_fs(KERNEL_DS);
2265             emulate_load_store_microMIPS(regs,
2266                 (void __user *)regs->cp0_badvaddr);
2267             set_fs(seg);
2268 
2269             return;
2270         }
2271 
2272         if (cpu_has_mips16) {
2273             seg = get_fs();
2274             if (!user_mode(regs))
2275                 set_fs(KERNEL_DS);
2276             emulate_load_store_MIPS16e(regs,
2277                 (void __user *)regs->cp0_badvaddr);
2278             set_fs(seg);
2279 
2280             return;
2281     }
2282 
2283         goto sigbus;
2284     }
2285 
2286     if (unaligned_action == UNALIGNED_ACTION_SHOW)
2287         show_registers(regs);
2288     pc = (unsigned int __user *)exception_epc(regs);
2289 
2290     seg = get_fs();
2291     if (!user_mode(regs))
2292         set_fs(KERNEL_DS);
2293     emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2294     set_fs(seg);
2295 
2296     return;
2297 
2298 sigbus:
2299     die_if_kernel("Kernel unaligned instruction access", regs);
2300     force_sig(SIGBUS, current);
2301 
2302     /*
2303      * XXX On return from the signal handler we should advance the epc
2304      */
2305     exception_exit(prev_state);
2306 }
2307 
2308 #ifdef CONFIG_DEBUG_FS
2309 static int __init debugfs_unaligned(void)
2310 {
2311     struct dentry *d;
2312 
2313     if (!mips_debugfs_dir)
2314         return -ENODEV;
2315     d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2316                    mips_debugfs_dir, &unaligned_instructions);
2317     if (!d)
2318         return -ENOMEM;
2319     d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2320                    mips_debugfs_dir, &unaligned_action);
2321     if (!d)
2322         return -ENOMEM;
2323     return 0;
2324 }
2325 arch_initcall(debugfs_unaligned);
2326 #endif