misc/sgi-gru/gru_instructions.h

0001 /*
0002  *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
0003  *
0004  *  This program is free software; you can redistribute it and/or modify
0005  *  it under the terms of the GNU Lesser General Public License as published by
0006  *  the Free Software Foundation; either version 2.1 of the License, or
0007  *  (at your option) any later version.
0008  *
0009  *  This program is distributed in the hope that it will be useful,
0010  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
0011  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0012  *  GNU Lesser General Public License for more details.
0013  *
0014  *  You should have received a copy of the GNU Lesser General Public License
0015  *  along with this program; if not, write to the Free Software
0016  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
0017  */
0018
0019 #ifndef __GRU_INSTRUCTIONS_H__
0020 #define __GRU_INSTRUCTIONS_H__
0021
0022 extern int gru_check_status_proc(void *cb);
0023 extern int gru_wait_proc(void *cb);
0024 extern void gru_wait_abort_proc(void *cb);
0025
0026
0027
0028 /*
0029  * Architecture dependent functions
0030  */
0031
0032 #if defined(CONFIG_IA64)
0033 #include <linux/compiler.h>
0034 #include <asm/intrinsics.h>
0035 #define __flush_cache(p)        ia64_fc((unsigned long)p)
0036 /* Use volatile on IA64 to ensure ordering via st4.rel */
0037 #define gru_ordered_store_ulong(p, v)                   \
0038         do {                            \
0039             barrier();                  \
0040             *((volatile unsigned long *)(p)) = v; /* force st.rel */    \
0041         } while (0)
0042 #elif defined(CONFIG_X86_64)
0043 #include <asm/cacheflush.h>
0044 #define __flush_cache(p)        clflush(p)
0045 #define gru_ordered_store_ulong(p, v)                   \
0046         do {                            \
0047             barrier();                  \
0048             *(unsigned long *)p = v;            \
0049         } while (0)
0050 #else
0051 #error "Unsupported architecture"
0052 #endif
0053
0054 /*
0055  * Control block status and exception codes
0056  */
0057 #define CBS_IDLE            0
0058 #define CBS_EXCEPTION           1
0059 #define CBS_ACTIVE          2
0060 #define CBS_CALL_OS         3
0061
0062 /* CB substatus bitmasks */
0063 #define CBSS_MSG_QUEUE_MASK     7
0064 #define CBSS_IMPLICIT_ABORT_ACTIVE_MASK 8
0065
0066 /* CB substatus message queue values (low 3 bits of substatus) */
0067 #define CBSS_NO_ERROR           0
0068 #define CBSS_LB_OVERFLOWED      1
0069 #define CBSS_QLIMIT_REACHED     2
0070 #define CBSS_PAGE_OVERFLOW      3
0071 #define CBSS_AMO_NACKED         4
0072 #define CBSS_PUT_NACKED         5
0073
0074 /*
0075  * Structure used to fetch exception detail for CBs that terminate with
0076  * CBS_EXCEPTION
0077  */
0078 struct control_block_extended_exc_detail {
0079     unsigned long   cb;
0080     int     opc;
0081     int     ecause;
0082     int     exopc;
0083     long        exceptdet0;
0084     int     exceptdet1;
0085     int     cbrstate;
0086     int     cbrexecstatus;
0087 };
0088
0089 /*
0090  * Instruction formats
0091  */
0092
0093 /*
0094  * Generic instruction format.
0095  * This definition has precise bit field definitions.
0096  */
0097 struct gru_instruction_bits {
0098     /* DW 0  - low */
0099     unsigned int        icmd:      1;
0100     unsigned char       ima:       3;   /* CB_DelRep, unmapped mode */
0101     unsigned char       reserved0: 4;
0102     unsigned int        xtype:     3;
0103     unsigned int        iaa0:      2;
0104     unsigned int        iaa1:      2;
0105     unsigned char       reserved1: 1;
0106     unsigned char       opc:       8;   /* opcode */
0107     unsigned char       exopc:     8;   /* extended opcode */
0108     /* DW 0  - high */
0109     unsigned int        idef2:    22;   /* TRi0 */
0110     unsigned char       reserved2: 2;
0111     unsigned char       istatus:   2;
0112     unsigned char       isubstatus:4;
0113     unsigned char       reserved3: 1;
0114     unsigned char       tlb_fault_color: 1;
0115     /* DW 1 */
0116     unsigned long       idef4;      /* 42 bits: TRi1, BufSize */
0117     /* DW 2-6 */
0118     unsigned long       idef1;      /* BAddr0 */
0119     unsigned long       idef5;      /* Nelem */
0120     unsigned long       idef6;      /* Stride, Operand1 */
0121     unsigned long       idef3;      /* BAddr1, Value, Operand2 */
0122     unsigned long       reserved4;
0123     /* DW 7 */
0124     unsigned long       avalue;      /* AValue */
0125 };
0126
0127 /*
0128  * Generic instruction with friendlier names. This format is used
0129  * for inline instructions.
0130  */
0131 struct gru_instruction {
0132     /* DW 0 */
0133     union {
0134         unsigned long       op64;    /* icmd,xtype,iaa0,ima,opc,tri0 */
0135     struct {
0136         unsigned int    op32;
0137         unsigned int    tri0;
0138     };
0139     };
0140     unsigned long       tri1_bufsize;       /* DW 1 */
0141     unsigned long       baddr0;         /* DW 2 */
0142     unsigned long       nelem;          /* DW 3 */
0143     unsigned long       op1_stride;     /* DW 4 */
0144     unsigned long       op2_value_baddr1;   /* DW 5 */
0145     unsigned long       reserved0;      /* DW 6 */
0146     unsigned long       avalue;         /* DW 7 */
0147 };
0148
0149 /* Some shifts and masks for the low 64 bits of a GRU command */
0150 #define GRU_CB_ICMD_SHFT    0
0151 #define GRU_CB_ICMD_MASK    0x1
0152 #define GRU_CB_XTYPE_SHFT   8
0153 #define GRU_CB_XTYPE_MASK   0x7
0154 #define GRU_CB_IAA0_SHFT    11
0155 #define GRU_CB_IAA0_MASK    0x3
0156 #define GRU_CB_IAA1_SHFT    13
0157 #define GRU_CB_IAA1_MASK    0x3
0158 #define GRU_CB_IMA_SHFT     1
0159 #define GRU_CB_IMA_MASK     0x3
0160 #define GRU_CB_OPC_SHFT     16
0161 #define GRU_CB_OPC_MASK     0xff
0162 #define GRU_CB_EXOPC_SHFT   24
0163 #define GRU_CB_EXOPC_MASK   0xff
0164 #define GRU_IDEF2_SHFT      32
0165 #define GRU_IDEF2_MASK      0x3ffff
0166 #define GRU_ISTATUS_SHFT    56
0167 #define GRU_ISTATUS_MASK    0x3
0168
0169 /* GRU instruction opcodes (opc field) */
0170 #define OP_NOP      0x00
0171 #define OP_BCOPY    0x01
0172 #define OP_VLOAD    0x02
0173 #define OP_IVLOAD   0x03
0174 #define OP_VSTORE   0x04
0175 #define OP_IVSTORE  0x05
0176 #define OP_VSET     0x06
0177 #define OP_IVSET    0x07
0178 #define OP_MESQ     0x08
0179 #define OP_GAMXR    0x09
0180 #define OP_GAMIR    0x0a
0181 #define OP_GAMIRR   0x0b
0182 #define OP_GAMER    0x0c
0183 #define OP_GAMERR   0x0d
0184 #define OP_BSTORE   0x0e
0185 #define OP_VFLUSH   0x0f
0186
0187
0188 /* Extended opcodes values (exopc field) */
0189
0190 /* GAMIR - AMOs with implicit operands */
0191 #define EOP_IR_FETCH    0x01 /* Plain fetch of memory */
0192 #define EOP_IR_CLR  0x02 /* Fetch and clear */
0193 #define EOP_IR_INC  0x05 /* Fetch and increment */
0194 #define EOP_IR_DEC  0x07 /* Fetch and decrement */
0195 #define EOP_IR_QCHK1    0x0d /* Queue check, 64 byte msg */
0196 #define EOP_IR_QCHK2    0x0e /* Queue check, 128 byte msg */
0197
0198 /* GAMIRR - Registered AMOs with implicit operands */
0199 #define EOP_IRR_FETCH   0x01 /* Registered fetch of memory */
0200 #define EOP_IRR_CLR 0x02 /* Registered fetch and clear */
0201 #define EOP_IRR_INC 0x05 /* Registered fetch and increment */
0202 #define EOP_IRR_DEC 0x07 /* Registered fetch and decrement */
0203 #define EOP_IRR_DECZ    0x0f /* Registered fetch and decrement, update on zero*/
0204
0205 /* GAMER - AMOs with explicit operands */
0206 #define EOP_ER_SWAP 0x00 /* Exchange argument and memory */
0207 #define EOP_ER_OR   0x01 /* Logical OR with memory */
0208 #define EOP_ER_AND  0x02 /* Logical AND with memory */
0209 #define EOP_ER_XOR  0x03 /* Logical XOR with memory */
0210 #define EOP_ER_ADD  0x04 /* Add value to memory */
0211 #define EOP_ER_CSWAP    0x08 /* Compare with operand2, write operand1 if match*/
0212 #define EOP_ER_CADD 0x0c /* Queue check, operand1*64 byte msg */
0213
0214 /* GAMERR - Registered AMOs with explicit operands */
0215 #define EOP_ERR_SWAP    0x00 /* Exchange argument and memory */
0216 #define EOP_ERR_OR  0x01 /* Logical OR with memory */
0217 #define EOP_ERR_AND 0x02 /* Logical AND with memory */
0218 #define EOP_ERR_XOR 0x03 /* Logical XOR with memory */
0219 #define EOP_ERR_ADD 0x04 /* Add value to memory */
0220 #define EOP_ERR_CSWAP   0x08 /* Compare with operand2, write operand1 if match*/
0221 #define EOP_ERR_EPOLL   0x09 /* Poll for equality */
0222 #define EOP_ERR_NPOLL   0x0a /* Poll for inequality */
0223
0224 /* GAMXR - SGI Arithmetic unit */
0225 #define EOP_XR_CSWAP    0x0b /* Masked compare exchange */
0226
0227
0228 /* Transfer types (xtype field) */
0229 #define XTYPE_B     0x0 /* byte */
0230 #define XTYPE_S     0x1 /* short (2-byte) */
0231 #define XTYPE_W     0x2 /* word (4-byte) */
0232 #define XTYPE_DW    0x3 /* doubleword (8-byte) */
0233 #define XTYPE_CL    0x6 /* cacheline (64-byte) */
0234
0235
0236 /* Instruction access attributes (iaa0, iaa1 fields) */
0237 #define IAA_RAM     0x0 /* normal cached RAM access */
0238 #define IAA_NCRAM   0x2 /* noncoherent RAM access */
0239 #define IAA_MMIO    0x1 /* noncoherent memory-mapped I/O space */
0240 #define IAA_REGISTER    0x3 /* memory-mapped registers, etc. */
0241
0242
0243 /* Instruction mode attributes (ima field) */
0244 #define IMA_MAPPED  0x0 /* Virtual mode  */
0245 #define IMA_CB_DELAY    0x1 /* hold read responses until status changes */
0246 #define IMA_UNMAPPED    0x2 /* bypass the TLBs (OS only) */
0247 #define IMA_INTERRUPT   0x4 /* Interrupt when instruction completes */
0248
0249 /* CBE ecause bits */
0250 #define CBE_CAUSE_RI                (1 << 0)
0251 #define CBE_CAUSE_INVALID_INSTRUCTION       (1 << 1)
0252 #define CBE_CAUSE_UNMAPPED_MODE_FORBIDDEN   (1 << 2)
0253 #define CBE_CAUSE_PE_CHECK_DATA_ERROR       (1 << 3)
0254 #define CBE_CAUSE_IAA_GAA_MISMATCH      (1 << 4)
0255 #define CBE_CAUSE_DATA_SEGMENT_LIMIT_EXCEPTION  (1 << 5)
0256 #define CBE_CAUSE_OS_FATAL_TLB_FAULT        (1 << 6)
0257 #define CBE_CAUSE_EXECUTION_HW_ERROR        (1 << 7)
0258 #define CBE_CAUSE_TLBHW_ERROR           (1 << 8)
0259 #define CBE_CAUSE_RA_REQUEST_TIMEOUT        (1 << 9)
0260 #define CBE_CAUSE_HA_REQUEST_TIMEOUT        (1 << 10)
0261 #define CBE_CAUSE_RA_RESPONSE_FATAL     (1 << 11)
0262 #define CBE_CAUSE_RA_RESPONSE_NON_FATAL     (1 << 12)
0263 #define CBE_CAUSE_HA_RESPONSE_FATAL     (1 << 13)
0264 #define CBE_CAUSE_HA_RESPONSE_NON_FATAL     (1 << 14)
0265 #define CBE_CAUSE_ADDRESS_SPACE_DECODE_ERROR    (1 << 15)
0266 #define CBE_CAUSE_PROTOCOL_STATE_DATA_ERROR (1 << 16)
0267 #define CBE_CAUSE_RA_RESPONSE_DATA_ERROR    (1 << 17)
0268 #define CBE_CAUSE_HA_RESPONSE_DATA_ERROR    (1 << 18)
0269 #define CBE_CAUSE_FORCED_ERROR          (1 << 19)
0270
0271 /* CBE cbrexecstatus bits */
0272 #define CBR_EXS_ABORT_OCC_BIT           0
0273 #define CBR_EXS_INT_OCC_BIT         1
0274 #define CBR_EXS_PENDING_BIT         2
0275 #define CBR_EXS_QUEUED_BIT          3
0276 #define CBR_EXS_TLB_INVAL_BIT           4
0277 #define CBR_EXS_EXCEPTION_BIT           5
0278 #define CBR_EXS_CB_INT_PENDING_BIT      6
0279
0280 #define CBR_EXS_ABORT_OCC           (1 << CBR_EXS_ABORT_OCC_BIT)
0281 #define CBR_EXS_INT_OCC             (1 << CBR_EXS_INT_OCC_BIT)
0282 #define CBR_EXS_PENDING             (1 << CBR_EXS_PENDING_BIT)
0283 #define CBR_EXS_QUEUED              (1 << CBR_EXS_QUEUED_BIT)
0284 #define CBR_EXS_TLB_INVAL           (1 << CBR_EXS_TLB_INVAL_BIT)
0285 #define CBR_EXS_EXCEPTION           (1 << CBR_EXS_EXCEPTION_BIT)
0286 #define CBR_EXS_CB_INT_PENDING          (1 << CBR_EXS_CB_INT_PENDING_BIT)
0287
0288 /*
0289  * Exceptions are retried for the following cases. If any OTHER bits are set
0290  * in ecause, the exception is not retryable.
0291  */
0292 #define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR |        \
0293                   CBE_CAUSE_TLBHW_ERROR |           \
0294                   CBE_CAUSE_RA_REQUEST_TIMEOUT |        \
0295                   CBE_CAUSE_RA_RESPONSE_NON_FATAL |     \
0296                   CBE_CAUSE_HA_RESPONSE_NON_FATAL |     \
0297                   CBE_CAUSE_RA_RESPONSE_DATA_ERROR |    \
0298                   CBE_CAUSE_HA_RESPONSE_DATA_ERROR      \
0299                   )
0300
0301 /* Message queue head structure */
0302 union gru_mesqhead {
0303     unsigned long   val;
0304     struct {
0305         unsigned int    head;
0306         unsigned int    limit;
0307     };
0308 };
0309
0310
0311 /* Generate the low word of a GRU instruction */
0312 static inline unsigned long
0313 __opdword(unsigned char opcode, unsigned char exopc, unsigned char xtype,
0314        unsigned char iaa0, unsigned char iaa1,
0315        unsigned long idef2, unsigned char ima)
0316 {
0317     return (1 << GRU_CB_ICMD_SHFT) |
0318        ((unsigned long)CBS_ACTIVE << GRU_ISTATUS_SHFT) |
0319        (idef2<< GRU_IDEF2_SHFT) |
0320        (iaa0 << GRU_CB_IAA0_SHFT) |
0321        (iaa1 << GRU_CB_IAA1_SHFT) |
0322        (ima << GRU_CB_IMA_SHFT) |
0323        (xtype << GRU_CB_XTYPE_SHFT) |
0324        (opcode << GRU_CB_OPC_SHFT) |
0325        (exopc << GRU_CB_EXOPC_SHFT);
0326 }
0327
0328 /*
0329  * Architecture specific intrinsics
0330  */
0331 static inline void gru_flush_cache(void *p)
0332 {
0333     __flush_cache(p);
0334 }
0335
0336 /*
0337  * Store the lower 64 bits of the command including the "start" bit. Then
0338  * start the instruction executing.
0339  */
0340 static inline void gru_start_instruction(struct gru_instruction *ins, unsigned long op64)
0341 {
0342     gru_ordered_store_ulong(ins, op64);
0343     mb();
0344     gru_flush_cache(ins);
0345 }
0346
0347
0348 /* Convert "hints" to IMA */
0349 #define CB_IMA(h)       ((h) | IMA_UNMAPPED)
0350
0351 /* Convert data segment cache line index into TRI0 / TRI1 value */
0352 #define GRU_DINDEX(i)       ((i) * GRU_CACHE_LINE_BYTES)
0353
0354 /* Inline functions for GRU instructions.
0355  *     Note:
0356  *      - nelem and stride are in elements
0357  *      - tri0/tri1 is in bytes for the beginning of the data segment.
0358  */
0359 static inline void gru_vload_phys(void *cb, unsigned long gpa,
0360         unsigned int tri0, int iaa, unsigned long hints)
0361 {
0362     struct gru_instruction *ins = (struct gru_instruction *)cb;
0363
0364     ins->baddr0 = (long)gpa | ((unsigned long)iaa << 62);
0365     ins->nelem = 1;
0366     ins->op1_stride = 1;
0367     gru_start_instruction(ins, __opdword(OP_VLOAD, 0, XTYPE_DW, iaa, 0,
0368                     (unsigned long)tri0, CB_IMA(hints)));
0369 }
0370
0371 static inline void gru_vstore_phys(void *cb, unsigned long gpa,
0372         unsigned int tri0, int iaa, unsigned long hints)
0373 {
0374     struct gru_instruction *ins = (struct gru_instruction *)cb;
0375
0376     ins->baddr0 = (long)gpa | ((unsigned long)iaa << 62);
0377     ins->nelem = 1;
0378     ins->op1_stride = 1;
0379     gru_start_instruction(ins, __opdword(OP_VSTORE, 0, XTYPE_DW, iaa, 0,
0380                     (unsigned long)tri0, CB_IMA(hints)));
0381 }
0382
0383 static inline void gru_vload(void *cb, unsigned long mem_addr,
0384         unsigned int tri0, unsigned char xtype, unsigned long nelem,
0385         unsigned long stride, unsigned long hints)
0386 {
0387     struct gru_instruction *ins = (struct gru_instruction *)cb;
0388
0389     ins->baddr0 = (long)mem_addr;
0390     ins->nelem = nelem;
0391     ins->op1_stride = stride;
0392     gru_start_instruction(ins, __opdword(OP_VLOAD, 0, xtype, IAA_RAM, 0,
0393                     (unsigned long)tri0, CB_IMA(hints)));
0394 }
0395
0396 static inline void gru_vstore(void *cb, unsigned long mem_addr,
0397         unsigned int tri0, unsigned char xtype, unsigned long nelem,
0398         unsigned long stride, unsigned long hints)
0399 {
0400     struct gru_instruction *ins = (void *)cb;
0401
0402     ins->baddr0 = (long)mem_addr;
0403     ins->nelem = nelem;
0404     ins->op1_stride = stride;
0405     gru_start_instruction(ins, __opdword(OP_VSTORE, 0, xtype, IAA_RAM, 0,
0406                     tri0, CB_IMA(hints)));
0407 }
0408
0409 static inline void gru_ivload(void *cb, unsigned long mem_addr,
0410         unsigned int tri0, unsigned int tri1, unsigned char xtype,
0411         unsigned long nelem, unsigned long hints)
0412 {
0413     struct gru_instruction *ins = (void *)cb;
0414
0415     ins->baddr0 = (long)mem_addr;
0416     ins->nelem = nelem;
0417     ins->tri1_bufsize = tri1;
0418     gru_start_instruction(ins, __opdword(OP_IVLOAD, 0, xtype, IAA_RAM, 0,
0419                     tri0, CB_IMA(hints)));
0420 }
0421
0422 static inline void gru_ivstore(void *cb, unsigned long mem_addr,
0423         unsigned int tri0, unsigned int tri1,
0424         unsigned char xtype, unsigned long nelem, unsigned long hints)
0425 {
0426     struct gru_instruction *ins = (void *)cb;
0427
0428     ins->baddr0 = (long)mem_addr;
0429     ins->nelem = nelem;
0430     ins->tri1_bufsize = tri1;
0431     gru_start_instruction(ins, __opdword(OP_IVSTORE, 0, xtype, IAA_RAM, 0,
0432                     tri0, CB_IMA(hints)));
0433 }
0434
0435 static inline void gru_vset(void *cb, unsigned long mem_addr,
0436         unsigned long value, unsigned char xtype, unsigned long nelem,
0437         unsigned long stride, unsigned long hints)
0438 {
0439     struct gru_instruction *ins = (void *)cb;
0440
0441     ins->baddr0 = (long)mem_addr;
0442     ins->op2_value_baddr1 = value;
0443     ins->nelem = nelem;
0444     ins->op1_stride = stride;
0445     gru_start_instruction(ins, __opdword(OP_VSET, 0, xtype, IAA_RAM, 0,
0446                      0, CB_IMA(hints)));
0447 }
0448
0449 static inline void gru_ivset(void *cb, unsigned long mem_addr,
0450         unsigned int tri1, unsigned long value, unsigned char xtype,
0451         unsigned long nelem, unsigned long hints)
0452 {
0453     struct gru_instruction *ins = (void *)cb;
0454
0455     ins->baddr0 = (long)mem_addr;
0456     ins->op2_value_baddr1 = value;
0457     ins->nelem = nelem;
0458     ins->tri1_bufsize = tri1;
0459     gru_start_instruction(ins, __opdword(OP_IVSET, 0, xtype, IAA_RAM, 0,
0460                     0, CB_IMA(hints)));
0461 }
0462
0463 static inline void gru_vflush(void *cb, unsigned long mem_addr,
0464         unsigned long nelem, unsigned char xtype, unsigned long stride,
0465         unsigned long hints)
0466 {
0467     struct gru_instruction *ins = (void *)cb;
0468
0469     ins->baddr0 = (long)mem_addr;
0470     ins->op1_stride = stride;
0471     ins->nelem = nelem;
0472     gru_start_instruction(ins, __opdword(OP_VFLUSH, 0, xtype, IAA_RAM, 0,
0473                     0, CB_IMA(hints)));
0474 }
0475
0476 static inline void gru_nop(void *cb, int hints)
0477 {
0478     struct gru_instruction *ins = (void *)cb;
0479
0480     gru_start_instruction(ins, __opdword(OP_NOP, 0, 0, 0, 0, 0, CB_IMA(hints)));
0481 }
0482
0483
0484 static inline void gru_bcopy(void *cb, const unsigned long src,
0485         unsigned long dest,
0486         unsigned int tri0, unsigned int xtype, unsigned long nelem,
0487         unsigned int bufsize, unsigned long hints)
0488 {
0489     struct gru_instruction *ins = (void *)cb;
0490
0491     ins->baddr0 = (long)src;
0492     ins->op2_value_baddr1 = (long)dest;
0493     ins->nelem = nelem;
0494     ins->tri1_bufsize = bufsize;
0495     gru_start_instruction(ins, __opdword(OP_BCOPY, 0, xtype, IAA_RAM,
0496                     IAA_RAM, tri0, CB_IMA(hints)));
0497 }
0498
0499 static inline void gru_bstore(void *cb, const unsigned long src,
0500         unsigned long dest, unsigned int tri0, unsigned int xtype,
0501         unsigned long nelem, unsigned long hints)
0502 {
0503     struct gru_instruction *ins = (void *)cb;
0504
0505     ins->baddr0 = (long)src;
0506     ins->op2_value_baddr1 = (long)dest;
0507     ins->nelem = nelem;
0508     gru_start_instruction(ins, __opdword(OP_BSTORE, 0, xtype, 0, IAA_RAM,
0509                     tri0, CB_IMA(hints)));
0510 }
0511
0512 static inline void gru_gamir(void *cb, int exopc, unsigned long src,
0513         unsigned int xtype, unsigned long hints)
0514 {
0515     struct gru_instruction *ins = (void *)cb;
0516
0517     ins->baddr0 = (long)src;
0518     gru_start_instruction(ins, __opdword(OP_GAMIR, exopc, xtype, IAA_RAM, 0,
0519                     0, CB_IMA(hints)));
0520 }
0521
0522 static inline void gru_gamirr(void *cb, int exopc, unsigned long src,
0523         unsigned int xtype, unsigned long hints)
0524 {
0525     struct gru_instruction *ins = (void *)cb;
0526
0527     ins->baddr0 = (long)src;
0528     gru_start_instruction(ins, __opdword(OP_GAMIRR, exopc, xtype, IAA_RAM, 0,
0529                     0, CB_IMA(hints)));
0530 }
0531
0532 static inline void gru_gamer(void *cb, int exopc, unsigned long src,
0533         unsigned int xtype,
0534         unsigned long operand1, unsigned long operand2,
0535         unsigned long hints)
0536 {
0537     struct gru_instruction *ins = (void *)cb;
0538
0539     ins->baddr0 = (long)src;
0540     ins->op1_stride = operand1;
0541     ins->op2_value_baddr1 = operand2;
0542     gru_start_instruction(ins, __opdword(OP_GAMER, exopc, xtype, IAA_RAM, 0,
0543                     0, CB_IMA(hints)));
0544 }
0545
0546 static inline void gru_gamerr(void *cb, int exopc, unsigned long src,
0547         unsigned int xtype, unsigned long operand1,
0548         unsigned long operand2, unsigned long hints)
0549 {
0550     struct gru_instruction *ins = (void *)cb;
0551
0552     ins->baddr0 = (long)src;
0553     ins->op1_stride = operand1;
0554     ins->op2_value_baddr1 = operand2;
0555     gru_start_instruction(ins, __opdword(OP_GAMERR, exopc, xtype, IAA_RAM, 0,
0556                     0, CB_IMA(hints)));
0557 }
0558
0559 static inline void gru_gamxr(void *cb, unsigned long src,
0560         unsigned int tri0, unsigned long hints)
0561 {
0562     struct gru_instruction *ins = (void *)cb;
0563
0564     ins->baddr0 = (long)src;
0565     ins->nelem = 4;
0566     gru_start_instruction(ins, __opdword(OP_GAMXR, EOP_XR_CSWAP, XTYPE_DW,
0567                  IAA_RAM, 0, 0, CB_IMA(hints)));
0568 }
0569
0570 static inline void gru_mesq(void *cb, unsigned long queue,
0571         unsigned long tri0, unsigned long nelem,
0572         unsigned long hints)
0573 {
0574     struct gru_instruction *ins = (void *)cb;
0575
0576     ins->baddr0 = (long)queue;
0577     ins->nelem = nelem;
0578     gru_start_instruction(ins, __opdword(OP_MESQ, 0, XTYPE_CL, IAA_RAM, 0,
0579                     tri0, CB_IMA(hints)));
0580 }
0581
0582 static inline unsigned long gru_get_amo_value(void *cb)
0583 {
0584     struct gru_instruction *ins = (void *)cb;
0585
0586     return ins->avalue;
0587 }
0588
0589 static inline int gru_get_amo_value_head(void *cb)
0590 {
0591     struct gru_instruction *ins = (void *)cb;
0592
0593     return ins->avalue & 0xffffffff;
0594 }
0595
0596 static inline int gru_get_amo_value_limit(void *cb)
0597 {
0598     struct gru_instruction *ins = (void *)cb;
0599
0600     return ins->avalue >> 32;
0601 }
0602
0603 static inline union gru_mesqhead  gru_mesq_head(int head, int limit)
0604 {
0605     union gru_mesqhead mqh;
0606
0607     mqh.head = head;
0608     mqh.limit = limit;
0609     return mqh;
0610 }
0611
0612 /*
0613  * Get struct control_block_extended_exc_detail for CB.
0614  */
0615 extern int gru_get_cb_exception_detail(void *cb,
0616                struct control_block_extended_exc_detail *excdet);
0617
0618 #define GRU_EXC_STR_SIZE        256
0619
0620
0621 /*
0622  * Control block definition for checking status
0623  */
0624 struct gru_control_block_status {
0625     unsigned int    icmd        :1;
0626     unsigned int    ima     :3;
0627     unsigned int    reserved0   :4;
0628     unsigned int    unused1     :24;
0629     unsigned int    unused2     :24;
0630     unsigned int    istatus     :2;
0631     unsigned int    isubstatus  :4;
0632     unsigned int    unused3     :2;
0633 };
0634
0635 /* Get CB status */
0636 static inline int gru_get_cb_status(void *cb)
0637 {
0638     struct gru_control_block_status *cbs = (void *)cb;
0639
0640     return cbs->istatus;
0641 }
0642
0643 /* Get CB message queue substatus */
0644 static inline int gru_get_cb_message_queue_substatus(void *cb)
0645 {
0646     struct gru_control_block_status *cbs = (void *)cb;
0647
0648     return cbs->isubstatus & CBSS_MSG_QUEUE_MASK;
0649 }
0650
0651 /* Get CB substatus */
0652 static inline int gru_get_cb_substatus(void *cb)
0653 {
0654     struct gru_control_block_status *cbs = (void *)cb;
0655
0656     return cbs->isubstatus;
0657 }
0658
0659 /*
0660  * User interface to check an instruction status. UPM and exceptions
0661  * are handled automatically. However, this function does NOT wait
0662  * for an active instruction to complete.
0663  *
0664  */
0665 static inline int gru_check_status(void *cb)
0666 {
0667     struct gru_control_block_status *cbs = (void *)cb;
0668     int ret;
0669
0670     ret = cbs->istatus;
0671     if (ret != CBS_ACTIVE)
0672         ret = gru_check_status_proc(cb);
0673     return ret;
0674 }
0675
0676 /*
0677  * User interface (via inline function) to wait for an instruction
0678  * to complete. Completion status (IDLE or EXCEPTION is returned
0679  * to the user. Exception due to hardware errors are automatically
0680  * retried before returning an exception.
0681  *
0682  */
0683 static inline int gru_wait(void *cb)
0684 {
0685     return gru_wait_proc(cb);
0686 }
0687
0688 /*
0689  * Wait for CB to complete. Aborts program if error. (Note: error does NOT
0690  * mean TLB mis - only fatal errors such as memory parity error or user
0691  * bugs will cause termination.
0692  */
0693 static inline void gru_wait_abort(void *cb)
0694 {
0695     gru_wait_abort_proc(cb);
0696 }
0697
0698 /*
0699  * Get a pointer to the start of a gseg
0700  *  p   - Any valid pointer within the gseg
0701  */
0702 static inline void *gru_get_gseg_pointer (void *p)
0703 {
0704     return (void *)((unsigned long)p & ~(GRU_GSEG_PAGESIZE - 1));
0705 }
0706
0707 /*
0708  * Get a pointer to a control block
0709  *  gseg    - GSeg address returned from gru_get_thread_gru_segment()
0710  *  index   - index of desired CB
0711  */
0712 static inline void *gru_get_cb_pointer(void *gseg,
0713                               int index)
0714 {
0715     return gseg + GRU_CB_BASE + index * GRU_HANDLE_STRIDE;
0716 }
0717
0718 /*
0719  * Get a pointer to a cacheline in the data segment portion of a GSeg
0720  *  gseg    - GSeg address returned from gru_get_thread_gru_segment()
0721  *  index   - index of desired cache line
0722  */
0723 static inline void *gru_get_data_pointer(void *gseg, int index)
0724 {
0725     return gseg + GRU_DS_BASE + index * GRU_CACHE_LINE_BYTES;
0726 }
0727
0728 /*
0729  * Convert a vaddr into the tri index within the GSEG
0730  *  vaddr       - virtual address of within gseg
0731  */
0732 static inline int gru_get_tri(void *vaddr)
0733 {
0734     return ((unsigned long)vaddr & (GRU_GSEG_PAGESIZE - 1)) - GRU_DS_BASE;
0735 }
0736 #endif      /* __GRU_INSTRUCTIONS_H__ */