Back to home page

LXR

 
 

    


0001 /*
0002  * This file is subject to the terms and conditions of the GNU General Public
0003  * License.  See the file "COPYING" in the main directory of this archive
0004  * for more details.
0005  *
0006  * Copyright (C) 2003-2006 Silicon Graphics, Inc.  All Rights Reserved.
0007  */
0008 
0009 #include <linux/types.h>
0010 #include <linux/interrupt.h>
0011 #include <linux/slab.h>
0012 #include <linux/pci.h>
0013 #include <asm/sn/sn_sal.h>
0014 #include <asm/sn/addrs.h>
0015 #include <asm/sn/io.h>
0016 #include <asm/sn/pcidev.h>
0017 #include <asm/sn/pcibus_provider_defs.h>
0018 #include <asm/sn/tioce_provider.h>
0019 
0020 /*
0021  * 1/26/2006
0022  *
0023  * WAR for SGI PV 944642.  For revA TIOCE, need to use the following recipe
0024  * (taken from the above PV) before and after accessing tioce internal MMR's
0025  * to avoid tioce lockups.
0026  *
0027  * The recipe as taken from the PV:
0028  *
0029  *  if(mmr address < 0x45000) {
0030  *      if(mmr address == 0 or 0x80)
0031  *          mmr wrt or read address 0xc0
0032  *      else if(mmr address == 0x148 or 0x200)
0033  *          mmr wrt or read address 0x28
0034  *      else
0035  *          mmr wrt or read address 0x158
0036  *
0037  *      do desired mmr access (rd or wrt)
0038  *
0039  *      if(mmr address == 0x100)
0040  *          mmr wrt or read address 0x38
0041  *      mmr wrt or read address 0xb050
0042  *  } else
0043  *      do desired mmr access
0044  *
0045  * According to hw, we can use reads instead of writes to the above address
0046  *
0047  * Note this WAR can only to be used for accessing internal MMR's in the
0048  * TIOCE Coretalk Address Range 0x0 - 0x07ff_ffff.  This includes the
0049  * "Local CE Registers and Memories" and "PCI Compatible Config Space" address
0050  * spaces from table 2-1 of the "CE Programmer's Reference Overview" document.
0051  *
0052  * All registers defined in struct tioce will meet that criteria.
0053  */
0054 
0055 static void inline
0056 tioce_mmr_war_pre(struct tioce_kernel *kern, void __iomem *mmr_addr)
0057 {
0058     u64 mmr_base;
0059     u64 mmr_offset;
0060 
0061     if (kern->ce_common->ce_rev != TIOCE_REV_A)
0062         return;
0063 
0064     mmr_base = kern->ce_common->ce_pcibus.bs_base;
0065     mmr_offset = (unsigned long)mmr_addr - mmr_base;
0066 
0067     if (mmr_offset < 0x45000) {
0068         u64 mmr_war_offset;
0069 
0070         if (mmr_offset == 0 || mmr_offset == 0x80)
0071             mmr_war_offset = 0xc0;
0072         else if (mmr_offset == 0x148 || mmr_offset == 0x200)
0073             mmr_war_offset = 0x28;
0074         else
0075             mmr_war_offset = 0x158;
0076 
0077         readq_relaxed((void __iomem *)(mmr_base + mmr_war_offset));
0078     }
0079 }
0080 
0081 static void inline
0082 tioce_mmr_war_post(struct tioce_kernel *kern, void __iomem *mmr_addr)
0083 {
0084     u64 mmr_base;
0085     u64 mmr_offset;
0086 
0087     if (kern->ce_common->ce_rev != TIOCE_REV_A)
0088         return;
0089 
0090     mmr_base = kern->ce_common->ce_pcibus.bs_base;
0091     mmr_offset = (unsigned long)mmr_addr - mmr_base;
0092 
0093     if (mmr_offset < 0x45000) {
0094         if (mmr_offset == 0x100)
0095             readq_relaxed((void __iomem *)(mmr_base + 0x38));
0096         readq_relaxed((void __iomem *)(mmr_base + 0xb050));
0097     }
0098 }
0099 
0100 /* load mmr contents into a variable */
0101 #define tioce_mmr_load(kern, mmrp, varp) do {\
0102     tioce_mmr_war_pre(kern, mmrp); \
0103     *(varp) = readq_relaxed(mmrp); \
0104     tioce_mmr_war_post(kern, mmrp); \
0105 } while (0)
0106 
0107 /* store variable contents into mmr */
0108 #define tioce_mmr_store(kern, mmrp, varp) do {\
0109     tioce_mmr_war_pre(kern, mmrp); \
0110     writeq(*varp, mmrp); \
0111     tioce_mmr_war_post(kern, mmrp); \
0112 } while (0)
0113 
0114 /* store immediate value into mmr */
0115 #define tioce_mmr_storei(kern, mmrp, val) do {\
0116     tioce_mmr_war_pre(kern, mmrp); \
0117     writeq(val, mmrp); \
0118     tioce_mmr_war_post(kern, mmrp); \
0119 } while (0)
0120 
0121 /* set bits (immediate value) into mmr */
0122 #define tioce_mmr_seti(kern, mmrp, bits) do {\
0123     u64 tmp; \
0124     tioce_mmr_load(kern, mmrp, &tmp); \
0125     tmp |= (bits); \
0126     tioce_mmr_store(kern, mmrp, &tmp); \
0127 } while (0)
0128 
0129 /* clear bits (immediate value) into mmr */
0130 #define tioce_mmr_clri(kern, mmrp, bits) do { \
0131     u64 tmp; \
0132     tioce_mmr_load(kern, mmrp, &tmp); \
0133     tmp &= ~(bits); \
0134     tioce_mmr_store(kern, mmrp, &tmp); \
0135 } while (0)
0136 
0137 /**
0138  * Bus address ranges for the 5 flavors of TIOCE DMA
0139  */
0140 
0141 #define TIOCE_D64_MIN   0x8000000000000000UL
0142 #define TIOCE_D64_MAX   0xffffffffffffffffUL
0143 #define TIOCE_D64_ADDR(a)   ((a) >= TIOCE_D64_MIN)
0144 
0145 #define TIOCE_D32_MIN   0x0000000080000000UL
0146 #define TIOCE_D32_MAX   0x00000000ffffffffUL
0147 #define TIOCE_D32_ADDR(a)   ((a) >= TIOCE_D32_MIN && (a) <= TIOCE_D32_MAX)
0148 
0149 #define TIOCE_M32_MIN   0x0000000000000000UL
0150 #define TIOCE_M32_MAX   0x000000007fffffffUL
0151 #define TIOCE_M32_ADDR(a)   ((a) >= TIOCE_M32_MIN && (a) <= TIOCE_M32_MAX)
0152 
0153 #define TIOCE_M40_MIN   0x0000004000000000UL
0154 #define TIOCE_M40_MAX   0x0000007fffffffffUL
0155 #define TIOCE_M40_ADDR(a)   ((a) >= TIOCE_M40_MIN && (a) <= TIOCE_M40_MAX)
0156 
0157 #define TIOCE_M40S_MIN  0x0000008000000000UL
0158 #define TIOCE_M40S_MAX  0x000000ffffffffffUL
0159 #define TIOCE_M40S_ADDR(a)  ((a) >= TIOCE_M40S_MIN && (a) <= TIOCE_M40S_MAX)
0160 
0161 /*
0162  * ATE manipulation macros.
0163  */
0164 
0165 #define ATE_PAGESHIFT(ps)   (__ffs(ps))
0166 #define ATE_PAGEMASK(ps)    ((ps)-1)
0167 
0168 #define ATE_PAGE(x, ps) ((x) >> ATE_PAGESHIFT(ps))
0169 #define ATE_NPAGES(start, len, pagesize) \
0170     (ATE_PAGE((start)+(len)-1, pagesize) - ATE_PAGE(start, pagesize) + 1)
0171 
0172 #define ATE_VALID(ate)  ((ate) & (1UL << 63))
0173 #define ATE_MAKE(addr, ps, msi) \
0174     (((addr) & ~ATE_PAGEMASK(ps)) | (1UL << 63) | ((msi)?(1UL << 62):0))
0175 
0176 /*
0177  * Flavors of ate-based mapping supported by tioce_alloc_map()
0178  */
0179 
0180 #define TIOCE_ATE_M32   1
0181 #define TIOCE_ATE_M40   2
0182 #define TIOCE_ATE_M40S  3
0183 
0184 #define KB(x)   ((u64)(x) << 10)
0185 #define MB(x)   ((u64)(x) << 20)
0186 #define GB(x)   ((u64)(x) << 30)
0187 
0188 /**
0189  * tioce_dma_d64 - create a DMA mapping using 64-bit direct mode
0190  * @ct_addr: system coretalk address
0191  *
0192  * Map @ct_addr into 64-bit CE bus space.  No device context is necessary
0193  * and no CE mapping are consumed.
0194  *
0195  * Bits 53:0 come from the coretalk address.  The remaining bits are set as
0196  * follows:
0197  *
0198  * 63    - must be 1 to indicate d64 mode to CE hardware
0199  * 62    - barrier bit ... controlled with tioce_dma_barrier()
0200  * 61    - msi bit ... specified through dma_flags
0201  * 60:54 - reserved, MBZ
0202  */
0203 static u64
0204 tioce_dma_d64(unsigned long ct_addr, int dma_flags)
0205 {
0206     u64 bus_addr;
0207 
0208     bus_addr = ct_addr | (1UL << 63);
0209     if (dma_flags & SN_DMA_MSI)
0210         bus_addr |= (1UL << 61);
0211 
0212     return bus_addr;
0213 }
0214 
0215 /**
0216  * pcidev_to_tioce - return misc ce related pointers given a pci_dev
0217  * @pci_dev: pci device context
0218  * @base: ptr to store struct tioce_mmr * for the CE holding this device
0219  * @kernel: ptr to store struct tioce_kernel * for the CE holding this device
0220  * @port: ptr to store the CE port number that this device is on
0221  *
0222  * Return pointers to various CE-related structures for the CE upstream of
0223  * @pci_dev.
0224  */
0225 static inline void
0226 pcidev_to_tioce(struct pci_dev *pdev, struct tioce __iomem **base,
0227         struct tioce_kernel **kernel, int *port)
0228 {
0229     struct pcidev_info *pcidev_info;
0230     struct tioce_common *ce_common;
0231     struct tioce_kernel *ce_kernel;
0232 
0233     pcidev_info = SN_PCIDEV_INFO(pdev);
0234     ce_common = (struct tioce_common *)pcidev_info->pdi_pcibus_info;
0235     ce_kernel = (struct tioce_kernel *)ce_common->ce_kernel_private;
0236 
0237     if (base)
0238         *base = (struct tioce __iomem *)ce_common->ce_pcibus.bs_base;
0239     if (kernel)
0240         *kernel = ce_kernel;
0241 
0242     /*
0243      * we use port as a zero-based value internally, even though the
0244      * documentation is 1-based.
0245      */
0246     if (port)
0247         *port =
0248             (pdev->bus->number < ce_kernel->ce_port1_secondary) ? 0 : 1;
0249 }
0250 
0251 /**
0252  * tioce_alloc_map - Given a coretalk address, map it to pcie bus address
0253  * space using one of the various ATE-based address modes.
0254  * @ce_kern: tioce context
0255  * @type: map mode to use
0256  * @port: 0-based port that the requesting device is downstream of
0257  * @ct_addr: the coretalk address to map
0258  * @len: number of bytes to map
0259  *
0260  * Given the addressing type, set up various parameters that define the
0261  * ATE pool to use.  Search for a contiguous block of entries to cover the
0262  * length, and if enough resources exist, fill in the ATEs and construct a
0263  * tioce_dmamap struct to track the mapping.
0264  */
0265 static u64
0266 tioce_alloc_map(struct tioce_kernel *ce_kern, int type, int port,
0267         u64 ct_addr, int len, int dma_flags)
0268 {
0269     int i;
0270     int j;
0271     int first;
0272     int last;
0273     int entries;
0274     int nates;
0275     u64 pagesize;
0276     int msi_capable, msi_wanted;
0277     u64 *ate_shadow;
0278     u64 __iomem *ate_reg;
0279     u64 addr;
0280     struct tioce __iomem *ce_mmr;
0281     u64 bus_base;
0282     struct tioce_dmamap *map;
0283 
0284     ce_mmr = (struct tioce __iomem *)ce_kern->ce_common->ce_pcibus.bs_base;
0285 
0286     switch (type) {
0287     case TIOCE_ATE_M32:
0288         /*
0289          * The first 64 entries of the ate3240 pool are dedicated to
0290          * super-page (TIOCE_ATE_M40S) mode.
0291          */
0292         first = 64;
0293         entries = TIOCE_NUM_M3240_ATES - 64;
0294         ate_shadow = ce_kern->ce_ate3240_shadow;
0295         ate_reg = ce_mmr->ce_ure_ate3240;
0296         pagesize = ce_kern->ce_ate3240_pagesize;
0297         bus_base = TIOCE_M32_MIN;
0298         msi_capable = 1;
0299         break;
0300     case TIOCE_ATE_M40:
0301         first = 0;
0302         entries = TIOCE_NUM_M40_ATES;
0303         ate_shadow = ce_kern->ce_ate40_shadow;
0304         ate_reg = ce_mmr->ce_ure_ate40;
0305         pagesize = MB(64);
0306         bus_base = TIOCE_M40_MIN;
0307         msi_capable = 0;
0308         break;
0309     case TIOCE_ATE_M40S:
0310         /*
0311          * ate3240 entries 0-31 are dedicated to port1 super-page
0312          * mappings.  ate3240 entries 32-63 are dedicated to port2.
0313          */
0314         first = port * 32;
0315         entries = 32;
0316         ate_shadow = ce_kern->ce_ate3240_shadow;
0317         ate_reg = ce_mmr->ce_ure_ate3240;
0318         pagesize = GB(16);
0319         bus_base = TIOCE_M40S_MIN;
0320         msi_capable = 0;
0321         break;
0322     default:
0323         return 0;
0324     }
0325 
0326     msi_wanted = dma_flags & SN_DMA_MSI;
0327     if (msi_wanted && !msi_capable)
0328         return 0;
0329 
0330     nates = ATE_NPAGES(ct_addr, len, pagesize);
0331     if (nates > entries)
0332         return 0;
0333 
0334     last = first + entries - nates;
0335     for (i = first; i <= last; i++) {
0336         if (ATE_VALID(ate_shadow[i]))
0337             continue;
0338 
0339         for (j = i; j < i + nates; j++)
0340             if (ATE_VALID(ate_shadow[j]))
0341                 break;
0342 
0343         if (j >= i + nates)
0344             break;
0345     }
0346 
0347     if (i > last)
0348         return 0;
0349 
0350     map = kzalloc(sizeof(struct tioce_dmamap), GFP_ATOMIC);
0351     if (!map)
0352         return 0;
0353 
0354     addr = ct_addr;
0355     for (j = 0; j < nates; j++) {
0356         u64 ate;
0357 
0358         ate = ATE_MAKE(addr, pagesize, msi_wanted);
0359         ate_shadow[i + j] = ate;
0360         tioce_mmr_storei(ce_kern, &ate_reg[i + j], ate);
0361         addr += pagesize;
0362     }
0363 
0364     map->refcnt = 1;
0365     map->nbytes = nates * pagesize;
0366     map->ct_start = ct_addr & ~ATE_PAGEMASK(pagesize);
0367     map->pci_start = bus_base + (i * pagesize);
0368     map->ate_hw = &ate_reg[i];
0369     map->ate_shadow = &ate_shadow[i];
0370     map->ate_count = nates;
0371 
0372     list_add(&map->ce_dmamap_list, &ce_kern->ce_dmamap_list);
0373 
0374     return (map->pci_start + (ct_addr - map->ct_start));
0375 }
0376 
0377 /**
0378  * tioce_dma_d32 - create a DMA mapping using 32-bit direct mode
0379  * @pdev: linux pci_dev representing the function
0380  * @paddr: system physical address
0381  *
0382  * Map @paddr into 32-bit bus space of the CE associated with @pcidev_info.
0383  */
0384 static u64
0385 tioce_dma_d32(struct pci_dev *pdev, u64 ct_addr, int dma_flags)
0386 {
0387     int dma_ok;
0388     int port;
0389     struct tioce __iomem *ce_mmr;
0390     struct tioce_kernel *ce_kern;
0391     u64 ct_upper;
0392     u64 ct_lower;
0393     dma_addr_t bus_addr;
0394 
0395     if (dma_flags & SN_DMA_MSI)
0396         return 0;
0397 
0398     ct_upper = ct_addr & ~0x3fffffffUL;
0399     ct_lower = ct_addr & 0x3fffffffUL;
0400 
0401     pcidev_to_tioce(pdev, &ce_mmr, &ce_kern, &port);
0402 
0403     if (ce_kern->ce_port[port].dirmap_refcnt == 0) {
0404         u64 tmp;
0405 
0406         ce_kern->ce_port[port].dirmap_shadow = ct_upper;
0407         tioce_mmr_storei(ce_kern, &ce_mmr->ce_ure_dir_map[port],
0408                  ct_upper);
0409         tmp = ce_mmr->ce_ure_dir_map[port];
0410         dma_ok = 1;
0411     } else
0412         dma_ok = (ce_kern->ce_port[port].dirmap_shadow == ct_upper);
0413 
0414     if (dma_ok) {
0415         ce_kern->ce_port[port].dirmap_refcnt++;
0416         bus_addr = TIOCE_D32_MIN + ct_lower;
0417     } else
0418         bus_addr = 0;
0419 
0420     return bus_addr;
0421 }
0422 
0423 /**
0424  * tioce_dma_barrier - swizzle a TIOCE bus address to include or exclude
0425  * the barrier bit.
0426  * @bus_addr:  bus address to swizzle
0427  *
0428  * Given a TIOCE bus address, set the appropriate bit to indicate barrier
0429  * attributes.
0430  */
0431 static u64
0432 tioce_dma_barrier(u64 bus_addr, int on)
0433 {
0434     u64 barrier_bit;
0435 
0436     /* barrier not supported in M40/M40S mode */
0437     if (TIOCE_M40_ADDR(bus_addr) || TIOCE_M40S_ADDR(bus_addr))
0438         return bus_addr;
0439 
0440     if (TIOCE_D64_ADDR(bus_addr))
0441         barrier_bit = (1UL << 62);
0442     else            /* must be m32 or d32 */
0443         barrier_bit = (1UL << 30);
0444 
0445     return (on) ? (bus_addr | barrier_bit) : (bus_addr & ~barrier_bit);
0446 }
0447 
0448 /**
0449  * tioce_dma_unmap - release CE mapping resources
0450  * @pdev: linux pci_dev representing the function
0451  * @bus_addr: bus address returned by an earlier tioce_dma_map
0452  * @dir: mapping direction (unused)
0453  *
0454  * Locate mapping resources associated with @bus_addr and release them.
0455  * For mappings created using the direct modes there are no resources
0456  * to release.
0457  */
0458 void
0459 tioce_dma_unmap(struct pci_dev *pdev, dma_addr_t bus_addr, int dir)
0460 {
0461     int i;
0462     int port;
0463     struct tioce_kernel *ce_kern;
0464     struct tioce __iomem *ce_mmr;
0465     unsigned long flags;
0466 
0467     bus_addr = tioce_dma_barrier(bus_addr, 0);
0468     pcidev_to_tioce(pdev, &ce_mmr, &ce_kern, &port);
0469 
0470     /* nothing to do for D64 */
0471 
0472     if (TIOCE_D64_ADDR(bus_addr))
0473         return;
0474 
0475     spin_lock_irqsave(&ce_kern->ce_lock, flags);
0476 
0477     if (TIOCE_D32_ADDR(bus_addr)) {
0478         if (--ce_kern->ce_port[port].dirmap_refcnt == 0) {
0479             ce_kern->ce_port[port].dirmap_shadow = 0;
0480             tioce_mmr_storei(ce_kern, &ce_mmr->ce_ure_dir_map[port],
0481                      0);
0482         }
0483     } else {
0484         struct tioce_dmamap *map;
0485 
0486         list_for_each_entry(map, &ce_kern->ce_dmamap_list,
0487                     ce_dmamap_list) {
0488             u64 last;
0489 
0490             last = map->pci_start + map->nbytes - 1;
0491             if (bus_addr >= map->pci_start && bus_addr <= last)
0492                 break;
0493         }
0494 
0495         if (&map->ce_dmamap_list == &ce_kern->ce_dmamap_list) {
0496             printk(KERN_WARNING
0497                    "%s:  %s - no map found for bus_addr 0x%llx\n",
0498                    __func__, pci_name(pdev), bus_addr);
0499         } else if (--map->refcnt == 0) {
0500             for (i = 0; i < map->ate_count; i++) {
0501                 map->ate_shadow[i] = 0;
0502                 tioce_mmr_storei(ce_kern, &map->ate_hw[i], 0);
0503             }
0504 
0505             list_del(&map->ce_dmamap_list);
0506             kfree(map);
0507         }
0508     }
0509 
0510     spin_unlock_irqrestore(&ce_kern->ce_lock, flags);
0511 }
0512 
0513 /**
0514  * tioce_do_dma_map - map pages for PCI DMA
0515  * @pdev: linux pci_dev representing the function
0516  * @paddr: host physical address to map
0517  * @byte_count: bytes to map
0518  *
0519  * This is the main wrapper for mapping host physical pages to CE PCI space.
0520  * The mapping mode used is based on the device's dma_mask.
0521  */
0522 static u64
0523 tioce_do_dma_map(struct pci_dev *pdev, u64 paddr, size_t byte_count,
0524          int barrier, int dma_flags)
0525 {
0526     unsigned long flags;
0527     u64 ct_addr;
0528     u64 mapaddr = 0;
0529     struct tioce_kernel *ce_kern;
0530     struct tioce_dmamap *map;
0531     int port;
0532     u64 dma_mask;
0533 
0534     dma_mask = (barrier) ? pdev->dev.coherent_dma_mask : pdev->dma_mask;
0535 
0536     /* cards must be able to address at least 31 bits */
0537     if (dma_mask < 0x7fffffffUL)
0538         return 0;
0539 
0540     if (SN_DMA_ADDRTYPE(dma_flags) == SN_DMA_ADDR_PHYS)
0541         ct_addr = PHYS_TO_TIODMA(paddr);
0542     else
0543         ct_addr = paddr;
0544 
0545     /*
0546      * If the device can generate 64 bit addresses, create a D64 map.
0547      */
0548     if (dma_mask == ~0UL) {
0549         mapaddr = tioce_dma_d64(ct_addr, dma_flags);
0550         if (mapaddr)
0551             goto dma_map_done;
0552     }
0553 
0554     pcidev_to_tioce(pdev, NULL, &ce_kern, &port);
0555 
0556     spin_lock_irqsave(&ce_kern->ce_lock, flags);
0557 
0558     /*
0559      * D64 didn't work ... See if we have an existing map that covers
0560      * this address range.  Must account for devices dma_mask here since
0561      * an existing map might have been done in a mode using more pci
0562      * address bits than this device can support.
0563      */
0564     list_for_each_entry(map, &ce_kern->ce_dmamap_list, ce_dmamap_list) {
0565         u64 last;
0566 
0567         last = map->ct_start + map->nbytes - 1;
0568         if (ct_addr >= map->ct_start &&
0569             ct_addr + byte_count - 1 <= last &&
0570             map->pci_start <= dma_mask) {
0571             map->refcnt++;
0572             mapaddr = map->pci_start + (ct_addr - map->ct_start);
0573             break;
0574         }
0575     }
0576 
0577     /*
0578      * If we don't have a map yet, and the card can generate 40
0579      * bit addresses, try the M40/M40S modes.  Note these modes do not
0580      * support a barrier bit, so if we need a consistent map these
0581      * won't work.
0582      */
0583     if (!mapaddr && !barrier && dma_mask >= 0xffffffffffUL) {
0584         /*
0585          * We have two options for 40-bit mappings:  16GB "super" ATEs
0586          * and 64MB "regular" ATEs.  We'll try both if needed for a
0587          * given mapping but which one we try first depends on the
0588          * size.  For requests >64MB, prefer to use a super page with
0589          * regular as the fallback. Otherwise, try in the reverse order.
0590          */
0591 
0592         if (byte_count > MB(64)) {
0593             mapaddr = tioce_alloc_map(ce_kern, TIOCE_ATE_M40S,
0594                           port, ct_addr, byte_count,
0595                           dma_flags);
0596             if (!mapaddr)
0597                 mapaddr =
0598                     tioce_alloc_map(ce_kern, TIOCE_ATE_M40, -1,
0599                             ct_addr, byte_count,
0600                             dma_flags);
0601         } else {
0602             mapaddr = tioce_alloc_map(ce_kern, TIOCE_ATE_M40, -1,
0603                           ct_addr, byte_count,
0604                           dma_flags);
0605             if (!mapaddr)
0606                 mapaddr =
0607                     tioce_alloc_map(ce_kern, TIOCE_ATE_M40S,
0608                             port, ct_addr, byte_count,
0609                             dma_flags);
0610         }
0611     }
0612 
0613     /*
0614      * 32-bit direct is the next mode to try
0615      */
0616     if (!mapaddr && dma_mask >= 0xffffffffUL)
0617         mapaddr = tioce_dma_d32(pdev, ct_addr, dma_flags);
0618 
0619     /*
0620      * Last resort, try 32-bit ATE-based map.
0621      */
0622     if (!mapaddr)
0623         mapaddr =
0624             tioce_alloc_map(ce_kern, TIOCE_ATE_M32, -1, ct_addr,
0625                     byte_count, dma_flags);
0626 
0627     spin_unlock_irqrestore(&ce_kern->ce_lock, flags);
0628 
0629 dma_map_done:
0630     if (mapaddr && barrier)
0631         mapaddr = tioce_dma_barrier(mapaddr, 1);
0632 
0633     return mapaddr;
0634 }
0635 
0636 /**
0637  * tioce_dma - standard pci dma map interface
0638  * @pdev: pci device requesting the map
0639  * @paddr: system physical address to map into pci space
0640  * @byte_count: # bytes to map
0641  *
0642  * Simply call tioce_do_dma_map() to create a map with the barrier bit clear
0643  * in the address.
0644  */
0645 static u64
0646 tioce_dma(struct pci_dev *pdev, unsigned long  paddr, size_t byte_count, int dma_flags)
0647 {
0648     return tioce_do_dma_map(pdev, paddr, byte_count, 0, dma_flags);
0649 }
0650 
0651 /**
0652  * tioce_dma_consistent - consistent pci dma map interface
0653  * @pdev: pci device requesting the map
0654  * @paddr: system physical address to map into pci space
0655  * @byte_count: # bytes to map
0656  *
0657  * Simply call tioce_do_dma_map() to create a map with the barrier bit set
0658  * in the address.
0659  */
0660 static u64
0661 tioce_dma_consistent(struct pci_dev *pdev, unsigned long  paddr, size_t byte_count, int dma_flags)
0662 {
0663     return tioce_do_dma_map(pdev, paddr, byte_count, 1, dma_flags);
0664 }
0665 
0666 /**
0667  * tioce_error_intr_handler - SGI TIO CE error interrupt handler
0668  * @irq: unused
0669  * @arg: pointer to tioce_common struct for the given CE
0670  *
0671  * Handle a CE error interrupt.  Simply a wrapper around a SAL call which
0672  * defers processing to the SGI prom.
0673  */
0674 static irqreturn_t
0675 tioce_error_intr_handler(int irq, void *arg)
0676 {
0677     struct tioce_common *soft = arg;
0678     struct ia64_sal_retval ret_stuff;
0679     ret_stuff.status = 0;
0680     ret_stuff.v0 = 0;
0681 
0682     SAL_CALL_NOLOCK(ret_stuff, (u64) SN_SAL_IOIF_ERROR_INTERRUPT,
0683             soft->ce_pcibus.bs_persist_segment,
0684             soft->ce_pcibus.bs_persist_busnum, 0, 0, 0, 0, 0);
0685 
0686     if (ret_stuff.v0)
0687         panic("tioce_error_intr_handler:  Fatal TIOCE error");
0688 
0689     return IRQ_HANDLED;
0690 }
0691 
0692 /**
0693  * tioce_reserve_m32 - reserve M32 ATEs for the indicated address range
0694  * @tioce_kernel: TIOCE context to reserve ATEs for
0695  * @base: starting bus address to reserve
0696  * @limit: last bus address to reserve
0697  *
0698  * If base/limit falls within the range of bus space mapped through the
0699  * M32 space, reserve the resources corresponding to the range.
0700  */
0701 static void
0702 tioce_reserve_m32(struct tioce_kernel *ce_kern, u64 base, u64 limit)
0703 {
0704     int ate_index, last_ate, ps;
0705     struct tioce __iomem *ce_mmr;
0706 
0707     ce_mmr = (struct tioce __iomem *)ce_kern->ce_common->ce_pcibus.bs_base;
0708     ps = ce_kern->ce_ate3240_pagesize;
0709     ate_index = ATE_PAGE(base, ps);
0710     last_ate = ate_index + ATE_NPAGES(base, limit-base+1, ps) - 1;
0711 
0712     if (ate_index < 64)
0713         ate_index = 64;
0714 
0715     if (last_ate >= TIOCE_NUM_M3240_ATES)
0716         last_ate = TIOCE_NUM_M3240_ATES - 1;
0717 
0718     while (ate_index <= last_ate) {
0719         u64 ate;
0720 
0721         ate = ATE_MAKE(0xdeadbeef, ps, 0);
0722         ce_kern->ce_ate3240_shadow[ate_index] = ate;
0723         tioce_mmr_storei(ce_kern, &ce_mmr->ce_ure_ate3240[ate_index],
0724                  ate);
0725         ate_index++;
0726     }
0727 }
0728 
0729 /**
0730  * tioce_kern_init - init kernel structures related to a given TIOCE
0731  * @tioce_common: ptr to a cached tioce_common struct that originated in prom
0732  */
0733 static struct tioce_kernel *
0734 tioce_kern_init(struct tioce_common *tioce_common)
0735 {
0736     int i;
0737     int ps;
0738     int dev;
0739     u32 tmp;
0740     unsigned int seg, bus;
0741     struct tioce __iomem *tioce_mmr;
0742     struct tioce_kernel *tioce_kern;
0743 
0744     tioce_kern = kzalloc(sizeof(struct tioce_kernel), GFP_KERNEL);
0745     if (!tioce_kern) {
0746         return NULL;
0747     }
0748 
0749     tioce_kern->ce_common = tioce_common;
0750     spin_lock_init(&tioce_kern->ce_lock);
0751     INIT_LIST_HEAD(&tioce_kern->ce_dmamap_list);
0752     tioce_common->ce_kernel_private = (u64) tioce_kern;
0753 
0754     /*
0755      * Determine the secondary bus number of the port2 logical PPB.
0756      * This is used to decide whether a given pci device resides on
0757      * port1 or port2.  Note:  We don't have enough plumbing set up
0758      * here to use pci_read_config_xxx() so use raw_pci_read().
0759      */
0760 
0761     seg = tioce_common->ce_pcibus.bs_persist_segment;
0762     bus = tioce_common->ce_pcibus.bs_persist_busnum;
0763 
0764     raw_pci_read(seg, bus, PCI_DEVFN(2, 0), PCI_SECONDARY_BUS, 1,&tmp);
0765     tioce_kern->ce_port1_secondary = (u8) tmp;
0766 
0767     /*
0768      * Set PMU pagesize to the largest size available, and zero out
0769      * the ATEs.
0770      */
0771 
0772     tioce_mmr = (struct tioce __iomem *)tioce_common->ce_pcibus.bs_base;
0773     tioce_mmr_clri(tioce_kern, &tioce_mmr->ce_ure_page_map,
0774                CE_URE_PAGESIZE_MASK);
0775     tioce_mmr_seti(tioce_kern, &tioce_mmr->ce_ure_page_map,
0776                CE_URE_256K_PAGESIZE);
0777     ps = tioce_kern->ce_ate3240_pagesize = KB(256);
0778 
0779     for (i = 0; i < TIOCE_NUM_M40_ATES; i++) {
0780         tioce_kern->ce_ate40_shadow[i] = 0;
0781         tioce_mmr_storei(tioce_kern, &tioce_mmr->ce_ure_ate40[i], 0);
0782     }
0783 
0784     for (i = 0; i < TIOCE_NUM_M3240_ATES; i++) {
0785         tioce_kern->ce_ate3240_shadow[i] = 0;
0786         tioce_mmr_storei(tioce_kern, &tioce_mmr->ce_ure_ate3240[i], 0);
0787     }
0788 
0789     /*
0790      * Reserve ATEs corresponding to reserved address ranges.  These
0791      * include:
0792      *
0793      *  Memory space covered by each PPB mem base/limit register
0794      *  Memory space covered by each PPB prefetch base/limit register
0795      *
0796      * These bus ranges are for pio (downstream) traffic only, and so
0797      * cannot be used for DMA.
0798      */
0799 
0800     for (dev = 1; dev <= 2; dev++) {
0801         u64 base, limit;
0802 
0803         /* mem base/limit */
0804 
0805         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0806                   PCI_MEMORY_BASE, 2, &tmp);
0807         base = (u64)tmp << 16;
0808 
0809         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0810                   PCI_MEMORY_LIMIT, 2, &tmp);
0811         limit = (u64)tmp << 16;
0812         limit |= 0xfffffUL;
0813 
0814         if (base < limit)
0815             tioce_reserve_m32(tioce_kern, base, limit);
0816 
0817         /*
0818          * prefetch mem base/limit.  The tioce ppb's have 64-bit
0819          * decoders, so read the upper portions w/o checking the
0820          * attributes.
0821          */
0822 
0823         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0824                   PCI_PREF_MEMORY_BASE, 2, &tmp);
0825         base = ((u64)tmp & PCI_PREF_RANGE_MASK) << 16;
0826 
0827         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0828                   PCI_PREF_BASE_UPPER32, 4, &tmp);
0829         base |= (u64)tmp << 32;
0830 
0831         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0832                   PCI_PREF_MEMORY_LIMIT, 2, &tmp);
0833 
0834         limit = ((u64)tmp & PCI_PREF_RANGE_MASK) << 16;
0835         limit |= 0xfffffUL;
0836 
0837         raw_pci_read(seg, bus, PCI_DEVFN(dev, 0),
0838                   PCI_PREF_LIMIT_UPPER32, 4, &tmp);
0839         limit |= (u64)tmp << 32;
0840 
0841         if ((base < limit) && TIOCE_M32_ADDR(base))
0842             tioce_reserve_m32(tioce_kern, base, limit);
0843     }
0844 
0845     return tioce_kern;
0846 }
0847 
0848 /**
0849  * tioce_force_interrupt - implement altix force_interrupt() backend for CE
0850  * @sn_irq_info: sn asic irq that we need an interrupt generated for
0851  *
0852  * Given an sn_irq_info struct, set the proper bit in ce_adm_force_int to
0853  * force a secondary interrupt to be generated.  This is to work around an
0854  * asic issue where there is a small window of opportunity for a legacy device
0855  * interrupt to be lost.
0856  */
0857 static void
0858 tioce_force_interrupt(struct sn_irq_info *sn_irq_info)
0859 {
0860     struct pcidev_info *pcidev_info;
0861     struct tioce_common *ce_common;
0862     struct tioce_kernel *ce_kern;
0863     struct tioce __iomem *ce_mmr;
0864     u64 force_int_val;
0865 
0866     if (!sn_irq_info->irq_bridge)
0867         return;
0868 
0869     if (sn_irq_info->irq_bridge_type != PCIIO_ASIC_TYPE_TIOCE)
0870         return;
0871 
0872     pcidev_info = (struct pcidev_info *)sn_irq_info->irq_pciioinfo;
0873     if (!pcidev_info)
0874         return;
0875 
0876     ce_common = (struct tioce_common *)pcidev_info->pdi_pcibus_info;
0877     ce_mmr = (struct tioce __iomem *)ce_common->ce_pcibus.bs_base;
0878     ce_kern = (struct tioce_kernel *)ce_common->ce_kernel_private;
0879 
0880     /*
0881      * TIOCE Rev A workaround (PV 945826), force an interrupt by writing
0882      * the TIO_INTx register directly (1/26/2006)
0883      */
0884     if (ce_common->ce_rev == TIOCE_REV_A) {
0885         u64 int_bit_mask = (1ULL << sn_irq_info->irq_int_bit);
0886         u64 status;
0887 
0888         tioce_mmr_load(ce_kern, &ce_mmr->ce_adm_int_status, &status);
0889         if (status & int_bit_mask) {
0890             u64 force_irq = (1 << 8) | sn_irq_info->irq_irq;
0891             u64 ctalk = sn_irq_info->irq_xtalkaddr;
0892             u64 nasid, offset;
0893 
0894             nasid = (ctalk & CTALK_NASID_MASK) >> CTALK_NASID_SHFT;
0895             offset = (ctalk & CTALK_NODE_OFFSET);
0896             HUB_S(TIO_IOSPACE_ADDR(nasid, offset), force_irq);
0897         }
0898 
0899         return;
0900     }
0901 
0902     /*
0903      * irq_int_bit is originally set up by prom, and holds the interrupt
0904      * bit shift (not mask) as defined by the bit definitions in the
0905      * ce_adm_int mmr.  These shifts are not the same for the
0906      * ce_adm_force_int register, so do an explicit mapping here to make
0907      * things clearer.
0908      */
0909 
0910     switch (sn_irq_info->irq_int_bit) {
0911     case CE_ADM_INT_PCIE_PORT1_DEV_A_SHFT:
0912         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT1_DEV_A_SHFT;
0913         break;
0914     case CE_ADM_INT_PCIE_PORT1_DEV_B_SHFT:
0915         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT1_DEV_B_SHFT;
0916         break;
0917     case CE_ADM_INT_PCIE_PORT1_DEV_C_SHFT:
0918         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT1_DEV_C_SHFT;
0919         break;
0920     case CE_ADM_INT_PCIE_PORT1_DEV_D_SHFT:
0921         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT1_DEV_D_SHFT;
0922         break;
0923     case CE_ADM_INT_PCIE_PORT2_DEV_A_SHFT:
0924         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT2_DEV_A_SHFT;
0925         break;
0926     case CE_ADM_INT_PCIE_PORT2_DEV_B_SHFT:
0927         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT2_DEV_B_SHFT;
0928         break;
0929     case CE_ADM_INT_PCIE_PORT2_DEV_C_SHFT:
0930         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT2_DEV_C_SHFT;
0931         break;
0932     case CE_ADM_INT_PCIE_PORT2_DEV_D_SHFT:
0933         force_int_val = 1UL << CE_ADM_FORCE_INT_PCIE_PORT2_DEV_D_SHFT;
0934         break;
0935     default:
0936         return;
0937     }
0938     tioce_mmr_storei(ce_kern, &ce_mmr->ce_adm_force_int, force_int_val);
0939 }
0940 
0941 /**
0942  * tioce_target_interrupt - implement set_irq_affinity for tioce resident
0943  * functions.  Note:  only applies to line interrupts, not MSI's.
0944  *
0945  * @sn_irq_info: SN IRQ context
0946  *
0947  * Given an sn_irq_info, set the associated CE device's interrupt destination
0948  * register.  Since the interrupt destination registers are on a per-ce-slot
0949  * basis, this will retarget line interrupts for all functions downstream of
0950  * the slot.
0951  */
0952 static void
0953 tioce_target_interrupt(struct sn_irq_info *sn_irq_info)
0954 {
0955     struct pcidev_info *pcidev_info;
0956     struct tioce_common *ce_common;
0957     struct tioce_kernel *ce_kern;
0958     struct tioce __iomem *ce_mmr;
0959     int bit;
0960     u64 vector;
0961 
0962     pcidev_info = (struct pcidev_info *)sn_irq_info->irq_pciioinfo;
0963     if (!pcidev_info)
0964         return;
0965 
0966     ce_common = (struct tioce_common *)pcidev_info->pdi_pcibus_info;
0967     ce_mmr = (struct tioce __iomem *)ce_common->ce_pcibus.bs_base;
0968     ce_kern = (struct tioce_kernel *)ce_common->ce_kernel_private;
0969 
0970     bit = sn_irq_info->irq_int_bit;
0971 
0972     tioce_mmr_seti(ce_kern, &ce_mmr->ce_adm_int_mask, (1UL << bit));
0973     vector = (u64)sn_irq_info->irq_irq << INTR_VECTOR_SHFT;
0974     vector |= sn_irq_info->irq_xtalkaddr;
0975     tioce_mmr_storei(ce_kern, &ce_mmr->ce_adm_int_dest[bit], vector);
0976     tioce_mmr_clri(ce_kern, &ce_mmr->ce_adm_int_mask, (1UL << bit));
0977 
0978     tioce_force_interrupt(sn_irq_info);
0979 }
0980 
0981 /**
0982  * tioce_bus_fixup - perform final PCI fixup for a TIO CE bus
0983  * @prom_bussoft: Common prom/kernel struct representing the bus
0984  *
0985  * Replicates the tioce_common pointed to by @prom_bussoft in kernel
0986  * space.  Allocates and initializes a kernel-only area for a given CE,
0987  * and sets up an irq for handling CE error interrupts.
0988  *
0989  * On successful setup, returns the kernel version of tioce_common back to
0990  * the caller.
0991  */
0992 static void *
0993 tioce_bus_fixup(struct pcibus_bussoft *prom_bussoft, struct pci_controller *controller)
0994 {
0995     struct tioce_common *tioce_common;
0996     struct tioce_kernel *tioce_kern;
0997     struct tioce __iomem *tioce_mmr;
0998 
0999     /*
1000      * Allocate kernel bus soft and copy from prom.
1001      */
1002 
1003     tioce_common = kzalloc(sizeof(struct tioce_common), GFP_KERNEL);
1004     if (!tioce_common)
1005         return NULL;
1006 
1007     memcpy(tioce_common, prom_bussoft, sizeof(struct tioce_common));
1008     tioce_common->ce_pcibus.bs_base = (unsigned long)
1009         ioremap(REGION_OFFSET(tioce_common->ce_pcibus.bs_base),
1010             sizeof(struct tioce_common));
1011 
1012     tioce_kern = tioce_kern_init(tioce_common);
1013     if (tioce_kern == NULL) {
1014         kfree(tioce_common);
1015         return NULL;
1016     }
1017 
1018     /*
1019      * Clear out any transient errors before registering the error
1020      * interrupt handler.
1021      */
1022 
1023     tioce_mmr = (struct tioce __iomem *)tioce_common->ce_pcibus.bs_base;
1024     tioce_mmr_seti(tioce_kern, &tioce_mmr->ce_adm_int_status_alias, ~0ULL);
1025     tioce_mmr_seti(tioce_kern, &tioce_mmr->ce_adm_error_summary_alias,
1026                ~0ULL);
1027     tioce_mmr_seti(tioce_kern, &tioce_mmr->ce_dre_comp_err_addr, 0ULL);
1028 
1029     if (request_irq(SGI_PCIASIC_ERROR,
1030             tioce_error_intr_handler,
1031             IRQF_SHARED, "TIOCE error", (void *)tioce_common))
1032         printk(KERN_WARNING
1033                "%s:  Unable to get irq %d.  "
1034                "Error interrupts won't be routed for "
1035                "TIOCE bus %04x:%02x\n",
1036                __func__, SGI_PCIASIC_ERROR,
1037                tioce_common->ce_pcibus.bs_persist_segment,
1038                tioce_common->ce_pcibus.bs_persist_busnum);
1039 
1040     irq_set_handler(SGI_PCIASIC_ERROR, handle_level_irq);
1041     sn_set_err_irq_affinity(SGI_PCIASIC_ERROR);
1042     return tioce_common;
1043 }
1044 
1045 static struct sn_pcibus_provider tioce_pci_interfaces = {
1046     .dma_map = tioce_dma,
1047     .dma_map_consistent = tioce_dma_consistent,
1048     .dma_unmap = tioce_dma_unmap,
1049     .bus_fixup = tioce_bus_fixup,
1050     .force_interrupt = tioce_force_interrupt,
1051     .target_interrupt = tioce_target_interrupt
1052 };
1053 
1054 /**
1055  * tioce_init_provider - init SN PCI provider ops for TIO CE
1056  */
1057 int
1058 tioce_init_provider(void)
1059 {
1060     sn_pci_provider[PCIIO_ASIC_TYPE_TIOCE] = &tioce_pci_interfaces;
1061     return 0;
1062 }