char/agp/nvidia-agp.c

0001 /*
0002  * Nvidia AGPGART routines.
0003  * Based upon a 2.4 agpgart diff by the folks from NVIDIA, and hacked up
0004  * to work in 2.5 by Dave Jones.
0005  */
0006
0007 #include <linux/module.h>
0008 #include <linux/pci.h>
0009 #include <linux/init.h>
0010 #include <linux/agp_backend.h>
0011 #include <linux/page-flags.h>
0012 #include <linux/mm.h>
0013 #include <linux/jiffies.h>
0014 #include "agp.h"
0015
0016 /* NVIDIA registers */
0017 #define NVIDIA_0_APSIZE     0x80
0018 #define NVIDIA_1_WBC        0xf0
0019 #define NVIDIA_2_GARTCTRL   0xd0
0020 #define NVIDIA_2_APBASE     0xd8
0021 #define NVIDIA_2_APLIMIT    0xdc
0022 #define NVIDIA_2_ATTBASE(i) (0xe0 + (i) * 4)
0023 #define NVIDIA_3_APBASE     0x50
0024 #define NVIDIA_3_APLIMIT    0x54
0025
0026
0027 static struct _nvidia_private {
0028     struct pci_dev *dev_1;
0029     struct pci_dev *dev_2;
0030     struct pci_dev *dev_3;
0031     volatile u32 __iomem *aperture;
0032     int num_active_entries;
0033     off_t pg_offset;
0034     u32 wbc_mask;
0035 } nvidia_private;
0036
0037
0038 static int nvidia_fetch_size(void)
0039 {
0040     int i;
0041     u8 size_value;
0042     struct aper_size_info_8 *values;
0043
0044     pci_read_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, &size_value);
0045     size_value &= 0x0f;
0046     values = A_SIZE_8(agp_bridge->driver->aperture_sizes);
0047
0048     for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) {
0049         if (size_value == values[i].size_value) {
0050             agp_bridge->previous_size =
0051                 agp_bridge->current_size = (void *) (values + i);
0052             agp_bridge->aperture_size_idx = i;
0053             return values[i].size;
0054         }
0055     }
0056
0057     return 0;
0058 }
0059
0060 #define SYSCFG          0xC0010010
0061 #define IORR_BASE0      0xC0010016
0062 #define IORR_MASK0      0xC0010017
0063 #define AMD_K7_NUM_IORR 2
0064
0065 static int nvidia_init_iorr(u32 base, u32 size)
0066 {
0067     u32 base_hi, base_lo;
0068     u32 mask_hi, mask_lo;
0069     u32 sys_hi, sys_lo;
0070     u32 iorr_addr, free_iorr_addr;
0071
0072     /* Find the iorr that is already used for the base */
0073     /* If not found, determine the uppermost available iorr */
0074     free_iorr_addr = AMD_K7_NUM_IORR;
0075     for (iorr_addr = 0; iorr_addr < AMD_K7_NUM_IORR; iorr_addr++) {
0076         rdmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi);
0077         rdmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi);
0078
0079         if ((base_lo & 0xfffff000) == (base & 0xfffff000))
0080             break;
0081
0082         if ((mask_lo & 0x00000800) == 0)
0083             free_iorr_addr = iorr_addr;
0084     }
0085
0086     if (iorr_addr >= AMD_K7_NUM_IORR) {
0087         iorr_addr = free_iorr_addr;
0088         if (iorr_addr >= AMD_K7_NUM_IORR)
0089             return -EINVAL;
0090     }
0091     base_hi = 0x0;
0092     base_lo = (base & ~0xfff) | 0x18;
0093     mask_hi = 0xf;
0094     mask_lo = ((~(size - 1)) & 0xfffff000) | 0x800;
0095     wrmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi);
0096     wrmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi);
0097
0098     rdmsr(SYSCFG, sys_lo, sys_hi);
0099     sys_lo |= 0x00100000;
0100     wrmsr(SYSCFG, sys_lo, sys_hi);
0101
0102     return 0;
0103 }
0104
0105 static int nvidia_configure(void)
0106 {
0107     int i, rc, num_dirs;
0108     u32 apbase, aplimit;
0109     phys_addr_t apbase_phys;
0110     struct aper_size_info_8 *current_size;
0111     u32 temp;
0112
0113     current_size = A_SIZE_8(agp_bridge->current_size);
0114
0115     /* aperture size */
0116     pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE,
0117         current_size->size_value);
0118
0119     /* address to map to */
0120     apbase = pci_bus_address(agp_bridge->dev, AGP_APERTURE_BAR);
0121     agp_bridge->gart_bus_addr = apbase;
0122     aplimit = apbase + (current_size->size * 1024 * 1024) - 1;
0123     pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APBASE, apbase);
0124     pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APLIMIT, aplimit);
0125     pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APBASE, apbase);
0126     pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APLIMIT, aplimit);
0127     if (0 != (rc = nvidia_init_iorr(apbase, current_size->size * 1024 * 1024)))
0128         return rc;
0129
0130     /* directory size is 64k */
0131     num_dirs = current_size->size / 64;
0132     nvidia_private.num_active_entries = current_size->num_entries;
0133     nvidia_private.pg_offset = 0;
0134     if (num_dirs == 0) {
0135         num_dirs = 1;
0136         nvidia_private.num_active_entries /= (64 / current_size->size);
0137         nvidia_private.pg_offset = (apbase & (64 * 1024 * 1024 - 1) &
0138             ~(current_size->size * 1024 * 1024 - 1)) / PAGE_SIZE;
0139     }
0140
0141     /* attbase */
0142     for (i = 0; i < 8; i++) {
0143         pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_ATTBASE(i),
0144             (agp_bridge->gatt_bus_addr + (i % num_dirs) * 64 * 1024) | 1);
0145     }
0146
0147     /* gtlb control */
0148     pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp);
0149     pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp | 0x11);
0150
0151     /* gart control */
0152     pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp);
0153     pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp | 0x100);
0154
0155     /* map aperture */
0156     apbase_phys = pci_resource_start(agp_bridge->dev, AGP_APERTURE_BAR);
0157     nvidia_private.aperture =
0158         (volatile u32 __iomem *) ioremap(apbase_phys, 33 * PAGE_SIZE);
0159
0160     if (!nvidia_private.aperture)
0161         return -ENOMEM;
0162
0163     return 0;
0164 }
0165
0166 static void nvidia_cleanup(void)
0167 {
0168     struct aper_size_info_8 *previous_size;
0169     u32 temp;
0170
0171     /* gart control */
0172     pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp);
0173     pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp & ~(0x100));
0174
0175     /* gtlb control */
0176     pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp);
0177     pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp & ~(0x11));
0178
0179     /* unmap aperture */
0180     iounmap((void __iomem *) nvidia_private.aperture);
0181
0182     /* restore previous aperture size */
0183     previous_size = A_SIZE_8(agp_bridge->previous_size);
0184     pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE,
0185         previous_size->size_value);
0186
0187     /* restore iorr for previous aperture size */
0188     nvidia_init_iorr(agp_bridge->gart_bus_addr,
0189         previous_size->size * 1024 * 1024);
0190 }
0191
0192
0193 /*
0194  * Note we can't use the generic routines, even though they are 99% the same.
0195  * Aperture sizes <64M still requires a full 64k GART directory, but
0196  * only use the portion of the TLB entries that correspond to the apertures
0197  * alignment inside the surrounding 64M block.
0198  */
0199 extern int agp_memory_reserved;
0200
0201 static int nvidia_insert_memory(struct agp_memory *mem, off_t pg_start, int type)
0202 {
0203     int i, j;
0204     int mask_type;
0205
0206     mask_type = agp_generic_type_to_mask_type(mem->bridge, type);
0207     if (mask_type != 0 || type != mem->type)
0208         return -EINVAL;
0209
0210     if (mem->page_count == 0)
0211         return 0;
0212
0213     if ((pg_start + mem->page_count) >
0214         (nvidia_private.num_active_entries - agp_memory_reserved/PAGE_SIZE))
0215         return -EINVAL;
0216
0217     for (j = pg_start; j < (pg_start + mem->page_count); j++) {
0218         if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j)))
0219             return -EBUSY;
0220     }
0221
0222     if (!mem->is_flushed) {
0223         global_cache_flush();
0224         mem->is_flushed = true;
0225     }
0226     for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
0227         writel(agp_bridge->driver->mask_memory(agp_bridge,
0228                    page_to_phys(mem->pages[i]), mask_type),
0229             agp_bridge->gatt_table+nvidia_private.pg_offset+j);
0230     }
0231
0232     /* PCI Posting. */
0233     readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j - 1);
0234
0235     agp_bridge->driver->tlb_flush(mem);
0236     return 0;
0237 }
0238
0239
0240 static int nvidia_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
0241 {
0242     int i;
0243
0244     int mask_type;
0245
0246     mask_type = agp_generic_type_to_mask_type(mem->bridge, type);
0247     if (mask_type != 0 || type != mem->type)
0248         return -EINVAL;
0249
0250     if (mem->page_count == 0)
0251         return 0;
0252
0253     for (i = pg_start; i < (mem->page_count + pg_start); i++)
0254         writel(agp_bridge->scratch_page, agp_bridge->gatt_table+nvidia_private.pg_offset+i);
0255
0256     agp_bridge->driver->tlb_flush(mem);
0257     return 0;
0258 }
0259
0260
0261 static void nvidia_tlbflush(struct agp_memory *mem)
0262 {
0263     unsigned long end;
0264     u32 wbc_reg;
0265     u32 __maybe_unused temp;
0266     int i;
0267
0268     /* flush chipset */
0269     if (nvidia_private.wbc_mask) {
0270         pci_read_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, &wbc_reg);
0271         wbc_reg |= nvidia_private.wbc_mask;
0272         pci_write_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, wbc_reg);
0273
0274         end = jiffies + 3*HZ;
0275         do {
0276             pci_read_config_dword(nvidia_private.dev_1,
0277                     NVIDIA_1_WBC, &wbc_reg);
0278             if (time_before_eq(end, jiffies)) {
0279                 printk(KERN_ERR PFX
0280                     "TLB flush took more than 3 seconds.\n");
0281             }
0282         } while (wbc_reg & nvidia_private.wbc_mask);
0283     }
0284
0285     /* flush TLB entries */
0286     for (i = 0; i < 32 + 1; i++)
0287         temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32)));
0288     for (i = 0; i < 32 + 1; i++)
0289         temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32)));
0290 }
0291
0292
0293 static const struct aper_size_info_8 nvidia_generic_sizes[5] =
0294 {
0295     {512, 131072, 7, 0},
0296     {256, 65536, 6, 8},
0297     {128, 32768, 5, 12},
0298     {64, 16384, 4, 14},
0299     /* The 32M mode still requires a 64k gatt */
0300     {32, 16384, 4, 15}
0301 };
0302
0303
0304 static const struct gatt_mask nvidia_generic_masks[] =
0305 {
0306     { .mask = 1, .type = 0}
0307 };
0308
0309
0310 static const struct agp_bridge_driver nvidia_driver = {
0311     .owner          = THIS_MODULE,
0312     .aperture_sizes     = nvidia_generic_sizes,
0313     .size_type      = U8_APER_SIZE,
0314     .num_aperture_sizes = 5,
0315     .needs_scratch_page = true,
0316     .configure      = nvidia_configure,
0317     .fetch_size     = nvidia_fetch_size,
0318     .cleanup        = nvidia_cleanup,
0319     .tlb_flush      = nvidia_tlbflush,
0320     .mask_memory        = agp_generic_mask_memory,
0321     .masks          = nvidia_generic_masks,
0322     .agp_enable     = agp_generic_enable,
0323     .cache_flush        = global_cache_flush,
0324     .create_gatt_table  = agp_generic_create_gatt_table,
0325     .free_gatt_table    = agp_generic_free_gatt_table,
0326     .insert_memory      = nvidia_insert_memory,
0327     .remove_memory      = nvidia_remove_memory,
0328     .alloc_by_type      = agp_generic_alloc_by_type,
0329     .free_by_type       = agp_generic_free_by_type,
0330     .agp_alloc_page     = agp_generic_alloc_page,
0331     .agp_alloc_pages    = agp_generic_alloc_pages,
0332     .agp_destroy_page   = agp_generic_destroy_page,
0333     .agp_destroy_pages  = agp_generic_destroy_pages,
0334     .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
0335 };
0336
0337 static int agp_nvidia_probe(struct pci_dev *pdev,
0338                 const struct pci_device_id *ent)
0339 {
0340     struct agp_bridge_data *bridge;
0341     u8 cap_ptr;
0342
0343     nvidia_private.dev_1 =
0344         pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
0345                         (unsigned int)pdev->bus->number,
0346                         PCI_DEVFN(0, 1));
0347     nvidia_private.dev_2 =
0348         pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
0349                         (unsigned int)pdev->bus->number,
0350                         PCI_DEVFN(0, 2));
0351     nvidia_private.dev_3 =
0352         pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
0353                         (unsigned int)pdev->bus->number,
0354                         PCI_DEVFN(30, 0));
0355
0356     if (!nvidia_private.dev_1 || !nvidia_private.dev_2 || !nvidia_private.dev_3) {
0357         printk(KERN_INFO PFX "Detected an NVIDIA nForce/nForce2 "
0358             "chipset, but could not find the secondary devices.\n");
0359         return -ENODEV;
0360     }
0361
0362     cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP);
0363     if (!cap_ptr)
0364         return -ENODEV;
0365
0366     switch (pdev->device) {
0367     case PCI_DEVICE_ID_NVIDIA_NFORCE:
0368         printk(KERN_INFO PFX "Detected NVIDIA nForce chipset\n");
0369         nvidia_private.wbc_mask = 0x00010000;
0370         break;
0371     case PCI_DEVICE_ID_NVIDIA_NFORCE2:
0372         printk(KERN_INFO PFX "Detected NVIDIA nForce2 chipset\n");
0373         nvidia_private.wbc_mask = 0x80000000;
0374         break;
0375     default:
0376         printk(KERN_ERR PFX "Unsupported NVIDIA chipset (device id: %04x)\n",
0377                 pdev->device);
0378         return -ENODEV;
0379     }
0380
0381     bridge = agp_alloc_bridge();
0382     if (!bridge)
0383         return -ENOMEM;
0384
0385     bridge->driver = &nvidia_driver;
0386     bridge->dev_private_data = &nvidia_private;
0387     bridge->dev = pdev;
0388     bridge->capndx = cap_ptr;
0389
0390     /* Fill in the mode register */
0391     pci_read_config_dword(pdev,
0392             bridge->capndx+PCI_AGP_STATUS,
0393             &bridge->mode);
0394
0395     pci_set_drvdata(pdev, bridge);
0396     return agp_add_bridge(bridge);
0397 }
0398
0399 static void agp_nvidia_remove(struct pci_dev *pdev)
0400 {
0401     struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
0402
0403     agp_remove_bridge(bridge);
0404     agp_put_bridge(bridge);
0405 }
0406
0407 #ifdef CONFIG_PM
0408 static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state)
0409 {
0410     pci_save_state(pdev);
0411     pci_set_power_state(pdev, PCI_D3hot);
0412
0413     return 0;
0414 }
0415
0416 static int agp_nvidia_resume(struct pci_dev *pdev)
0417 {
0418     /* set power state 0 and restore PCI space */
0419     pci_set_power_state(pdev, PCI_D0);
0420     pci_restore_state(pdev);
0421
0422     /* reconfigure AGP hardware again */
0423     nvidia_configure();
0424
0425     return 0;
0426 }
0427 #endif
0428
0429
0430 static const struct pci_device_id agp_nvidia_pci_table[] = {
0431     {
0432     .class      = (PCI_CLASS_BRIDGE_HOST << 8),
0433     .class_mask = ~0,
0434     .vendor     = PCI_VENDOR_ID_NVIDIA,
0435     .device     = PCI_DEVICE_ID_NVIDIA_NFORCE,
0436     .subvendor  = PCI_ANY_ID,
0437     .subdevice  = PCI_ANY_ID,
0438     },
0439     {
0440     .class      = (PCI_CLASS_BRIDGE_HOST << 8),
0441     .class_mask = ~0,
0442     .vendor     = PCI_VENDOR_ID_NVIDIA,
0443     .device     = PCI_DEVICE_ID_NVIDIA_NFORCE2,
0444     .subvendor  = PCI_ANY_ID,
0445     .subdevice  = PCI_ANY_ID,
0446     },
0447     { }
0448 };
0449
0450 MODULE_DEVICE_TABLE(pci, agp_nvidia_pci_table);
0451
0452 static struct pci_driver agp_nvidia_pci_driver = {
0453     .name       = "agpgart-nvidia",
0454     .id_table   = agp_nvidia_pci_table,
0455     .probe      = agp_nvidia_probe,
0456     .remove     = agp_nvidia_remove,
0457 #ifdef CONFIG_PM
0458     .suspend    = agp_nvidia_suspend,
0459     .resume     = agp_nvidia_resume,
0460 #endif
0461 };
0462
0463 static int __init agp_nvidia_init(void)
0464 {
0465     if (agp_off)
0466         return -EINVAL;
0467     return pci_register_driver(&agp_nvidia_pci_driver);
0468 }
0469
0470 static void __exit agp_nvidia_cleanup(void)
0471 {
0472     pci_unregister_driver(&agp_nvidia_pci_driver);
0473     pci_dev_put(nvidia_private.dev_1);
0474     pci_dev_put(nvidia_private.dev_2);
0475     pci_dev_put(nvidia_private.dev_3);
0476 }
0477
0478 module_init(agp_nvidia_init);
0479 module_exit(agp_nvidia_cleanup);
0480
0481 MODULE_LICENSE("GPL and additional rights");
0482 MODULE_AUTHOR("NVIDIA Corporation");
0483