char/agp/amd-k7-agp.c

0001 /*
0002  * AMD K7 AGPGART routines.
0003  */
0004
0005 #include <linux/module.h>
0006 #include <linux/pci.h>
0007 #include <linux/init.h>
0008 #include <linux/agp_backend.h>
0009 #include <linux/page-flags.h>
0010 #include <linux/mm.h>
0011 #include <linux/slab.h>
0012 #include <asm/set_memory.h>
0013 #include "agp.h"
0014
0015 #define AMD_MMBASE_BAR  1
0016 #define AMD_APSIZE  0xac
0017 #define AMD_MODECNTL    0xb0
0018 #define AMD_MODECNTL2   0xb2
0019 #define AMD_GARTENABLE  0x02    /* In mmio region (16-bit register) */
0020 #define AMD_ATTBASE 0x04    /* In mmio region (32-bit register) */
0021 #define AMD_TLBFLUSH    0x0c    /* In mmio region (32-bit register) */
0022 #define AMD_CACHEENTRY  0x10    /* In mmio region (32-bit register) */
0023
0024 static const struct pci_device_id agp_amdk7_pci_table[];
0025
0026 struct amd_page_map {
0027     unsigned long *real;
0028     unsigned long __iomem *remapped;
0029 };
0030
0031 static struct _amd_irongate_private {
0032     volatile u8 __iomem *registers;
0033     struct amd_page_map **gatt_pages;
0034     int num_tables;
0035 } amd_irongate_private;
0036
0037 static int amd_create_page_map(struct amd_page_map *page_map)
0038 {
0039     int i;
0040
0041     page_map->real = (unsigned long *) __get_free_page(GFP_KERNEL);
0042     if (page_map->real == NULL)
0043         return -ENOMEM;
0044
0045     set_memory_uc((unsigned long)page_map->real, 1);
0046     page_map->remapped = page_map->real;
0047
0048     for (i = 0; i < PAGE_SIZE / sizeof(unsigned long); i++) {
0049         writel(agp_bridge->scratch_page, page_map->remapped+i);
0050         readl(page_map->remapped+i);    /* PCI Posting. */
0051     }
0052
0053     return 0;
0054 }
0055
0056 static void amd_free_page_map(struct amd_page_map *page_map)
0057 {
0058     set_memory_wb((unsigned long)page_map->real, 1);
0059     free_page((unsigned long) page_map->real);
0060 }
0061
0062 static void amd_free_gatt_pages(void)
0063 {
0064     int i;
0065     struct amd_page_map **tables;
0066     struct amd_page_map *entry;
0067
0068     tables = amd_irongate_private.gatt_pages;
0069     for (i = 0; i < amd_irongate_private.num_tables; i++) {
0070         entry = tables[i];
0071         if (entry != NULL) {
0072             if (entry->real != NULL)
0073                 amd_free_page_map(entry);
0074             kfree(entry);
0075         }
0076     }
0077     kfree(tables);
0078     amd_irongate_private.gatt_pages = NULL;
0079 }
0080
0081 static int amd_create_gatt_pages(int nr_tables)
0082 {
0083     struct amd_page_map **tables;
0084     struct amd_page_map *entry;
0085     int retval = 0;
0086     int i;
0087
0088     tables = kcalloc(nr_tables + 1, sizeof(struct amd_page_map *),
0089              GFP_KERNEL);
0090     if (tables == NULL)
0091         return -ENOMEM;
0092
0093     for (i = 0; i < nr_tables; i++) {
0094         entry = kzalloc(sizeof(struct amd_page_map), GFP_KERNEL);
0095         tables[i] = entry;
0096         if (entry == NULL) {
0097             retval = -ENOMEM;
0098             break;
0099         }
0100         retval = amd_create_page_map(entry);
0101         if (retval != 0)
0102             break;
0103     }
0104     amd_irongate_private.num_tables = i;
0105     amd_irongate_private.gatt_pages = tables;
0106
0107     if (retval != 0)
0108         amd_free_gatt_pages();
0109
0110     return retval;
0111 }
0112
0113 /* Since we don't need contiguous memory we just try
0114  * to get the gatt table once
0115  */
0116
0117 #define GET_PAGE_DIR_OFF(addr) (addr >> 22)
0118 #define GET_PAGE_DIR_IDX(addr) (GET_PAGE_DIR_OFF(addr) - \
0119     GET_PAGE_DIR_OFF(agp_bridge->gart_bus_addr))
0120 #define GET_GATT_OFF(addr) ((addr & 0x003ff000) >> 12)
0121 #define GET_GATT(addr) (amd_irongate_private.gatt_pages[\
0122     GET_PAGE_DIR_IDX(addr)]->remapped)
0123
0124 static int amd_create_gatt_table(struct agp_bridge_data *bridge)
0125 {
0126     struct aper_size_info_lvl2 *value;
0127     struct amd_page_map page_dir;
0128     unsigned long __iomem *cur_gatt;
0129     unsigned long addr;
0130     int retval;
0131     int i;
0132
0133     value = A_SIZE_LVL2(agp_bridge->current_size);
0134     retval = amd_create_page_map(&page_dir);
0135     if (retval != 0)
0136         return retval;
0137
0138     retval = amd_create_gatt_pages(value->num_entries / 1024);
0139     if (retval != 0) {
0140         amd_free_page_map(&page_dir);
0141         return retval;
0142     }
0143
0144     agp_bridge->gatt_table_real = (u32 *)page_dir.real;
0145     agp_bridge->gatt_table = (u32 __iomem *)page_dir.remapped;
0146     agp_bridge->gatt_bus_addr = virt_to_phys(page_dir.real);
0147
0148     /* Get the address for the gart region.
0149      * This is a bus address even on the alpha, b/c its
0150      * used to program the agp master not the cpu
0151      */
0152
0153     addr = pci_bus_address(agp_bridge->dev, AGP_APERTURE_BAR);
0154     agp_bridge->gart_bus_addr = addr;
0155
0156     /* Calculate the agp offset */
0157     for (i = 0; i < value->num_entries / 1024; i++, addr += 0x00400000) {
0158         writel(virt_to_phys(amd_irongate_private.gatt_pages[i]->real) | 1,
0159             page_dir.remapped+GET_PAGE_DIR_OFF(addr));
0160         readl(page_dir.remapped+GET_PAGE_DIR_OFF(addr));    /* PCI Posting. */
0161     }
0162
0163     for (i = 0; i < value->num_entries; i++) {
0164         addr = (i * PAGE_SIZE) + agp_bridge->gart_bus_addr;
0165         cur_gatt = GET_GATT(addr);
0166         writel(agp_bridge->scratch_page, cur_gatt+GET_GATT_OFF(addr));
0167         readl(cur_gatt+GET_GATT_OFF(addr)); /* PCI Posting. */
0168     }
0169
0170     return 0;
0171 }
0172
0173 static int amd_free_gatt_table(struct agp_bridge_data *bridge)
0174 {
0175     struct amd_page_map page_dir;
0176
0177     page_dir.real = (unsigned long *)agp_bridge->gatt_table_real;
0178     page_dir.remapped = (unsigned long __iomem *)agp_bridge->gatt_table;
0179
0180     amd_free_gatt_pages();
0181     amd_free_page_map(&page_dir);
0182     return 0;
0183 }
0184
0185 static int amd_irongate_fetch_size(void)
0186 {
0187     int i;
0188     u32 temp;
0189     struct aper_size_info_lvl2 *values;
0190
0191     pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
0192     temp = (temp & 0x0000000e);
0193     values = A_SIZE_LVL2(agp_bridge->driver->aperture_sizes);
0194     for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) {
0195         if (temp == values[i].size_value) {
0196             agp_bridge->previous_size =
0197                 agp_bridge->current_size = (void *) (values + i);
0198
0199             agp_bridge->aperture_size_idx = i;
0200             return values[i].size;
0201         }
0202     }
0203
0204     return 0;
0205 }
0206
0207 static int amd_irongate_configure(void)
0208 {
0209     struct aper_size_info_lvl2 *current_size;
0210     phys_addr_t reg;
0211     u32 temp;
0212     u16 enable_reg;
0213
0214     current_size = A_SIZE_LVL2(agp_bridge->current_size);
0215
0216     if (!amd_irongate_private.registers) {
0217         /* Get the memory mapped registers */
0218         reg = pci_resource_start(agp_bridge->dev, AMD_MMBASE_BAR);
0219         amd_irongate_private.registers = (volatile u8 __iomem *) ioremap(reg, 4096);
0220         if (!amd_irongate_private.registers)
0221             return -ENOMEM;
0222     }
0223
0224     /* Write out the address of the gatt table */
0225     writel(agp_bridge->gatt_bus_addr, amd_irongate_private.registers+AMD_ATTBASE);
0226     readl(amd_irongate_private.registers+AMD_ATTBASE);  /* PCI Posting. */
0227
0228     /* Write the Sync register */
0229     pci_write_config_byte(agp_bridge->dev, AMD_MODECNTL, 0x80);
0230
0231     /* Set indexing mode */
0232     pci_write_config_byte(agp_bridge->dev, AMD_MODECNTL2, 0x00);
0233
0234     /* Write the enable register */
0235     enable_reg = readw(amd_irongate_private.registers+AMD_GARTENABLE);
0236     enable_reg = (enable_reg | 0x0004);
0237     writew(enable_reg, amd_irongate_private.registers+AMD_GARTENABLE);
0238     readw(amd_irongate_private.registers+AMD_GARTENABLE);   /* PCI Posting. */
0239
0240     /* Write out the size register */
0241     pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
0242     temp = (((temp & ~(0x0000000e)) | current_size->size_value) | 1);
0243     pci_write_config_dword(agp_bridge->dev, AMD_APSIZE, temp);
0244
0245     /* Flush the tlb */
0246     writel(1, amd_irongate_private.registers+AMD_TLBFLUSH);
0247     readl(amd_irongate_private.registers+AMD_TLBFLUSH); /* PCI Posting.*/
0248     return 0;
0249 }
0250
0251 static void amd_irongate_cleanup(void)
0252 {
0253     struct aper_size_info_lvl2 *previous_size;
0254     u32 temp;
0255     u16 enable_reg;
0256
0257     previous_size = A_SIZE_LVL2(agp_bridge->previous_size);
0258
0259     enable_reg = readw(amd_irongate_private.registers+AMD_GARTENABLE);
0260     enable_reg = (enable_reg & ~(0x0004));
0261     writew(enable_reg, amd_irongate_private.registers+AMD_GARTENABLE);
0262     readw(amd_irongate_private.registers+AMD_GARTENABLE);   /* PCI Posting. */
0263
0264     /* Write back the previous size and disable gart translation */
0265     pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
0266     temp = ((temp & ~(0x0000000f)) | previous_size->size_value);
0267     pci_write_config_dword(agp_bridge->dev, AMD_APSIZE, temp);
0268     iounmap((void __iomem *) amd_irongate_private.registers);
0269 }
0270
0271 /*
0272  * This routine could be implemented by taking the addresses
0273  * written to the GATT, and flushing them individually.  However
0274  * currently it just flushes the whole table.  Which is probably
0275  * more efficient, since agp_memory blocks can be a large number of
0276  * entries.
0277  */
0278
0279 static void amd_irongate_tlbflush(struct agp_memory *temp)
0280 {
0281     writel(1, amd_irongate_private.registers+AMD_TLBFLUSH);
0282     readl(amd_irongate_private.registers+AMD_TLBFLUSH); /* PCI Posting. */
0283 }
0284
0285 static int amd_insert_memory(struct agp_memory *mem, off_t pg_start, int type)
0286 {
0287     int i, j, num_entries;
0288     unsigned long __iomem *cur_gatt;
0289     unsigned long addr;
0290
0291     num_entries = A_SIZE_LVL2(agp_bridge->current_size)->num_entries;
0292
0293     if (type != mem->type ||
0294         agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type))
0295         return -EINVAL;
0296
0297     if ((pg_start + mem->page_count) > num_entries)
0298         return -EINVAL;
0299
0300     j = pg_start;
0301     while (j < (pg_start + mem->page_count)) {
0302         addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
0303         cur_gatt = GET_GATT(addr);
0304         if (!PGE_EMPTY(agp_bridge, readl(cur_gatt+GET_GATT_OFF(addr))))
0305             return -EBUSY;
0306         j++;
0307     }
0308
0309     if (!mem->is_flushed) {
0310         global_cache_flush();
0311         mem->is_flushed = true;
0312     }
0313
0314     for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
0315         addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
0316         cur_gatt = GET_GATT(addr);
0317         writel(agp_generic_mask_memory(agp_bridge,
0318                            page_to_phys(mem->pages[i]),
0319                            mem->type),
0320                cur_gatt+GET_GATT_OFF(addr));
0321         readl(cur_gatt+GET_GATT_OFF(addr)); /* PCI Posting. */
0322     }
0323     amd_irongate_tlbflush(mem);
0324     return 0;
0325 }
0326
0327 static int amd_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
0328 {
0329     int i;
0330     unsigned long __iomem *cur_gatt;
0331     unsigned long addr;
0332
0333     if (type != mem->type ||
0334         agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type))
0335         return -EINVAL;
0336
0337     for (i = pg_start; i < (mem->page_count + pg_start); i++) {
0338         addr = (i * PAGE_SIZE) + agp_bridge->gart_bus_addr;
0339         cur_gatt = GET_GATT(addr);
0340         writel(agp_bridge->scratch_page, cur_gatt+GET_GATT_OFF(addr));
0341         readl(cur_gatt+GET_GATT_OFF(addr)); /* PCI Posting. */
0342     }
0343
0344     amd_irongate_tlbflush(mem);
0345     return 0;
0346 }
0347
0348 static const struct aper_size_info_lvl2 amd_irongate_sizes[7] =
0349 {
0350     {2048, 524288, 0x0000000c},
0351     {1024, 262144, 0x0000000a},
0352     {512, 131072, 0x00000008},
0353     {256, 65536, 0x00000006},
0354     {128, 32768, 0x00000004},
0355     {64, 16384, 0x00000002},
0356     {32, 8192, 0x00000000}
0357 };
0358
0359 static const struct gatt_mask amd_irongate_masks[] =
0360 {
0361     {.mask = 1, .type = 0}
0362 };
0363
0364 static const struct agp_bridge_driver amd_irongate_driver = {
0365     .owner          = THIS_MODULE,
0366     .aperture_sizes     = amd_irongate_sizes,
0367     .size_type      = LVL2_APER_SIZE,
0368     .num_aperture_sizes = 7,
0369     .needs_scratch_page = true,
0370     .configure      = amd_irongate_configure,
0371     .fetch_size     = amd_irongate_fetch_size,
0372     .cleanup        = amd_irongate_cleanup,
0373     .tlb_flush      = amd_irongate_tlbflush,
0374     .mask_memory        = agp_generic_mask_memory,
0375     .masks          = amd_irongate_masks,
0376     .agp_enable     = agp_generic_enable,
0377     .cache_flush        = global_cache_flush,
0378     .create_gatt_table  = amd_create_gatt_table,
0379     .free_gatt_table    = amd_free_gatt_table,
0380     .insert_memory      = amd_insert_memory,
0381     .remove_memory      = amd_remove_memory,
0382     .alloc_by_type      = agp_generic_alloc_by_type,
0383     .free_by_type       = agp_generic_free_by_type,
0384     .agp_alloc_page     = agp_generic_alloc_page,
0385     .agp_alloc_pages    = agp_generic_alloc_pages,
0386     .agp_destroy_page   = agp_generic_destroy_page,
0387     .agp_destroy_pages  = agp_generic_destroy_pages,
0388     .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
0389 };
0390
0391 static struct agp_device_ids amd_agp_device_ids[] =
0392 {
0393     {
0394         .device_id  = PCI_DEVICE_ID_AMD_FE_GATE_7006,
0395         .chipset_name   = "Irongate",
0396     },
0397     {
0398         .device_id  = PCI_DEVICE_ID_AMD_FE_GATE_700E,
0399         .chipset_name   = "761",
0400     },
0401     {
0402         .device_id  = PCI_DEVICE_ID_AMD_FE_GATE_700C,
0403         .chipset_name   = "760MP",
0404     },
0405     { }, /* dummy final entry, always present */
0406 };
0407
0408 static int agp_amdk7_probe(struct pci_dev *pdev,
0409                const struct pci_device_id *ent)
0410 {
0411     struct agp_bridge_data *bridge;
0412     u8 cap_ptr;
0413     int j;
0414
0415     cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP);
0416     if (!cap_ptr)
0417         return -ENODEV;
0418
0419     j = ent - agp_amdk7_pci_table;
0420     dev_info(&pdev->dev, "AMD %s chipset\n",
0421          amd_agp_device_ids[j].chipset_name);
0422
0423     bridge = agp_alloc_bridge();
0424     if (!bridge)
0425         return -ENOMEM;
0426
0427     bridge->driver = &amd_irongate_driver;
0428     bridge->dev_private_data = &amd_irongate_private;
0429     bridge->dev = pdev;
0430     bridge->capndx = cap_ptr;
0431
0432     /* 751 Errata (22564_B-1.PDF)
0433        erratum 20: strobe glitch with Nvidia NV10 GeForce cards.
0434        system controller may experience noise due to strong drive strengths
0435      */
0436     if (agp_bridge->dev->device == PCI_DEVICE_ID_AMD_FE_GATE_7006) {
0437         struct pci_dev *gfxcard=NULL;
0438
0439         cap_ptr = 0;
0440         while (!cap_ptr) {
0441             gfxcard = pci_get_class(PCI_CLASS_DISPLAY_VGA<<8, gfxcard);
0442             if (!gfxcard) {
0443                 dev_info(&pdev->dev, "no AGP VGA controller\n");
0444                 return -ENODEV;
0445             }
0446             cap_ptr = pci_find_capability(gfxcard, PCI_CAP_ID_AGP);
0447         }
0448
0449         /* With so many variants of NVidia cards, it's simpler just
0450            to blacklist them all, and then whitelist them as needed
0451            (if necessary at all). */
0452         if (gfxcard->vendor == PCI_VENDOR_ID_NVIDIA) {
0453             agp_bridge->flags |= AGP_ERRATA_1X;
0454             dev_info(&pdev->dev, "AMD 751 chipset with NVidia GeForce; forcing 1X due to errata\n");
0455         }
0456         pci_dev_put(gfxcard);
0457     }
0458
0459     /* 761 Errata (23613_F.pdf)
0460      * Revisions B0/B1 were a disaster.
0461      * erratum 44: SYSCLK/AGPCLK skew causes 2X failures -- Force mode to 1X
0462      * erratum 45: Timing problem prevents fast writes -- Disable fast write.
0463      * erratum 46: Setup violation on AGP SBA pins - Disable side band addressing.
0464      * With this lot disabled, we should prevent lockups. */
0465     if (agp_bridge->dev->device == PCI_DEVICE_ID_AMD_FE_GATE_700E) {
0466         if (pdev->revision == 0x10 || pdev->revision == 0x11) {
0467             agp_bridge->flags = AGP_ERRATA_FASTWRITES;
0468             agp_bridge->flags |= AGP_ERRATA_SBA;
0469             agp_bridge->flags |= AGP_ERRATA_1X;
0470             dev_info(&pdev->dev, "AMD 761 chipset with errata; disabling AGP fast writes & SBA and forcing to 1X\n");
0471         }
0472     }
0473
0474     /* Fill in the mode register */
0475     pci_read_config_dword(pdev,
0476             bridge->capndx+PCI_AGP_STATUS,
0477             &bridge->mode);
0478
0479     pci_set_drvdata(pdev, bridge);
0480     return agp_add_bridge(bridge);
0481 }
0482
0483 static void agp_amdk7_remove(struct pci_dev *pdev)
0484 {
0485     struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
0486
0487     agp_remove_bridge(bridge);
0488     agp_put_bridge(bridge);
0489 }
0490
0491 #ifdef CONFIG_PM
0492
0493 static int agp_amdk7_suspend(struct pci_dev *pdev, pm_message_t state)
0494 {
0495     pci_save_state(pdev);
0496     pci_set_power_state(pdev, pci_choose_state(pdev, state));
0497
0498     return 0;
0499 }
0500
0501 static int agp_amdk7_resume(struct pci_dev *pdev)
0502 {
0503     pci_set_power_state(pdev, PCI_D0);
0504     pci_restore_state(pdev);
0505
0506     return amd_irongate_driver.configure();
0507 }
0508
0509 #endif /* CONFIG_PM */
0510
0511 /* must be the same order as name table above */
0512 static const struct pci_device_id agp_amdk7_pci_table[] = {
0513     {
0514     .class      = (PCI_CLASS_BRIDGE_HOST << 8),
0515     .class_mask = ~0,
0516     .vendor     = PCI_VENDOR_ID_AMD,
0517     .device     = PCI_DEVICE_ID_AMD_FE_GATE_7006,
0518     .subvendor  = PCI_ANY_ID,
0519     .subdevice  = PCI_ANY_ID,
0520     },
0521     {
0522     .class      = (PCI_CLASS_BRIDGE_HOST << 8),
0523     .class_mask = ~0,
0524     .vendor     = PCI_VENDOR_ID_AMD,
0525     .device     = PCI_DEVICE_ID_AMD_FE_GATE_700E,
0526     .subvendor  = PCI_ANY_ID,
0527     .subdevice  = PCI_ANY_ID,
0528     },
0529     {
0530     .class      = (PCI_CLASS_BRIDGE_HOST << 8),
0531     .class_mask = ~0,
0532     .vendor     = PCI_VENDOR_ID_AMD,
0533     .device     = PCI_DEVICE_ID_AMD_FE_GATE_700C,
0534     .subvendor  = PCI_ANY_ID,
0535     .subdevice  = PCI_ANY_ID,
0536     },
0537     { }
0538 };
0539
0540 MODULE_DEVICE_TABLE(pci, agp_amdk7_pci_table);
0541
0542 static struct pci_driver agp_amdk7_pci_driver = {
0543     .name       = "agpgart-amdk7",
0544     .id_table   = agp_amdk7_pci_table,
0545     .probe      = agp_amdk7_probe,
0546     .remove     = agp_amdk7_remove,
0547 #ifdef CONFIG_PM
0548     .suspend    = agp_amdk7_suspend,
0549     .resume     = agp_amdk7_resume,
0550 #endif
0551 };
0552
0553 static int __init agp_amdk7_init(void)
0554 {
0555     if (agp_off)
0556         return -EINVAL;
0557     return pci_register_driver(&agp_amdk7_pci_driver);
0558 }
0559
0560 static void __exit agp_amdk7_cleanup(void)
0561 {
0562     pci_unregister_driver(&agp_amdk7_pci_driver);
0563 }
0564
0565 module_init(agp_amdk7_init);
0566 module_exit(agp_amdk7_cleanup);
0567
0568 MODULE_LICENSE("GPL and additional rights");