Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * This file is provided under a dual BSD/GPLv2 license.  When using or
0003  *   redistributing this file, you may do so under either license.
0004  *
0005  *   GPL LICENSE SUMMARY
0006  *
0007  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
0008  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
0009  *
0010  *   This program is free software; you can redistribute it and/or modify
0011  *   it under the terms of version 2 of the GNU General Public License as
0012  *   published by the Free Software Foundation.
0013  *
0014  *   BSD LICENSE
0015  *
0016  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
0017  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
0018  *
0019  *   Redistribution and use in source and binary forms, with or without
0020  *   modification, are permitted provided that the following conditions
0021  *   are met:
0022  *
0023  *     * Redistributions of source code must retain the above copyright
0024  *       notice, this list of conditions and the following disclaimer.
0025  *     * Redistributions in binary form must reproduce the above copy
0026  *       notice, this list of conditions and the following disclaimer in
0027  *       the documentation and/or other materials provided with the
0028  *       distribution.
0029  *     * Neither the name of AMD Corporation nor the names of its
0030  *       contributors may be used to endorse or promote products derived
0031  *       from this software without specific prior written permission.
0032  *
0033  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
0034  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
0035  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
0036  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
0037  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
0038  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
0039  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
0040  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
0041  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
0042  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
0043  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0044  *
0045  * AMD PCIe NTB Linux driver
0046  *
0047  * Contact Information:
0048  * Xiangliang Yu <Xiangliang.Yu@amd.com>
0049  */
0050 
0051 #include <linux/debugfs.h>
0052 #include <linux/delay.h>
0053 #include <linux/init.h>
0054 #include <linux/interrupt.h>
0055 #include <linux/module.h>
0056 #include <linux/acpi.h>
0057 #include <linux/pci.h>
0058 #include <linux/random.h>
0059 #include <linux/slab.h>
0060 #include <linux/ntb.h>
0061 
0062 #include "ntb_hw_amd.h"
0063 
0064 #define NTB_NAME    "ntb_hw_amd"
0065 #define NTB_DESC    "AMD(R) PCI-E Non-Transparent Bridge Driver"
0066 #define NTB_VER     "1.0"
0067 
0068 MODULE_DESCRIPTION(NTB_DESC);
0069 MODULE_VERSION(NTB_VER);
0070 MODULE_LICENSE("Dual BSD/GPL");
0071 MODULE_AUTHOR("AMD Inc.");
0072 
0073 static const struct file_operations amd_ntb_debugfs_info;
0074 static struct dentry *debugfs_dir;
0075 
0076 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
0077 {
0078     if (idx < 0 || idx > ndev->mw_count)
0079         return -EINVAL;
0080 
0081     return ndev->dev_data->mw_idx << idx;
0082 }
0083 
0084 static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
0085 {
0086     if (pidx != NTB_DEF_PEER_IDX)
0087         return -EINVAL;
0088 
0089     return ntb_ndev(ntb)->mw_count;
0090 }
0091 
0092 static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
0093                 resource_size_t *addr_align,
0094                 resource_size_t *size_align,
0095                 resource_size_t *size_max)
0096 {
0097     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0098     int bar;
0099 
0100     if (pidx != NTB_DEF_PEER_IDX)
0101         return -EINVAL;
0102 
0103     bar = ndev_mw_to_bar(ndev, idx);
0104     if (bar < 0)
0105         return bar;
0106 
0107     if (addr_align)
0108         *addr_align = SZ_4K;
0109 
0110     if (size_align)
0111         *size_align = 1;
0112 
0113     if (size_max)
0114         *size_max = pci_resource_len(ndev->ntb.pdev, bar);
0115 
0116     return 0;
0117 }
0118 
0119 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
0120                 dma_addr_t addr, resource_size_t size)
0121 {
0122     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0123     unsigned long xlat_reg, limit_reg = 0;
0124     resource_size_t mw_size;
0125     void __iomem *mmio, *peer_mmio;
0126     u64 base_addr, limit, reg_val;
0127     int bar;
0128 
0129     if (pidx != NTB_DEF_PEER_IDX)
0130         return -EINVAL;
0131 
0132     bar = ndev_mw_to_bar(ndev, idx);
0133     if (bar < 0)
0134         return bar;
0135 
0136     mw_size = pci_resource_len(ntb->pdev, bar);
0137 
0138     /* make sure the range fits in the usable mw size */
0139     if (size > mw_size)
0140         return -EINVAL;
0141 
0142     mmio = ndev->self_mmio;
0143     peer_mmio = ndev->peer_mmio;
0144 
0145     base_addr = pci_resource_start(ntb->pdev, bar);
0146 
0147     if (bar != 1) {
0148         xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
0149         limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
0150 
0151         /* Set the limit if supported */
0152         limit = size;
0153 
0154         /* set and verify setting the translation address */
0155         write64(addr, peer_mmio + xlat_reg);
0156         reg_val = read64(peer_mmio + xlat_reg);
0157         if (reg_val != addr) {
0158             write64(0, peer_mmio + xlat_reg);
0159             return -EIO;
0160         }
0161 
0162         /* set and verify setting the limit */
0163         write64(limit, peer_mmio + limit_reg);
0164         reg_val = read64(peer_mmio + limit_reg);
0165         if (reg_val != limit) {
0166             write64(base_addr, mmio + limit_reg);
0167             write64(0, peer_mmio + xlat_reg);
0168             return -EIO;
0169         }
0170     } else {
0171         xlat_reg = AMD_BAR1XLAT_OFFSET;
0172         limit_reg = AMD_BAR1LMT_OFFSET;
0173 
0174         /* Set the limit if supported */
0175         limit = size;
0176 
0177         /* set and verify setting the translation address */
0178         write64(addr, peer_mmio + xlat_reg);
0179         reg_val = read64(peer_mmio + xlat_reg);
0180         if (reg_val != addr) {
0181             write64(0, peer_mmio + xlat_reg);
0182             return -EIO;
0183         }
0184 
0185         /* set and verify setting the limit */
0186         writel(limit, peer_mmio + limit_reg);
0187         reg_val = readl(peer_mmio + limit_reg);
0188         if (reg_val != limit) {
0189             writel(base_addr, mmio + limit_reg);
0190             writel(0, peer_mmio + xlat_reg);
0191             return -EIO;
0192         }
0193     }
0194 
0195     return 0;
0196 }
0197 
0198 static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev)
0199 {
0200     struct pci_dev *pdev = NULL;
0201     struct pci_dev *pci_swds = NULL;
0202     struct pci_dev *pci_swus = NULL;
0203     u32 stat;
0204     int rc;
0205 
0206     if (ndev->ntb.topo == NTB_TOPO_SEC) {
0207         /* Locate the pointer to Downstream Switch for this device */
0208         pci_swds = pci_upstream_bridge(ndev->ntb.pdev);
0209         if (pci_swds) {
0210             /*
0211              * Locate the pointer to Upstream Switch for
0212              * the Downstream Switch.
0213              */
0214             pci_swus = pci_upstream_bridge(pci_swds);
0215             if (pci_swus) {
0216                 rc = pcie_capability_read_dword(pci_swus,
0217                                 PCI_EXP_LNKCTL,
0218                                 &stat);
0219                 if (rc)
0220                     return 0;
0221             } else {
0222                 return 0;
0223             }
0224         } else {
0225             return 0;
0226         }
0227     } else if (ndev->ntb.topo == NTB_TOPO_PRI) {
0228         /*
0229          * For NTB primary, we simply read the Link Status and control
0230          * register of the NTB device itself.
0231          */
0232         pdev = ndev->ntb.pdev;
0233         rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat);
0234         if (rc)
0235             return 0;
0236     } else {
0237         /* Catch all for everything else */
0238         return 0;
0239     }
0240 
0241     ndev->lnk_sta = stat;
0242 
0243     return 1;
0244 }
0245 
0246 static int amd_link_is_up(struct amd_ntb_dev *ndev)
0247 {
0248     int ret;
0249 
0250     /*
0251      * We consider the link to be up under two conditions:
0252      *
0253      *   - When a link-up event is received. This is indicated by
0254      *     AMD_LINK_UP_EVENT set in peer_sta.
0255      *   - When driver on both sides of the link have been loaded.
0256      *     This is indicated by bit 1 being set in the peer
0257      *     SIDEINFO register.
0258      *
0259      * This function should return 1 when the latter of the above
0260      * two conditions is true.
0261      *
0262      * Now consider the sequence of events - Link-Up event occurs,
0263      * then the peer side driver loads. In this case, we would have
0264      * received LINK_UP event and bit 1 of peer SIDEINFO is also
0265      * set. What happens now if the link goes down? Bit 1 of
0266      * peer SIDEINFO remains set, but LINK_DOWN bit is set in
0267      * peer_sta. So we should return 0 from this function. Not only
0268      * that, we clear bit 1 of peer SIDEINFO to 0, since the peer
0269      * side driver did not even get a chance to clear it before
0270      * the link went down. This can be the case of surprise link
0271      * removal.
0272      *
0273      * LINK_UP event will always occur before the peer side driver
0274      * gets loaded the very first time. So there can be a case when
0275      * the LINK_UP event has occurred, but the peer side driver hasn't
0276      * yet loaded. We return 0 in that case.
0277      *
0278      * There is also a special case when the primary side driver is
0279      * unloaded and then loaded again. Since there is no change in
0280      * the status of NTB secondary in this case, there is no Link-Up
0281      * or Link-Down notification received. We recognize this condition
0282      * with peer_sta being set to 0.
0283      *
0284      * If bit 1 of peer SIDEINFO register is not set, then we
0285      * simply return 0 irrespective of the link up or down status
0286      * set in peer_sta.
0287      */
0288     ret = amd_poll_link(ndev);
0289     if (ret) {
0290         /*
0291          * We need to check the below only for NTB primary. For NTB
0292          * secondary, simply checking the result of PSIDE_INFO
0293          * register will suffice.
0294          */
0295         if (ndev->ntb.topo == NTB_TOPO_PRI) {
0296             if ((ndev->peer_sta & AMD_LINK_UP_EVENT) ||
0297                 (ndev->peer_sta == 0))
0298                 return ret;
0299             else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) {
0300                 /* Clear peer sideinfo register */
0301                 amd_clear_side_info_reg(ndev, true);
0302 
0303                 return 0;
0304             }
0305         } else { /* NTB_TOPO_SEC */
0306             return ret;
0307         }
0308     }
0309 
0310     return 0;
0311 }
0312 
0313 static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
0314                   enum ntb_speed *speed,
0315                   enum ntb_width *width)
0316 {
0317     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0318     int ret = 0;
0319 
0320     if (amd_link_is_up(ndev)) {
0321         if (speed)
0322             *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
0323         if (width)
0324             *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
0325 
0326         dev_dbg(&ntb->pdev->dev, "link is up.\n");
0327 
0328         ret = 1;
0329     } else {
0330         if (speed)
0331             *speed = NTB_SPEED_NONE;
0332         if (width)
0333             *width = NTB_WIDTH_NONE;
0334 
0335         dev_dbg(&ntb->pdev->dev, "link is down.\n");
0336     }
0337 
0338     return ret;
0339 }
0340 
0341 static int amd_ntb_link_enable(struct ntb_dev *ntb,
0342                    enum ntb_speed max_speed,
0343                    enum ntb_width max_width)
0344 {
0345     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0346     void __iomem *mmio = ndev->self_mmio;
0347 
0348     /* Enable event interrupt */
0349     ndev->int_mask &= ~AMD_EVENT_INTMASK;
0350     writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
0351 
0352     if (ndev->ntb.topo == NTB_TOPO_SEC)
0353         return -EINVAL;
0354     dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
0355 
0356     return 0;
0357 }
0358 
0359 static int amd_ntb_link_disable(struct ntb_dev *ntb)
0360 {
0361     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0362     void __iomem *mmio = ndev->self_mmio;
0363 
0364     /* Disable event interrupt */
0365     ndev->int_mask |= AMD_EVENT_INTMASK;
0366     writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
0367 
0368     if (ndev->ntb.topo == NTB_TOPO_SEC)
0369         return -EINVAL;
0370     dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
0371 
0372     return 0;
0373 }
0374 
0375 static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
0376 {
0377     /* The same as for inbound MWs */
0378     return ntb_ndev(ntb)->mw_count;
0379 }
0380 
0381 static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
0382                     phys_addr_t *base, resource_size_t *size)
0383 {
0384     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0385     int bar;
0386 
0387     bar = ndev_mw_to_bar(ndev, idx);
0388     if (bar < 0)
0389         return bar;
0390 
0391     if (base)
0392         *base = pci_resource_start(ndev->ntb.pdev, bar);
0393 
0394     if (size)
0395         *size = pci_resource_len(ndev->ntb.pdev, bar);
0396 
0397     return 0;
0398 }
0399 
0400 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
0401 {
0402     return ntb_ndev(ntb)->db_valid_mask;
0403 }
0404 
0405 static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
0406 {
0407     return ntb_ndev(ntb)->db_count;
0408 }
0409 
0410 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
0411 {
0412     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0413 
0414     if (db_vector < 0 || db_vector > ndev->db_count)
0415         return 0;
0416 
0417     return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
0418 }
0419 
0420 static u64 amd_ntb_db_read(struct ntb_dev *ntb)
0421 {
0422     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0423     void __iomem *mmio = ndev->self_mmio;
0424 
0425     return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
0426 }
0427 
0428 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
0429 {
0430     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0431     void __iomem *mmio = ndev->self_mmio;
0432 
0433     writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
0434 
0435     return 0;
0436 }
0437 
0438 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
0439 {
0440     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0441     void __iomem *mmio = ndev->self_mmio;
0442     unsigned long flags;
0443 
0444     if (db_bits & ~ndev->db_valid_mask)
0445         return -EINVAL;
0446 
0447     spin_lock_irqsave(&ndev->db_mask_lock, flags);
0448     ndev->db_mask |= db_bits;
0449     writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
0450     spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
0451 
0452     return 0;
0453 }
0454 
0455 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
0456 {
0457     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0458     void __iomem *mmio = ndev->self_mmio;
0459     unsigned long flags;
0460 
0461     if (db_bits & ~ndev->db_valid_mask)
0462         return -EINVAL;
0463 
0464     spin_lock_irqsave(&ndev->db_mask_lock, flags);
0465     ndev->db_mask &= ~db_bits;
0466     writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
0467     spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
0468 
0469     return 0;
0470 }
0471 
0472 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
0473 {
0474     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0475     void __iomem *mmio = ndev->self_mmio;
0476 
0477     writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
0478 
0479     return 0;
0480 }
0481 
0482 static int amd_ntb_spad_count(struct ntb_dev *ntb)
0483 {
0484     return ntb_ndev(ntb)->spad_count;
0485 }
0486 
0487 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
0488 {
0489     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0490     void __iomem *mmio = ndev->self_mmio;
0491     u32 offset;
0492 
0493     if (idx < 0 || idx >= ndev->spad_count)
0494         return 0;
0495 
0496     offset = ndev->self_spad + (idx << 2);
0497     return readl(mmio + AMD_SPAD_OFFSET + offset);
0498 }
0499 
0500 static int amd_ntb_spad_write(struct ntb_dev *ntb,
0501                   int idx, u32 val)
0502 {
0503     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0504     void __iomem *mmio = ndev->self_mmio;
0505     u32 offset;
0506 
0507     if (idx < 0 || idx >= ndev->spad_count)
0508         return -EINVAL;
0509 
0510     offset = ndev->self_spad + (idx << 2);
0511     writel(val, mmio + AMD_SPAD_OFFSET + offset);
0512 
0513     return 0;
0514 }
0515 
0516 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
0517 {
0518     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0519     void __iomem *mmio = ndev->self_mmio;
0520     u32 offset;
0521 
0522     if (sidx < 0 || sidx >= ndev->spad_count)
0523         return -EINVAL;
0524 
0525     offset = ndev->peer_spad + (sidx << 2);
0526     return readl(mmio + AMD_SPAD_OFFSET + offset);
0527 }
0528 
0529 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
0530                    int sidx, u32 val)
0531 {
0532     struct amd_ntb_dev *ndev = ntb_ndev(ntb);
0533     void __iomem *mmio = ndev->self_mmio;
0534     u32 offset;
0535 
0536     if (sidx < 0 || sidx >= ndev->spad_count)
0537         return -EINVAL;
0538 
0539     offset = ndev->peer_spad + (sidx << 2);
0540     writel(val, mmio + AMD_SPAD_OFFSET + offset);
0541 
0542     return 0;
0543 }
0544 
0545 static const struct ntb_dev_ops amd_ntb_ops = {
0546     .mw_count       = amd_ntb_mw_count,
0547     .mw_get_align       = amd_ntb_mw_get_align,
0548     .mw_set_trans       = amd_ntb_mw_set_trans,
0549     .peer_mw_count      = amd_ntb_peer_mw_count,
0550     .peer_mw_get_addr   = amd_ntb_peer_mw_get_addr,
0551     .link_is_up     = amd_ntb_link_is_up,
0552     .link_enable        = amd_ntb_link_enable,
0553     .link_disable       = amd_ntb_link_disable,
0554     .db_valid_mask      = amd_ntb_db_valid_mask,
0555     .db_vector_count    = amd_ntb_db_vector_count,
0556     .db_vector_mask     = amd_ntb_db_vector_mask,
0557     .db_read        = amd_ntb_db_read,
0558     .db_clear       = amd_ntb_db_clear,
0559     .db_set_mask        = amd_ntb_db_set_mask,
0560     .db_clear_mask      = amd_ntb_db_clear_mask,
0561     .peer_db_set        = amd_ntb_peer_db_set,
0562     .spad_count     = amd_ntb_spad_count,
0563     .spad_read      = amd_ntb_spad_read,
0564     .spad_write     = amd_ntb_spad_write,
0565     .peer_spad_read     = amd_ntb_peer_spad_read,
0566     .peer_spad_write    = amd_ntb_peer_spad_write,
0567 };
0568 
0569 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
0570 {
0571     void __iomem *mmio = ndev->self_mmio;
0572     int reg;
0573 
0574     reg = readl(mmio + AMD_SMUACK_OFFSET);
0575     reg |= bit;
0576     writel(reg, mmio + AMD_SMUACK_OFFSET);
0577 }
0578 
0579 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
0580 {
0581     void __iomem *mmio = ndev->self_mmio;
0582     struct device *dev = &ndev->ntb.pdev->dev;
0583     u32 status;
0584 
0585     status = readl(mmio + AMD_INTSTAT_OFFSET);
0586     if (!(status & AMD_EVENT_INTMASK))
0587         return;
0588 
0589     dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
0590 
0591     status &= AMD_EVENT_INTMASK;
0592     switch (status) {
0593     case AMD_PEER_FLUSH_EVENT:
0594         ndev->peer_sta |= AMD_PEER_FLUSH_EVENT;
0595         dev_info(dev, "Flush is done.\n");
0596         break;
0597     case AMD_PEER_RESET_EVENT:
0598     case AMD_LINK_DOWN_EVENT:
0599         ndev->peer_sta |= status;
0600         if (status == AMD_LINK_DOWN_EVENT)
0601             ndev->peer_sta &= ~AMD_LINK_UP_EVENT;
0602 
0603         amd_ack_smu(ndev, status);
0604 
0605         /* link down first */
0606         ntb_link_event(&ndev->ntb);
0607         /* polling peer status */
0608         schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
0609 
0610         break;
0611     case AMD_PEER_D3_EVENT:
0612     case AMD_PEER_PMETO_EVENT:
0613     case AMD_LINK_UP_EVENT:
0614         ndev->peer_sta |= status;
0615         if (status == AMD_LINK_UP_EVENT)
0616             ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT;
0617         else if (status == AMD_PEER_D3_EVENT)
0618             ndev->peer_sta &= ~AMD_PEER_D0_EVENT;
0619 
0620         amd_ack_smu(ndev, status);
0621 
0622         /* link down */
0623         ntb_link_event(&ndev->ntb);
0624 
0625         break;
0626     case AMD_PEER_D0_EVENT:
0627         mmio = ndev->peer_mmio;
0628         status = readl(mmio + AMD_PMESTAT_OFFSET);
0629         /* check if this is WAKEUP event */
0630         if (status & 0x1)
0631             dev_info(dev, "Wakeup is done.\n");
0632 
0633         ndev->peer_sta |= AMD_PEER_D0_EVENT;
0634         ndev->peer_sta &= ~AMD_PEER_D3_EVENT;
0635         amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
0636 
0637         /* start a timer to poll link status */
0638         schedule_delayed_work(&ndev->hb_timer,
0639                       AMD_LINK_HB_TIMEOUT);
0640         break;
0641     default:
0642         dev_info(dev, "event status = 0x%x.\n", status);
0643         break;
0644     }
0645 
0646     /* Clear the interrupt status */
0647     writel(status, mmio + AMD_INTSTAT_OFFSET);
0648 }
0649 
0650 static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec)
0651 {
0652     struct device *dev = &ndev->ntb.pdev->dev;
0653     u64 status;
0654 
0655     status = amd_ntb_db_read(&ndev->ntb);
0656 
0657     dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec);
0658 
0659     /*
0660      * Since we had reserved highest order bit of DB for signaling peer of
0661      * a special event, this is the only status bit we should be concerned
0662      * here now.
0663      */
0664     if (status & BIT(ndev->db_last_bit)) {
0665         ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit));
0666         /* send link down event notification */
0667         ntb_link_event(&ndev->ntb);
0668 
0669         /*
0670          * If we are here, that means the peer has signalled a special
0671          * event which notifies that the peer driver has been
0672          * un-loaded for some reason. Since there is a chance that the
0673          * peer will load its driver again sometime, we schedule link
0674          * polling routine.
0675          */
0676         schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
0677     }
0678 }
0679 
0680 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
0681 {
0682     dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
0683 
0684     if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
0685         amd_handle_event(ndev, vec);
0686 
0687     if (vec < AMD_DB_CNT) {
0688         amd_handle_db_event(ndev, vec);
0689         ntb_db_event(&ndev->ntb, vec);
0690     }
0691 
0692     return IRQ_HANDLED;
0693 }
0694 
0695 static irqreturn_t ndev_vec_isr(int irq, void *dev)
0696 {
0697     struct amd_ntb_vec *nvec = dev;
0698 
0699     return ndev_interrupt(nvec->ndev, nvec->num);
0700 }
0701 
0702 static irqreturn_t ndev_irq_isr(int irq, void *dev)
0703 {
0704     struct amd_ntb_dev *ndev = dev;
0705 
0706     return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
0707 }
0708 
0709 static int ndev_init_isr(struct amd_ntb_dev *ndev,
0710              int msix_min, int msix_max)
0711 {
0712     struct pci_dev *pdev;
0713     int rc, i, msix_count, node;
0714 
0715     pdev = ndev->ntb.pdev;
0716 
0717     node = dev_to_node(&pdev->dev);
0718 
0719     ndev->db_mask = ndev->db_valid_mask;
0720 
0721     /* Try to set up msix irq */
0722     ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
0723                  GFP_KERNEL, node);
0724     if (!ndev->vec)
0725         goto err_msix_vec_alloc;
0726 
0727     ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
0728                   GFP_KERNEL, node);
0729     if (!ndev->msix)
0730         goto err_msix_alloc;
0731 
0732     for (i = 0; i < msix_max; ++i)
0733         ndev->msix[i].entry = i;
0734 
0735     msix_count = pci_enable_msix_range(pdev, ndev->msix,
0736                        msix_min, msix_max);
0737     if (msix_count < 0)
0738         goto err_msix_enable;
0739 
0740     /* NOTE: Disable MSIX if msix count is less than 16 because of
0741      * hardware limitation.
0742      */
0743     if (msix_count < msix_min) {
0744         pci_disable_msix(pdev);
0745         goto err_msix_enable;
0746     }
0747 
0748     for (i = 0; i < msix_count; ++i) {
0749         ndev->vec[i].ndev = ndev;
0750         ndev->vec[i].num = i;
0751         rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
0752                  "ndev_vec_isr", &ndev->vec[i]);
0753         if (rc)
0754             goto err_msix_request;
0755     }
0756 
0757     dev_dbg(&pdev->dev, "Using msix interrupts\n");
0758     ndev->db_count = msix_min;
0759     ndev->msix_vec_count = msix_max;
0760     return 0;
0761 
0762 err_msix_request:
0763     while (i-- > 0)
0764         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
0765     pci_disable_msix(pdev);
0766 err_msix_enable:
0767     kfree(ndev->msix);
0768 err_msix_alloc:
0769     kfree(ndev->vec);
0770 err_msix_vec_alloc:
0771     ndev->msix = NULL;
0772     ndev->vec = NULL;
0773 
0774     /* Try to set up msi irq */
0775     rc = pci_enable_msi(pdev);
0776     if (rc)
0777         goto err_msi_enable;
0778 
0779     rc = request_irq(pdev->irq, ndev_irq_isr, 0,
0780              "ndev_irq_isr", ndev);
0781     if (rc)
0782         goto err_msi_request;
0783 
0784     dev_dbg(&pdev->dev, "Using msi interrupts\n");
0785     ndev->db_count = 1;
0786     ndev->msix_vec_count = 1;
0787     return 0;
0788 
0789 err_msi_request:
0790     pci_disable_msi(pdev);
0791 err_msi_enable:
0792 
0793     /* Try to set up intx irq */
0794     pci_intx(pdev, 1);
0795 
0796     rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
0797              "ndev_irq_isr", ndev);
0798     if (rc)
0799         goto err_intx_request;
0800 
0801     dev_dbg(&pdev->dev, "Using intx interrupts\n");
0802     ndev->db_count = 1;
0803     ndev->msix_vec_count = 1;
0804     return 0;
0805 
0806 err_intx_request:
0807     return rc;
0808 }
0809 
0810 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
0811 {
0812     struct pci_dev *pdev;
0813     void __iomem *mmio = ndev->self_mmio;
0814     int i;
0815 
0816     pdev = ndev->ntb.pdev;
0817 
0818     /* Mask all doorbell interrupts */
0819     ndev->db_mask = ndev->db_valid_mask;
0820     writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
0821 
0822     if (ndev->msix) {
0823         i = ndev->msix_vec_count;
0824         while (i--)
0825             free_irq(ndev->msix[i].vector, &ndev->vec[i]);
0826         pci_disable_msix(pdev);
0827         kfree(ndev->msix);
0828         kfree(ndev->vec);
0829     } else {
0830         free_irq(pdev->irq, ndev);
0831         if (pci_dev_msi_enabled(pdev))
0832             pci_disable_msi(pdev);
0833         else
0834             pci_intx(pdev, 0);
0835     }
0836 }
0837 
0838 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
0839                  size_t count, loff_t *offp)
0840 {
0841     struct amd_ntb_dev *ndev;
0842     void __iomem *mmio;
0843     char *buf;
0844     size_t buf_size;
0845     ssize_t ret, off;
0846     union { u64 v64; u32 v32; u16 v16; } u;
0847 
0848     ndev = filp->private_data;
0849     mmio = ndev->self_mmio;
0850 
0851     buf_size = min(count, 0x800ul);
0852 
0853     buf = kmalloc(buf_size, GFP_KERNEL);
0854     if (!buf)
0855         return -ENOMEM;
0856 
0857     off = 0;
0858 
0859     off += scnprintf(buf + off, buf_size - off,
0860              "NTB Device Information:\n");
0861 
0862     off += scnprintf(buf + off, buf_size - off,
0863              "Connection Topology -\t%s\n",
0864              ntb_topo_string(ndev->ntb.topo));
0865 
0866     off += scnprintf(buf + off, buf_size - off,
0867              "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
0868 
0869     if (!amd_link_is_up(ndev)) {
0870         off += scnprintf(buf + off, buf_size - off,
0871                  "Link Status -\t\tDown\n");
0872     } else {
0873         off += scnprintf(buf + off, buf_size - off,
0874                  "Link Status -\t\tUp\n");
0875         off += scnprintf(buf + off, buf_size - off,
0876                  "Link Speed -\t\tPCI-E Gen %u\n",
0877                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
0878         off += scnprintf(buf + off, buf_size - off,
0879                  "Link Width -\t\tx%u\n",
0880                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
0881     }
0882 
0883     off += scnprintf(buf + off, buf_size - off,
0884              "Memory Window Count -\t%u\n", ndev->mw_count);
0885     off += scnprintf(buf + off, buf_size - off,
0886              "Scratchpad Count -\t%u\n", ndev->spad_count);
0887     off += scnprintf(buf + off, buf_size - off,
0888              "Doorbell Count -\t%u\n", ndev->db_count);
0889     off += scnprintf(buf + off, buf_size - off,
0890              "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
0891 
0892     off += scnprintf(buf + off, buf_size - off,
0893              "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
0894 
0895     u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
0896     off += scnprintf(buf + off, buf_size - off,
0897              "Doorbell Mask -\t\t\t%#06x\n", u.v32);
0898 
0899     u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
0900     off += scnprintf(buf + off, buf_size - off,
0901              "Doorbell Bell -\t\t\t%#06x\n", u.v32);
0902 
0903     off += scnprintf(buf + off, buf_size - off,
0904              "\nNTB Incoming XLAT:\n");
0905 
0906     u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
0907     off += scnprintf(buf + off, buf_size - off,
0908              "XLAT1 -\t\t%#018llx\n", u.v64);
0909 
0910     u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
0911     off += scnprintf(buf + off, buf_size - off,
0912              "XLAT23 -\t\t%#018llx\n", u.v64);
0913 
0914     u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
0915     off += scnprintf(buf + off, buf_size - off,
0916              "XLAT45 -\t\t%#018llx\n", u.v64);
0917 
0918     u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
0919     off += scnprintf(buf + off, buf_size - off,
0920              "LMT1 -\t\t\t%#06x\n", u.v32);
0921 
0922     u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
0923     off += scnprintf(buf + off, buf_size - off,
0924              "LMT23 -\t\t\t%#018llx\n", u.v64);
0925 
0926     u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
0927     off += scnprintf(buf + off, buf_size - off,
0928              "LMT45 -\t\t\t%#018llx\n", u.v64);
0929 
0930     ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
0931     kfree(buf);
0932     return ret;
0933 }
0934 
0935 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
0936 {
0937     if (!debugfs_dir) {
0938         ndev->debugfs_dir = NULL;
0939         ndev->debugfs_info = NULL;
0940     } else {
0941         ndev->debugfs_dir =
0942             debugfs_create_dir(pci_name(ndev->ntb.pdev),
0943                        debugfs_dir);
0944         if (!ndev->debugfs_dir)
0945             ndev->debugfs_info = NULL;
0946         else
0947             ndev->debugfs_info =
0948                 debugfs_create_file("info", S_IRUSR,
0949                             ndev->debugfs_dir, ndev,
0950                             &amd_ntb_debugfs_info);
0951     }
0952 }
0953 
0954 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
0955 {
0956     debugfs_remove_recursive(ndev->debugfs_dir);
0957 }
0958 
0959 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
0960                     struct pci_dev *pdev)
0961 {
0962     ndev->ntb.pdev = pdev;
0963     ndev->ntb.topo = NTB_TOPO_NONE;
0964     ndev->ntb.ops = &amd_ntb_ops;
0965     ndev->int_mask = AMD_EVENT_INTMASK;
0966     spin_lock_init(&ndev->db_mask_lock);
0967 }
0968 
0969 static int amd_poll_link(struct amd_ntb_dev *ndev)
0970 {
0971     void __iomem *mmio = ndev->peer_mmio;
0972     u32 reg;
0973 
0974     reg = readl(mmio + AMD_SIDEINFO_OFFSET);
0975     reg &= AMD_SIDE_READY;
0976 
0977     dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
0978 
0979     ndev->cntl_sta = reg;
0980 
0981     amd_ntb_get_link_status(ndev);
0982 
0983     return ndev->cntl_sta;
0984 }
0985 
0986 static void amd_link_hb(struct work_struct *work)
0987 {
0988     struct amd_ntb_dev *ndev = hb_ndev(work);
0989 
0990     if (amd_poll_link(ndev))
0991         ntb_link_event(&ndev->ntb);
0992 
0993     if (!amd_link_is_up(ndev))
0994         schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
0995 }
0996 
0997 static int amd_init_isr(struct amd_ntb_dev *ndev)
0998 {
0999     return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
1000 }
1001 
1002 static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1003 {
1004     void __iomem *mmio = NULL;
1005     unsigned int reg;
1006 
1007     if (peer)
1008         mmio = ndev->peer_mmio;
1009     else
1010         mmio = ndev->self_mmio;
1011 
1012     reg = readl(mmio + AMD_SIDEINFO_OFFSET);
1013     if (!(reg & AMD_SIDE_READY)) {
1014         reg |= AMD_SIDE_READY;
1015         writel(reg, mmio + AMD_SIDEINFO_OFFSET);
1016     }
1017 }
1018 
1019 static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1020 {
1021     void __iomem *mmio = NULL;
1022     unsigned int reg;
1023 
1024     if (peer)
1025         mmio = ndev->peer_mmio;
1026     else
1027         mmio = ndev->self_mmio;
1028 
1029     reg = readl(mmio + AMD_SIDEINFO_OFFSET);
1030     if (reg & AMD_SIDE_READY) {
1031         reg &= ~AMD_SIDE_READY;
1032         writel(reg, mmio + AMD_SIDEINFO_OFFSET);
1033         readl(mmio + AMD_SIDEINFO_OFFSET);
1034     }
1035 }
1036 
1037 static void amd_init_side_info(struct amd_ntb_dev *ndev)
1038 {
1039     void __iomem *mmio = ndev->self_mmio;
1040     u32 ntb_ctl;
1041 
1042     amd_set_side_info_reg(ndev, false);
1043 
1044     ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
1045     ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
1046     writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
1047 }
1048 
1049 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
1050 {
1051     void __iomem *mmio = ndev->self_mmio;
1052     u32 ntb_ctl;
1053 
1054     amd_clear_side_info_reg(ndev, false);
1055 
1056     ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
1057     ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
1058     writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
1059 }
1060 
1061 static int amd_init_ntb(struct amd_ntb_dev *ndev)
1062 {
1063     void __iomem *mmio = ndev->self_mmio;
1064 
1065     ndev->mw_count = ndev->dev_data->mw_count;
1066     ndev->spad_count = AMD_SPADS_CNT;
1067     ndev->db_count = AMD_DB_CNT;
1068 
1069     switch (ndev->ntb.topo) {
1070     case NTB_TOPO_PRI:
1071     case NTB_TOPO_SEC:
1072         ndev->spad_count >>= 1;
1073         if (ndev->ntb.topo == NTB_TOPO_PRI) {
1074             ndev->self_spad = 0;
1075             ndev->peer_spad = 0x20;
1076         } else {
1077             ndev->self_spad = 0x20;
1078             ndev->peer_spad = 0;
1079         }
1080 
1081         INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
1082         schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
1083 
1084         break;
1085     default:
1086         dev_err(&ndev->ntb.pdev->dev,
1087             "AMD NTB does not support B2B mode.\n");
1088         return -EINVAL;
1089     }
1090 
1091     /* Mask event interrupts */
1092     writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
1093 
1094     return 0;
1095 }
1096 
1097 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
1098 {
1099     void __iomem *mmio = ndev->self_mmio;
1100     u32 info;
1101 
1102     info = readl(mmio + AMD_SIDEINFO_OFFSET);
1103     if (info & AMD_SIDE_MASK)
1104         return NTB_TOPO_SEC;
1105     else
1106         return NTB_TOPO_PRI;
1107 }
1108 
1109 static int amd_init_dev(struct amd_ntb_dev *ndev)
1110 {
1111     void __iomem *mmio = ndev->self_mmio;
1112     struct pci_dev *pdev;
1113     int rc = 0;
1114 
1115     pdev = ndev->ntb.pdev;
1116 
1117     ndev->ntb.topo = amd_get_topo(ndev);
1118     dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
1119         ntb_topo_string(ndev->ntb.topo));
1120 
1121     rc = amd_init_ntb(ndev);
1122     if (rc)
1123         return rc;
1124 
1125     rc = amd_init_isr(ndev);
1126     if (rc) {
1127         dev_err(&pdev->dev, "fail to init isr.\n");
1128         return rc;
1129     }
1130 
1131     ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1132     /*
1133      * We reserve the highest order bit of the DB register which will
1134      * be used to notify peer when the driver on this side is being
1135      * un-loaded.
1136      */
1137     ndev->db_last_bit =
1138             find_last_bit((unsigned long *)&ndev->db_valid_mask,
1139                       hweight64(ndev->db_valid_mask));
1140     writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET);
1141     /*
1142      * Since now there is one less bit to account for, the DB count
1143      * and DB mask should be adjusted accordingly.
1144      */
1145     ndev->db_count -= 1;
1146     ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1147 
1148     /* Enable Link-Up and Link-Down event interrupts */
1149     ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT);
1150     writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
1151 
1152     return 0;
1153 }
1154 
1155 static void amd_deinit_dev(struct amd_ntb_dev *ndev)
1156 {
1157     cancel_delayed_work_sync(&ndev->hb_timer);
1158 
1159     ndev_deinit_isr(ndev);
1160 }
1161 
1162 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
1163                 struct pci_dev *pdev)
1164 {
1165     int rc;
1166 
1167     pci_set_drvdata(pdev, ndev);
1168 
1169     rc = pci_enable_device(pdev);
1170     if (rc)
1171         goto err_pci_enable;
1172 
1173     rc = pci_request_regions(pdev, NTB_NAME);
1174     if (rc)
1175         goto err_pci_regions;
1176 
1177     pci_set_master(pdev);
1178 
1179     rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1180     if (rc) {
1181         rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1182         if (rc)
1183             goto err_dma_mask;
1184         dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1185     }
1186 
1187     ndev->self_mmio = pci_iomap(pdev, 0, 0);
1188     if (!ndev->self_mmio) {
1189         rc = -EIO;
1190         goto err_dma_mask;
1191     }
1192     ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1193 
1194     return 0;
1195 
1196 err_dma_mask:
1197     pci_clear_master(pdev);
1198     pci_release_regions(pdev);
1199 err_pci_regions:
1200     pci_disable_device(pdev);
1201 err_pci_enable:
1202     pci_set_drvdata(pdev, NULL);
1203     return rc;
1204 }
1205 
1206 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1207 {
1208     struct pci_dev *pdev = ndev->ntb.pdev;
1209 
1210     pci_iounmap(pdev, ndev->self_mmio);
1211 
1212     pci_clear_master(pdev);
1213     pci_release_regions(pdev);
1214     pci_disable_device(pdev);
1215     pci_set_drvdata(pdev, NULL);
1216 }
1217 
1218 static int amd_ntb_pci_probe(struct pci_dev *pdev,
1219                  const struct pci_device_id *id)
1220 {
1221     struct amd_ntb_dev *ndev;
1222     int rc, node;
1223 
1224     node = dev_to_node(&pdev->dev);
1225 
1226     ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1227     if (!ndev) {
1228         rc = -ENOMEM;
1229         goto err_ndev;
1230     }
1231 
1232     ndev->dev_data = (struct ntb_dev_data *)id->driver_data;
1233 
1234     ndev_init_struct(ndev, pdev);
1235 
1236     rc = amd_ntb_init_pci(ndev, pdev);
1237     if (rc)
1238         goto err_init_pci;
1239 
1240     rc = amd_init_dev(ndev);
1241     if (rc)
1242         goto err_init_dev;
1243 
1244     /* write side info */
1245     amd_init_side_info(ndev);
1246 
1247     amd_poll_link(ndev);
1248 
1249     ndev_init_debugfs(ndev);
1250 
1251     rc = ntb_register_device(&ndev->ntb);
1252     if (rc)
1253         goto err_register;
1254 
1255     dev_info(&pdev->dev, "NTB device registered.\n");
1256 
1257     return 0;
1258 
1259 err_register:
1260     ndev_deinit_debugfs(ndev);
1261     amd_deinit_dev(ndev);
1262 err_init_dev:
1263     amd_ntb_deinit_pci(ndev);
1264 err_init_pci:
1265     kfree(ndev);
1266 err_ndev:
1267     return rc;
1268 }
1269 
1270 static void amd_ntb_pci_remove(struct pci_dev *pdev)
1271 {
1272     struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1273 
1274     /*
1275      * Clear the READY bit in SIDEINFO register before sending DB event
1276      * to the peer. This will make sure that when the peer handles the
1277      * DB event, it correctly reads this bit as being 0.
1278      */
1279     amd_deinit_side_info(ndev);
1280     ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
1281     ntb_unregister_device(&ndev->ntb);
1282     ndev_deinit_debugfs(ndev);
1283     amd_deinit_dev(ndev);
1284     amd_ntb_deinit_pci(ndev);
1285     kfree(ndev);
1286 }
1287 
1288 static void amd_ntb_pci_shutdown(struct pci_dev *pdev)
1289 {
1290     struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1291 
1292     /* Send link down notification */
1293     ntb_link_event(&ndev->ntb);
1294 
1295     amd_deinit_side_info(ndev);
1296     ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
1297     ntb_unregister_device(&ndev->ntb);
1298     ndev_deinit_debugfs(ndev);
1299     amd_deinit_dev(ndev);
1300     amd_ntb_deinit_pci(ndev);
1301     kfree(ndev);
1302 }
1303 
1304 static const struct file_operations amd_ntb_debugfs_info = {
1305     .owner = THIS_MODULE,
1306     .open = simple_open,
1307     .read = ndev_debugfs_read,
1308 };
1309 
1310 static const struct ntb_dev_data dev_data[] = {
1311     { /* for device 145b */
1312         .mw_count = 3,
1313         .mw_idx = 1,
1314     },
1315     { /* for device 148b */
1316         .mw_count = 2,
1317         .mw_idx = 2,
1318     },
1319 };
1320 
1321 static const struct pci_device_id amd_ntb_pci_tbl[] = {
1322     { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] },
1323     { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] },
1324     { PCI_VDEVICE(AMD, 0x14c0), (kernel_ulong_t)&dev_data[1] },
1325     { PCI_VDEVICE(AMD, 0x14c3), (kernel_ulong_t)&dev_data[1] },
1326     { PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] },
1327     { 0, }
1328 };
1329 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1330 
1331 static struct pci_driver amd_ntb_pci_driver = {
1332     .name       = KBUILD_MODNAME,
1333     .id_table   = amd_ntb_pci_tbl,
1334     .probe      = amd_ntb_pci_probe,
1335     .remove     = amd_ntb_pci_remove,
1336     .shutdown   = amd_ntb_pci_shutdown,
1337 };
1338 
1339 static int __init amd_ntb_pci_driver_init(void)
1340 {
1341     pr_info("%s %s\n", NTB_DESC, NTB_VER);
1342 
1343     if (debugfs_initialized())
1344         debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1345 
1346     return pci_register_driver(&amd_ntb_pci_driver);
1347 }
1348 module_init(amd_ntb_pci_driver_init);
1349 
1350 static void __exit amd_ntb_pci_driver_exit(void)
1351 {
1352     pci_unregister_driver(&amd_ntb_pci_driver);
1353     debugfs_remove_recursive(debugfs_dir);
1354 }
1355 module_exit(amd_ntb_pci_driver_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!