microsoft/mana/shm_channel.c

0001 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
0002 /* Copyright (c) 2021, Microsoft Corporation. */
0003
0004 #include <linux/delay.h>
0005 #include <linux/device.h>
0006 #include <linux/io.h>
0007 #include <linux/mm.h>
0008
0009 #include "shm_channel.h"
0010
0011 #define PAGE_FRAME_L48_WIDTH_BYTES 6
0012 #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
0013 #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
0014 #define PAGE_FRAME_H4_WIDTH_BITS 4
0015 #define VECTOR_MASK 0xFFFF
0016 #define SHMEM_VF_RESET_STATE ((u32)-1)
0017
0018 #define SMC_MSG_TYPE_ESTABLISH_HWC 1
0019 #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
0020
0021 #define SMC_MSG_TYPE_DESTROY_HWC 2
0022 #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
0023
0024 #define SMC_MSG_DIRECTION_REQUEST 0
0025 #define SMC_MSG_DIRECTION_RESPONSE 1
0026
0027 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
0028  * them are naturally aligned and hence don't need __packed.
0029  */
0030
0031 /* Shared memory channel protocol header
0032  *
0033  * msg_type: set on request and response; response matches request.
0034  * msg_version: newer PF writes back older response (matching request)
0035  *  older PF acts on latest version known and sets that version in result
0036  *  (less than request).
0037  * direction: 0 for request, VF->PF; 1 for response, PF->VF.
0038  * status: 0 on request,
0039  *   operation result on response (success = 0, failure = 1 or greater).
0040  * reset_vf: If set on either establish or destroy request, indicates perform
0041  *  FLR before/after the operation.
0042  * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
0043  */
0044 union smc_proto_hdr {
0045     u32 as_uint32;
0046
0047     struct {
0048         u8 msg_type : 3;
0049         u8 msg_version  : 3;
0050         u8 reserved_1   : 1;
0051         u8 direction    : 1;
0052
0053         u8 status;
0054
0055         u8 reserved_2;
0056
0057         u8 reset_vf : 1;
0058         u8 reserved_3   : 6;
0059         u8 owner_is_pf  : 1;
0060     };
0061 }; /* HW DATA */
0062
0063 #define SMC_APERTURE_BITS 256
0064 #define SMC_BASIC_UNIT (sizeof(u32))
0065 #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
0066 #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
0067
0068 static int mana_smc_poll_register(void __iomem *base, bool reset)
0069 {
0070     void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
0071     u32 last_dword;
0072     int i;
0073
0074     /* Poll the hardware for the ownership bit. This should be pretty fast,
0075      * but let's do it in a loop just in case the hardware or the PF
0076      * driver are temporarily busy.
0077      */
0078     for (i = 0; i < 20 * 1000; i++)  {
0079         last_dword = readl(ptr);
0080
0081         /* shmem reads as 0xFFFFFFFF in the reset case */
0082         if (reset && last_dword == SHMEM_VF_RESET_STATE)
0083             return 0;
0084
0085         /* If bit_31 is set, the PF currently owns the SMC. */
0086         if (!(last_dword & BIT(31)))
0087             return 0;
0088
0089         usleep_range(1000, 2000);
0090     }
0091
0092     return -ETIMEDOUT;
0093 }
0094
0095 static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
0096                   u32 msg_version, bool reset_vf)
0097 {
0098     void __iomem *base = sc->base;
0099     union smc_proto_hdr hdr;
0100     int err;
0101
0102     /* Wait for PF to respond. */
0103     err = mana_smc_poll_register(base, reset_vf);
0104     if (err)
0105         return err;
0106
0107     hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
0108
0109     if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
0110         return 0;
0111
0112     /* Validate protocol fields from the PF driver */
0113     if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
0114         hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
0115         dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
0116             hdr.as_uint32, msg_type, msg_version);
0117         return -EPROTO;
0118     }
0119
0120     /* Validate the operation result */
0121     if (hdr.status != 0) {
0122         dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
0123         return -EPROTO;
0124     }
0125
0126     return 0;
0127 }
0128
0129 void mana_smc_init(struct shm_channel *sc, struct device *dev,
0130            void __iomem *base)
0131 {
0132     sc->dev = dev;
0133     sc->base = base;
0134 }
0135
0136 int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
0137                u64 cq_addr, u64 rq_addr, u64 sq_addr,
0138                u32 eq_msix_index)
0139 {
0140     union smc_proto_hdr *hdr;
0141     u16 all_addr_h4bits = 0;
0142     u16 frame_addr_seq = 0;
0143     u64 frame_addr = 0;
0144     u8 shm_buf[32];
0145     u64 *shmem;
0146     u32 *dword;
0147     u8 *ptr;
0148     int err;
0149     int i;
0150
0151     /* Ensure VF already has possession of shared memory */
0152     err = mana_smc_poll_register(sc->base, false);
0153     if (err) {
0154         dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
0155         return err;
0156     }
0157
0158     if (!PAGE_ALIGNED(eq_addr) || !PAGE_ALIGNED(cq_addr) ||
0159         !PAGE_ALIGNED(rq_addr) || !PAGE_ALIGNED(sq_addr))
0160         return -EINVAL;
0161
0162     if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
0163         return -EINVAL;
0164
0165     /* Scheme for packing four addresses and extra info into 256 bits.
0166      *
0167      * Addresses must be page frame aligned, so only frame address bits
0168      * are transferred.
0169      *
0170      * 52-bit frame addresses are split into the lower 48 bits and upper
0171      * 4 bits. Lower 48 bits of 4 address are written sequentially from
0172      * the start of the 256-bit shared memory region followed by 16 bits
0173      * containing the upper 4 bits of the 4 addresses in sequence.
0174      *
0175      * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
0176      *
0177      * The final 32-bit dword is used for protocol control information as
0178      * defined in smc_proto_hdr.
0179      */
0180
0181     memset(shm_buf, 0, sizeof(shm_buf));
0182     ptr = shm_buf;
0183
0184     /* EQ addr: low 48 bits of frame address */
0185     shmem = (u64 *)ptr;
0186     frame_addr = PHYS_PFN(eq_addr);
0187     *shmem = frame_addr & PAGE_FRAME_L48_MASK;
0188     all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
0189         (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
0190     ptr += PAGE_FRAME_L48_WIDTH_BYTES;
0191
0192     /* CQ addr: low 48 bits of frame address */
0193     shmem = (u64 *)ptr;
0194     frame_addr = PHYS_PFN(cq_addr);
0195     *shmem = frame_addr & PAGE_FRAME_L48_MASK;
0196     all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
0197         (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
0198     ptr += PAGE_FRAME_L48_WIDTH_BYTES;
0199
0200     /* RQ addr: low 48 bits of frame address */
0201     shmem = (u64 *)ptr;
0202     frame_addr = PHYS_PFN(rq_addr);
0203     *shmem = frame_addr & PAGE_FRAME_L48_MASK;
0204     all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
0205         (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
0206     ptr += PAGE_FRAME_L48_WIDTH_BYTES;
0207
0208     /* SQ addr: low 48 bits of frame address */
0209     shmem = (u64 *)ptr;
0210     frame_addr = PHYS_PFN(sq_addr);
0211     *shmem = frame_addr & PAGE_FRAME_L48_MASK;
0212     all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
0213         (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
0214     ptr += PAGE_FRAME_L48_WIDTH_BYTES;
0215
0216     /* High 4 bits of the four frame addresses */
0217     *((u16 *)ptr) = all_addr_h4bits;
0218     ptr += sizeof(u16);
0219
0220     /* EQ MSIX vector number */
0221     *((u16 *)ptr) = (u16)eq_msix_index;
0222     ptr += sizeof(u16);
0223
0224     /* 32-bit protocol header in final dword */
0225     *((u32 *)ptr) = 0;
0226
0227     hdr = (union smc_proto_hdr *)ptr;
0228     hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
0229     hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
0230     hdr->direction = SMC_MSG_DIRECTION_REQUEST;
0231     hdr->reset_vf = reset_vf;
0232
0233     /* Write 256-message buffer to shared memory (final 32-bit write
0234      * triggers HW to set possession bit to PF).
0235      */
0236     dword = (u32 *)shm_buf;
0237     for (i = 0; i < SMC_APERTURE_DWORDS; i++)
0238         writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
0239
0240     /* Read shmem response (polling for VF possession) and validate.
0241      * For setup, waiting for response on shared memory is not strictly
0242      * necessary, since wait occurs later for results to appear in EQE's.
0243      */
0244     err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
0245                      SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
0246                      reset_vf);
0247     if (err) {
0248         dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
0249         return err;
0250     }
0251
0252     return 0;
0253 }
0254
0255 int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
0256 {
0257     union smc_proto_hdr hdr = {};
0258     int err;
0259
0260     /* Ensure already has possession of shared memory */
0261     err = mana_smc_poll_register(sc->base, false);
0262     if (err) {
0263         dev_err(sc->dev, "Timeout when tearing down HWC\n");
0264         return err;
0265     }
0266
0267     /* Set up protocol header for HWC destroy message */
0268     hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
0269     hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
0270     hdr.direction = SMC_MSG_DIRECTION_REQUEST;
0271     hdr.reset_vf = reset_vf;
0272
0273     /* Write message in high 32 bits of 256-bit shared memory, causing HW
0274      * to set possession bit to PF.
0275      */
0276     writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
0277
0278     /* Read shmem response (polling for VF possession) and validate.
0279      * For teardown, waiting for response is required to ensure hardware
0280      * invalidates MST entries before software frees memory.
0281      */
0282     err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
0283                      SMC_MSG_TYPE_DESTROY_HWC_VERSION,
0284                      reset_vf);
0285     if (err) {
0286         dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
0287         return err;
0288     }
0289
0290     return 0;
0291 }