nfp/nfpcore/nfp_mutex.c

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
0003
0004 #include <linux/delay.h>
0005 #include <linux/device.h>
0006 #include <linux/jiffies.h>
0007 #include <linux/types.h>
0008 #include <linux/slab.h>
0009 #include <linux/wait.h>
0010
0011 #include "nfp_cpp.h"
0012 #include "nfp6000/nfp6000.h"
0013
0014 struct nfp_cpp_mutex {
0015     struct nfp_cpp *cpp;
0016     int target;
0017     u16 depth;
0018     unsigned long long address;
0019     u32 key;
0020 };
0021
0022 static u32 nfp_mutex_locked(u16 interface)
0023 {
0024     return (u32)interface << 16 | 0x000f;
0025 }
0026
0027 static u32 nfp_mutex_unlocked(u16 interface)
0028 {
0029     return (u32)interface << 16 | 0x0000;
0030 }
0031
0032 static u32 nfp_mutex_owner(u32 val)
0033 {
0034     return val >> 16;
0035 }
0036
0037 static bool nfp_mutex_is_locked(u32 val)
0038 {
0039     return (val & 0xffff) == 0x000f;
0040 }
0041
0042 static bool nfp_mutex_is_unlocked(u32 val)
0043 {
0044     return (val & 0xffff) == 0000;
0045 }
0046
0047 /* If you need more than 65536 recursive locks, please rethink your code. */
0048 #define NFP_MUTEX_DEPTH_MAX         0xffff
0049
0050 static int
0051 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
0052 {
0053     /* Not permitted on invalid interfaces */
0054     if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
0055         NFP_CPP_INTERFACE_TYPE_INVALID)
0056         return -EINVAL;
0057
0058     /* Address must be 64-bit aligned */
0059     if (address & 7)
0060         return -EINVAL;
0061
0062     if (*target != NFP_CPP_TARGET_MU)
0063         return -EINVAL;
0064
0065     return 0;
0066 }
0067
0068 /**
0069  * nfp_cpp_mutex_init() - Initialize a mutex location
0070  * @cpp:    NFP CPP handle
0071  * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
0072  * @address:    Offset into the address space of the NFP CPP target ID
0073  * @key:    Unique 32-bit value for this mutex
0074  *
0075  * The CPP target:address must point to a 64-bit aligned location, and
0076  * will initialize 64 bits of data at the location.
0077  *
0078  * This creates the initial mutex state, as locked by this
0079  * nfp_cpp_interface().
0080  *
0081  * This function should only be called when setting up
0082  * the initial lock state upon boot-up of the system.
0083  *
0084  * Return: 0 on success, or -errno on failure
0085  */
0086 int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
0087                int target, unsigned long long address, u32 key)
0088 {
0089     const u32 muw = NFP_CPP_ID(target, 4, 0);    /* atomic_write */
0090     u16 interface = nfp_cpp_interface(cpp);
0091     int err;
0092
0093     err = nfp_cpp_mutex_validate(interface, &target, address);
0094     if (err)
0095         return err;
0096
0097     err = nfp_cpp_writel(cpp, muw, address + 4, key);
0098     if (err)
0099         return err;
0100
0101     err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
0102     if (err)
0103         return err;
0104
0105     return 0;
0106 }
0107
0108 /**
0109  * nfp_cpp_mutex_alloc() - Create a mutex handle
0110  * @cpp:    NFP CPP handle
0111  * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
0112  * @address:    Offset into the address space of the NFP CPP target ID
0113  * @key:    32-bit unique key (must match the key at this location)
0114  *
0115  * The CPP target:address must point to a 64-bit aligned location, and
0116  * reserve 64 bits of data at the location for use by the handle.
0117  *
0118  * Only target/address pairs that point to entities that support the
0119  * MU Atomic Engine's CmpAndSwap32 command are supported.
0120  *
0121  * Return:  A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
0122  */
0123 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
0124                       unsigned long long address, u32 key)
0125 {
0126     const u32 mur = NFP_CPP_ID(target, 3, 0);    /* atomic_read */
0127     u16 interface = nfp_cpp_interface(cpp);
0128     struct nfp_cpp_mutex *mutex;
0129     int err;
0130     u32 tmp;
0131
0132     err = nfp_cpp_mutex_validate(interface, &target, address);
0133     if (err)
0134         return NULL;
0135
0136     err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
0137     if (err < 0)
0138         return NULL;
0139
0140     if (tmp != key)
0141         return NULL;
0142
0143     mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
0144     if (!mutex)
0145         return NULL;
0146
0147     mutex->cpp = cpp;
0148     mutex->target = target;
0149     mutex->address = address;
0150     mutex->key = key;
0151     mutex->depth = 0;
0152
0153     return mutex;
0154 }
0155
0156 /**
0157  * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
0158  * @mutex:  NFP CPP Mutex handle
0159  */
0160 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
0161 {
0162     kfree(mutex);
0163 }
0164
0165 /**
0166  * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
0167  * @mutex:  NFP CPP Mutex handle
0168  *
0169  * Return: 0 on success, or -errno on failure
0170  */
0171 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
0172 {
0173     unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
0174     unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ;
0175     unsigned int timeout_ms = 1;
0176     int err;
0177
0178     /* We can't use a waitqueue here, because the unlocker
0179      * might be on a separate CPU.
0180      *
0181      * So just wait for now.
0182      */
0183     for (;;) {
0184         err = nfp_cpp_mutex_trylock(mutex);
0185         if (err != -EBUSY)
0186             break;
0187
0188         err = msleep_interruptible(timeout_ms);
0189         if (err != 0) {
0190             nfp_info(mutex->cpp,
0191                  "interrupted waiting for NFP mutex\n");
0192             return -ERESTARTSYS;
0193         }
0194
0195         if (time_is_before_eq_jiffies(warn_at)) {
0196             warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
0197             nfp_warn(mutex->cpp,
0198                  "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n",
0199                  mutex->depth,
0200                  mutex->target, mutex->address, mutex->key);
0201         }
0202         if (time_is_before_eq_jiffies(err_at)) {
0203             nfp_err(mutex->cpp, "Error: mutex wait timed out\n");
0204             return -EBUSY;
0205         }
0206     }
0207
0208     return err;
0209 }
0210
0211 /**
0212  * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
0213  * @mutex:  NFP CPP Mutex handle
0214  *
0215  * Return: 0 on success, or -errno on failure
0216  */
0217 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
0218 {
0219     const u32 muw = NFP_CPP_ID(mutex->target, 4, 0);    /* atomic_write */
0220     const u32 mur = NFP_CPP_ID(mutex->target, 3, 0);    /* atomic_read */
0221     struct nfp_cpp *cpp = mutex->cpp;
0222     u32 key, value;
0223     u16 interface;
0224     int err;
0225
0226     interface = nfp_cpp_interface(cpp);
0227
0228     if (mutex->depth > 1) {
0229         mutex->depth--;
0230         return 0;
0231     }
0232
0233     err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
0234     if (err < 0)
0235         return err;
0236
0237     if (key != mutex->key)
0238         return -EPERM;
0239
0240     err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
0241     if (err < 0)
0242         return err;
0243
0244     if (value != nfp_mutex_locked(interface))
0245         return -EACCES;
0246
0247     err = nfp_cpp_writel(cpp, muw, mutex->address,
0248                  nfp_mutex_unlocked(interface));
0249     if (err < 0)
0250         return err;
0251
0252     mutex->depth = 0;
0253     return 0;
0254 }
0255
0256 /**
0257  * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
0258  * @mutex:  NFP CPP Mutex handle
0259  *
0260  * Return:      0 if the lock succeeded, -errno on failure
0261  */
0262 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
0263 {
0264     const u32 muw = NFP_CPP_ID(mutex->target, 4, 0);    /* atomic_write */
0265     const u32 mus = NFP_CPP_ID(mutex->target, 5, 3);    /* test_set_imm */
0266     const u32 mur = NFP_CPP_ID(mutex->target, 3, 0);    /* atomic_read */
0267     struct nfp_cpp *cpp = mutex->cpp;
0268     u32 key, value, tmp;
0269     int err;
0270
0271     if (mutex->depth > 0) {
0272         if (mutex->depth == NFP_MUTEX_DEPTH_MAX)
0273             return -E2BIG;
0274         mutex->depth++;
0275         return 0;
0276     }
0277
0278     /* Verify that the lock marker is not damaged */
0279     err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
0280     if (err < 0)
0281         return err;
0282
0283     if (key != mutex->key)
0284         return -EPERM;
0285
0286     /* Compare against the unlocked state, and if true,
0287      * write the interface id into the top 16 bits, and
0288      * mark as locked.
0289      */
0290     value = nfp_mutex_locked(nfp_cpp_interface(cpp));
0291
0292     /* We use test_set_imm here, as it implies a read
0293      * of the current state, and sets the bits in the
0294      * bytemask of the command to 1s. Since the mutex
0295      * is guaranteed to be 64-bit aligned, the bytemask
0296      * of this 32-bit command is ensured to be 8'b00001111,
0297      * which implies that the lower 4 bits will be set to
0298      * ones regardless of the initial state.
0299      *
0300      * Since this is a 'Readback' operation, with no Pull
0301      * data, we can treat this as a normal Push (read)
0302      * atomic, which returns the original value.
0303      */
0304     err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
0305     if (err < 0)
0306         return err;
0307
0308     /* Was it unlocked? */
0309     if (nfp_mutex_is_unlocked(tmp)) {
0310         /* The read value can only be 0x....0000 in the unlocked state.
0311          * If there was another contending for this lock, then
0312          * the lock state would be 0x....000f
0313          */
0314
0315         /* Write our owner ID into the lock
0316          * While not strictly necessary, this helps with
0317          * debug and bookkeeping.
0318          */
0319         err = nfp_cpp_writel(cpp, muw, mutex->address, value);
0320         if (err < 0)
0321             return err;
0322
0323         mutex->depth = 1;
0324         return 0;
0325     }
0326
0327     return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
0328 }
0329
0330 /**
0331  * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint
0332  * @cpp:    NFP CPP handle
0333  * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
0334  * @address:    Offset into the address space of the NFP CPP target ID
0335  *
0336  * Release lock if held by local system.  Extreme care is advised, call only
0337  * when no local lock users can exist.
0338  *
0339  * Return:      0 if the lock was OK, 1 if locked by us, -errno on invalid mutex
0340  */
0341 int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
0342               unsigned long long address)
0343 {
0344     const u32 mur = NFP_CPP_ID(target, 3, 0);   /* atomic_read */
0345     const u32 muw = NFP_CPP_ID(target, 4, 0);   /* atomic_write */
0346     u16 interface = nfp_cpp_interface(cpp);
0347     int err;
0348     u32 tmp;
0349
0350     err = nfp_cpp_mutex_validate(interface, &target, address);
0351     if (err)
0352         return err;
0353
0354     /* Check lock */
0355     err = nfp_cpp_readl(cpp, mur, address, &tmp);
0356     if (err < 0)
0357         return err;
0358
0359     if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface)
0360         return 0;
0361
0362     /* Bust the lock */
0363     err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface));
0364     if (err < 0)
0365         return err;
0366
0367     return 1;
0368 }