misc/ocxl/core.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 // Copyright 2019 IBM Corp.
0003 #include <linux/idr.h>
0004 #include "ocxl_internal.h"
0005
0006 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
0007 {
0008     return (get_device(&fn->dev) == NULL) ? NULL : fn;
0009 }
0010
0011 static void ocxl_fn_put(struct ocxl_fn *fn)
0012 {
0013     put_device(&fn->dev);
0014 }
0015
0016 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
0017 {
0018     struct ocxl_afu *afu;
0019
0020     afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL);
0021     if (!afu)
0022         return NULL;
0023
0024     kref_init(&afu->kref);
0025     mutex_init(&afu->contexts_lock);
0026     mutex_init(&afu->afu_control_lock);
0027     idr_init(&afu->contexts_idr);
0028     afu->fn = fn;
0029     ocxl_fn_get(fn);
0030     return afu;
0031 }
0032
0033 static void free_afu(struct kref *kref)
0034 {
0035     struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref);
0036
0037     idr_destroy(&afu->contexts_idr);
0038     ocxl_fn_put(afu->fn);
0039     kfree(afu);
0040 }
0041
0042 void ocxl_afu_get(struct ocxl_afu *afu)
0043 {
0044     kref_get(&afu->kref);
0045 }
0046 EXPORT_SYMBOL_GPL(ocxl_afu_get);
0047
0048 void ocxl_afu_put(struct ocxl_afu *afu)
0049 {
0050     kref_put(&afu->kref, free_afu);
0051 }
0052 EXPORT_SYMBOL_GPL(ocxl_afu_put);
0053
0054 static int assign_afu_actag(struct ocxl_afu *afu)
0055 {
0056     struct ocxl_fn *fn = afu->fn;
0057     int actag_count, actag_offset;
0058     struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
0059
0060     /*
0061      * if there were not enough actags for the function, each afu
0062      * reduces its count as well
0063      */
0064     actag_count = afu->config.actag_supported *
0065         fn->actag_enabled / fn->actag_supported;
0066     actag_offset = ocxl_actag_afu_alloc(fn, actag_count);
0067     if (actag_offset < 0) {
0068         dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n",
0069             actag_count, actag_offset);
0070         return actag_offset;
0071     }
0072     afu->actag_base = fn->actag_base + actag_offset;
0073     afu->actag_enabled = actag_count;
0074
0075     ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos,
0076                 afu->actag_base, afu->actag_enabled);
0077     dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n",
0078         afu->actag_base, afu->actag_enabled);
0079     return 0;
0080 }
0081
0082 static void reclaim_afu_actag(struct ocxl_afu *afu)
0083 {
0084     struct ocxl_fn *fn = afu->fn;
0085     int start_offset, size;
0086
0087     start_offset = afu->actag_base - fn->actag_base;
0088     size = afu->actag_enabled;
0089     ocxl_actag_afu_free(afu->fn, start_offset, size);
0090 }
0091
0092 static int assign_afu_pasid(struct ocxl_afu *afu)
0093 {
0094     struct ocxl_fn *fn = afu->fn;
0095     int pasid_count, pasid_offset;
0096     struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
0097
0098     /*
0099      * We only support the case where the function configuration
0100      * requested enough PASIDs to cover all AFUs.
0101      */
0102     pasid_count = 1 << afu->config.pasid_supported_log;
0103     pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count);
0104     if (pasid_offset < 0) {
0105         dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n",
0106             pasid_count, pasid_offset);
0107         return pasid_offset;
0108     }
0109     afu->pasid_base = fn->pasid_base + pasid_offset;
0110     afu->pasid_count = 0;
0111     afu->pasid_max = pasid_count;
0112
0113     ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos,
0114                 afu->pasid_base,
0115                 afu->config.pasid_supported_log);
0116     dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n",
0117         afu->pasid_base, pasid_count);
0118     return 0;
0119 }
0120
0121 static void reclaim_afu_pasid(struct ocxl_afu *afu)
0122 {
0123     struct ocxl_fn *fn = afu->fn;
0124     int start_offset, size;
0125
0126     start_offset = afu->pasid_base - fn->pasid_base;
0127     size = 1 << afu->config.pasid_supported_log;
0128     ocxl_pasid_afu_free(afu->fn, start_offset, size);
0129 }
0130
0131 static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
0132 {
0133     struct pci_dev *dev = to_pci_dev(fn->dev.parent);
0134     int rc, idx;
0135
0136     if (bar != 0 && bar != 2 && bar != 4)
0137         return -EINVAL;
0138
0139     idx = bar >> 1;
0140     if (fn->bar_used[idx]++ == 0) {
0141         rc = pci_request_region(dev, bar, "ocxl");
0142         if (rc)
0143             return rc;
0144     }
0145     return 0;
0146 }
0147
0148 static void release_fn_bar(struct ocxl_fn *fn, int bar)
0149 {
0150     struct pci_dev *dev = to_pci_dev(fn->dev.parent);
0151     int idx;
0152
0153     if (bar != 0 && bar != 2 && bar != 4)
0154         return;
0155
0156     idx = bar >> 1;
0157     if (--fn->bar_used[idx] == 0)
0158         pci_release_region(dev, bar);
0159     WARN_ON(fn->bar_used[idx] < 0);
0160 }
0161
0162 static int map_mmio_areas(struct ocxl_afu *afu)
0163 {
0164     int rc;
0165     struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent);
0166
0167     rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);
0168     if (rc)
0169         return rc;
0170
0171     rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);
0172     if (rc) {
0173         release_fn_bar(afu->fn, afu->config.global_mmio_bar);
0174         return rc;
0175     }
0176
0177     afu->global_mmio_start =
0178         pci_resource_start(pci_dev, afu->config.global_mmio_bar) +
0179         afu->config.global_mmio_offset;
0180     afu->pp_mmio_start =
0181         pci_resource_start(pci_dev, afu->config.pp_mmio_bar) +
0182         afu->config.pp_mmio_offset;
0183
0184     afu->global_mmio_ptr = ioremap(afu->global_mmio_start,
0185                 afu->config.global_mmio_size);
0186     if (!afu->global_mmio_ptr) {
0187         release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
0188         release_fn_bar(afu->fn, afu->config.global_mmio_bar);
0189         dev_err(&pci_dev->dev, "Error mapping global mmio area\n");
0190         return -ENOMEM;
0191     }
0192
0193     /*
0194      * Leave an empty page between the per-process mmio area and
0195      * the AFU interrupt mappings
0196      */
0197     afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
0198     return 0;
0199 }
0200
0201 static void unmap_mmio_areas(struct ocxl_afu *afu)
0202 {
0203     if (afu->global_mmio_ptr) {
0204         iounmap(afu->global_mmio_ptr);
0205         afu->global_mmio_ptr = NULL;
0206     }
0207     afu->global_mmio_start = 0;
0208     afu->pp_mmio_start = 0;
0209     release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
0210     release_fn_bar(afu->fn, afu->config.global_mmio_bar);
0211 }
0212
0213 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
0214 {
0215     int rc;
0216
0217     rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);
0218     if (rc)
0219         return rc;
0220
0221     rc = assign_afu_actag(afu);
0222     if (rc)
0223         return rc;
0224
0225     rc = assign_afu_pasid(afu);
0226     if (rc)
0227         goto err_free_actag;
0228
0229     rc = map_mmio_areas(afu);
0230     if (rc)
0231         goto err_free_pasid;
0232
0233     return 0;
0234
0235 err_free_pasid:
0236     reclaim_afu_pasid(afu);
0237 err_free_actag:
0238     reclaim_afu_actag(afu);
0239     return rc;
0240 }
0241
0242 static void deconfigure_afu(struct ocxl_afu *afu)
0243 {
0244     unmap_mmio_areas(afu);
0245     reclaim_afu_pasid(afu);
0246     reclaim_afu_actag(afu);
0247 }
0248
0249 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
0250 {
0251     ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);
0252
0253     return 0;
0254 }
0255
0256 static void deactivate_afu(struct ocxl_afu *afu)
0257 {
0258     struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
0259
0260     ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);
0261 }
0262
0263 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
0264 {
0265     int rc;
0266     struct ocxl_afu *afu;
0267
0268     afu = alloc_afu(fn);
0269     if (!afu)
0270         return -ENOMEM;
0271
0272     rc = configure_afu(afu, afu_idx, dev);
0273     if (rc) {
0274         ocxl_afu_put(afu);
0275         return rc;
0276     }
0277
0278     rc = activate_afu(dev, afu);
0279     if (rc) {
0280         deconfigure_afu(afu);
0281         ocxl_afu_put(afu);
0282         return rc;
0283     }
0284
0285     list_add_tail(&afu->list, &fn->afu_list);
0286
0287     return 0;
0288 }
0289
0290 static void remove_afu(struct ocxl_afu *afu)
0291 {
0292     list_del(&afu->list);
0293     ocxl_context_detach_all(afu);
0294     deactivate_afu(afu);
0295     deconfigure_afu(afu);
0296     ocxl_afu_put(afu); // matches the implicit get in alloc_afu
0297 }
0298
0299 static struct ocxl_fn *alloc_function(void)
0300 {
0301     struct ocxl_fn *fn;
0302
0303     fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL);
0304     if (!fn)
0305         return NULL;
0306
0307     INIT_LIST_HEAD(&fn->afu_list);
0308     INIT_LIST_HEAD(&fn->pasid_list);
0309     INIT_LIST_HEAD(&fn->actag_list);
0310
0311     return fn;
0312 }
0313
0314 static void free_function(struct ocxl_fn *fn)
0315 {
0316     WARN_ON(!list_empty(&fn->afu_list));
0317     WARN_ON(!list_empty(&fn->pasid_list));
0318     kfree(fn);
0319 }
0320
0321 static void free_function_dev(struct device *dev)
0322 {
0323     struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev);
0324
0325     free_function(fn);
0326 }
0327
0328 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
0329 {
0330     fn->dev.parent = &dev->dev;
0331     fn->dev.release = free_function_dev;
0332     return dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));
0333 }
0334
0335 static int assign_function_actag(struct ocxl_fn *fn)
0336 {
0337     struct pci_dev *dev = to_pci_dev(fn->dev.parent);
0338     u16 base, enabled, supported;
0339     int rc;
0340
0341     rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported);
0342     if (rc)
0343         return rc;
0344
0345     fn->actag_base = base;
0346     fn->actag_enabled = enabled;
0347     fn->actag_supported = supported;
0348
0349     ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,
0350             fn->actag_base, fn->actag_enabled);
0351     dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
0352         fn->actag_base, fn->actag_enabled);
0353     return 0;
0354 }
0355
0356 static int set_function_pasid(struct ocxl_fn *fn)
0357 {
0358     struct pci_dev *dev = to_pci_dev(fn->dev.parent);
0359     int rc, desired_count, max_count;
0360
0361     /* A function may not require any PASID */
0362     if (fn->config.max_pasid_log < 0)
0363         return 0;
0364
0365     rc = ocxl_config_get_pasid_info(dev, &max_count);
0366     if (rc)
0367         return rc;
0368
0369     desired_count = 1 << fn->config.max_pasid_log;
0370
0371     if (desired_count > max_count) {
0372         dev_err(&fn->dev,
0373             "Function requires more PASIDs than is available (%d vs. %d)\n",
0374             desired_count, max_count);
0375         return -ENOSPC;
0376     }
0377
0378     fn->pasid_base = 0;
0379     return 0;
0380 }
0381
0382 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
0383 {
0384     int rc;
0385
0386     rc = pci_enable_device(dev);
0387     if (rc) {
0388         dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
0389         return rc;
0390     }
0391
0392     /*
0393      * Once it has been confirmed to work on our hardware, we
0394      * should reset the function, to force the adapter to restart
0395      * from scratch.
0396      * A function reset would also reset all its AFUs.
0397      *
0398      * Some hints for implementation:
0399      *
0400      * - there's not status bit to know when the reset is done. We
0401      *   should try reading the config space to know when it's
0402      *   done.
0403      * - probably something like:
0404      *  Reset
0405      *  wait 100ms
0406      *  issue config read
0407      *  allow device up to 1 sec to return success on config
0408      *  read before declaring it broken
0409      *
0410      * Some shared logic on the card (CFG, TLX) won't be reset, so
0411      * there's no guarantee that it will be enough.
0412      */
0413     rc = ocxl_config_read_function(dev, &fn->config);
0414     if (rc)
0415         return rc;
0416
0417     rc = set_function_device(fn, dev);
0418     if (rc)
0419         return rc;
0420
0421     rc = assign_function_actag(fn);
0422     if (rc)
0423         return rc;
0424
0425     rc = set_function_pasid(fn);
0426     if (rc)
0427         return rc;
0428
0429     rc = ocxl_link_setup(dev, 0, &fn->link);
0430     if (rc)
0431         return rc;
0432
0433     rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);
0434     if (rc) {
0435         ocxl_link_release(dev, fn->link);
0436         return rc;
0437     }
0438     return 0;
0439 }
0440
0441 static void deconfigure_function(struct ocxl_fn *fn)
0442 {
0443     struct pci_dev *dev = to_pci_dev(fn->dev.parent);
0444
0445     ocxl_link_release(dev, fn->link);
0446     pci_disable_device(dev);
0447 }
0448
0449 static struct ocxl_fn *init_function(struct pci_dev *dev)
0450 {
0451     struct ocxl_fn *fn;
0452     int rc;
0453
0454     fn = alloc_function();
0455     if (!fn)
0456         return ERR_PTR(-ENOMEM);
0457
0458     rc = configure_function(fn, dev);
0459     if (rc) {
0460         free_function(fn);
0461         return ERR_PTR(rc);
0462     }
0463
0464     rc = device_register(&fn->dev);
0465     if (rc) {
0466         deconfigure_function(fn);
0467         put_device(&fn->dev);
0468         return ERR_PTR(rc);
0469     }
0470     return fn;
0471 }
0472
0473 // Device detection & initialisation
0474
0475 struct ocxl_fn *ocxl_function_open(struct pci_dev *dev)
0476 {
0477     int rc, afu_count = 0;
0478     u8 afu;
0479     struct ocxl_fn *fn;
0480
0481     if (!radix_enabled()) {
0482         dev_err(&dev->dev, "Unsupported memory model (hash)\n");
0483         return ERR_PTR(-ENODEV);
0484     }
0485
0486     fn = init_function(dev);
0487     if (IS_ERR(fn)) {
0488         dev_err(&dev->dev, "function init failed: %li\n",
0489             PTR_ERR(fn));
0490         return fn;
0491     }
0492
0493     for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
0494         rc = ocxl_config_check_afu_index(dev, &fn->config, afu);
0495         if (rc > 0) {
0496             rc = init_afu(dev, fn, afu);
0497             if (rc) {
0498                 dev_err(&dev->dev,
0499                     "Can't initialize AFU index %d\n", afu);
0500                 continue;
0501             }
0502             afu_count++;
0503         }
0504     }
0505     dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
0506     return fn;
0507 }
0508 EXPORT_SYMBOL_GPL(ocxl_function_open);
0509
0510 struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn)
0511 {
0512     return &fn->afu_list;
0513 }
0514 EXPORT_SYMBOL_GPL(ocxl_function_afu_list);
0515
0516 struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx)
0517 {
0518     struct ocxl_afu *afu;
0519
0520     list_for_each_entry(afu, &fn->afu_list, list) {
0521         if (afu->config.idx == afu_idx)
0522             return afu;
0523     }
0524
0525     return NULL;
0526 }
0527 EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu);
0528
0529 const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn)
0530 {
0531     return &fn->config;
0532 }
0533 EXPORT_SYMBOL_GPL(ocxl_function_config);
0534
0535 void ocxl_function_close(struct ocxl_fn *fn)
0536 {
0537     struct ocxl_afu *afu, *tmp;
0538
0539     list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
0540         remove_afu(afu);
0541     }
0542
0543     deconfigure_function(fn);
0544     device_unregister(&fn->dev);
0545 }
0546 EXPORT_SYMBOL_GPL(ocxl_function_close);
0547
0548 // AFU Metadata
0549
0550 struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu)
0551 {
0552     return &afu->config;
0553 }
0554 EXPORT_SYMBOL_GPL(ocxl_afu_config);
0555
0556 void ocxl_afu_set_private(struct ocxl_afu *afu, void *private)
0557 {
0558     afu->private = private;
0559 }
0560 EXPORT_SYMBOL_GPL(ocxl_afu_set_private);
0561
0562 void *ocxl_afu_get_private(struct ocxl_afu *afu)
0563 {
0564     if (afu)
0565         return afu->private;
0566
0567     return NULL;
0568 }
0569 EXPORT_SYMBOL_GPL(ocxl_afu_get_private);