perf/util/mmap.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2011-2017, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
0004  *
0005  * Parts came from evlist.c builtin-{top,stat,record}.c, see those files for further
0006  * copyright notes.
0007  */
0008
0009 #include <sys/mman.h>
0010 #include <inttypes.h>
0011 #include <asm/bug.h>
0012 #include <linux/zalloc.h>
0013 #include <stdlib.h>
0014 #include <string.h>
0015 #include <unistd.h> // sysconf()
0016 #include <perf/mmap.h>
0017 #ifdef HAVE_LIBNUMA_SUPPORT
0018 #include <numaif.h>
0019 #endif
0020 #include "cpumap.h"
0021 #include "debug.h"
0022 #include "event.h"
0023 #include "mmap.h"
0024 #include "../perf.h"
0025 #include <internal/lib.h> /* page_size */
0026 #include <linux/bitmap.h>
0027
0028 #define MASK_SIZE 1023
0029 void mmap_cpu_mask__scnprintf(struct mmap_cpu_mask *mask, const char *tag)
0030 {
0031     char buf[MASK_SIZE + 1];
0032     size_t len;
0033
0034     len = bitmap_scnprintf(mask->bits, mask->nbits, buf, MASK_SIZE);
0035     buf[len] = '\0';
0036     pr_debug("%p: %s mask[%zd]: %s\n", mask, tag, mask->nbits, buf);
0037 }
0038
0039 size_t mmap__mmap_len(struct mmap *map)
0040 {
0041     return perf_mmap__mmap_len(&map->core);
0042 }
0043
0044 int __weak auxtrace_mmap__mmap(struct auxtrace_mmap *mm __maybe_unused,
0045                    struct auxtrace_mmap_params *mp __maybe_unused,
0046                    void *userpg __maybe_unused,
0047                    int fd __maybe_unused)
0048 {
0049     return 0;
0050 }
0051
0052 void __weak auxtrace_mmap__munmap(struct auxtrace_mmap *mm __maybe_unused)
0053 {
0054 }
0055
0056 void __weak auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp __maybe_unused,
0057                        off_t auxtrace_offset __maybe_unused,
0058                        unsigned int auxtrace_pages __maybe_unused,
0059                        bool auxtrace_overwrite __maybe_unused)
0060 {
0061 }
0062
0063 void __weak auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp __maybe_unused,
0064                       struct evlist *evlist __maybe_unused,
0065                       struct evsel *evsel __maybe_unused,
0066                       int idx __maybe_unused)
0067 {
0068 }
0069
0070 #ifdef HAVE_AIO_SUPPORT
0071 static int perf_mmap__aio_enabled(struct mmap *map)
0072 {
0073     return map->aio.nr_cblocks > 0;
0074 }
0075
0076 #ifdef HAVE_LIBNUMA_SUPPORT
0077 static int perf_mmap__aio_alloc(struct mmap *map, int idx)
0078 {
0079     map->aio.data[idx] = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE,
0080                   MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
0081     if (map->aio.data[idx] == MAP_FAILED) {
0082         map->aio.data[idx] = NULL;
0083         return -1;
0084     }
0085
0086     return 0;
0087 }
0088
0089 static void perf_mmap__aio_free(struct mmap *map, int idx)
0090 {
0091     if (map->aio.data[idx]) {
0092         munmap(map->aio.data[idx], mmap__mmap_len(map));
0093         map->aio.data[idx] = NULL;
0094     }
0095 }
0096
0097 static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity)
0098 {
0099     void *data;
0100     size_t mmap_len;
0101     unsigned long *node_mask;
0102     unsigned long node_index;
0103     int err = 0;
0104
0105     if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) {
0106         data = map->aio.data[idx];
0107         mmap_len = mmap__mmap_len(map);
0108         node_index = cpu__get_node(cpu);
0109         node_mask = bitmap_zalloc(node_index + 1);
0110         if (!node_mask) {
0111             pr_err("Failed to allocate node mask for mbind: error %m\n");
0112             return -1;
0113         }
0114         set_bit(node_index, node_mask);
0115         if (mbind(data, mmap_len, MPOL_BIND, node_mask, node_index + 1 + 1, 0)) {
0116             pr_err("Failed to bind [%p-%p] AIO buffer to node %lu: error %m\n",
0117                 data, data + mmap_len, node_index);
0118             err = -1;
0119         }
0120         bitmap_free(node_mask);
0121     }
0122
0123     return err;
0124 }
0125 #else /* !HAVE_LIBNUMA_SUPPORT */
0126 static int perf_mmap__aio_alloc(struct mmap *map, int idx)
0127 {
0128     map->aio.data[idx] = malloc(mmap__mmap_len(map));
0129     if (map->aio.data[idx] == NULL)
0130         return -1;
0131
0132     return 0;
0133 }
0134
0135 static void perf_mmap__aio_free(struct mmap *map, int idx)
0136 {
0137     zfree(&(map->aio.data[idx]));
0138 }
0139
0140 static int perf_mmap__aio_bind(struct mmap *map __maybe_unused, int idx __maybe_unused,
0141         struct perf_cpu cpu __maybe_unused, int affinity __maybe_unused)
0142 {
0143     return 0;
0144 }
0145 #endif
0146
0147 static int perf_mmap__aio_mmap(struct mmap *map, struct mmap_params *mp)
0148 {
0149     int delta_max, i, prio, ret;
0150
0151     map->aio.nr_cblocks = mp->nr_cblocks;
0152     if (map->aio.nr_cblocks) {
0153         map->aio.aiocb = calloc(map->aio.nr_cblocks, sizeof(struct aiocb *));
0154         if (!map->aio.aiocb) {
0155             pr_debug2("failed to allocate aiocb for data buffer, error %m\n");
0156             return -1;
0157         }
0158         map->aio.cblocks = calloc(map->aio.nr_cblocks, sizeof(struct aiocb));
0159         if (!map->aio.cblocks) {
0160             pr_debug2("failed to allocate cblocks for data buffer, error %m\n");
0161             return -1;
0162         }
0163         map->aio.data = calloc(map->aio.nr_cblocks, sizeof(void *));
0164         if (!map->aio.data) {
0165             pr_debug2("failed to allocate data buffer, error %m\n");
0166             return -1;
0167         }
0168         delta_max = sysconf(_SC_AIO_PRIO_DELTA_MAX);
0169         for (i = 0; i < map->aio.nr_cblocks; ++i) {
0170             ret = perf_mmap__aio_alloc(map, i);
0171             if (ret == -1) {
0172                 pr_debug2("failed to allocate data buffer area, error %m");
0173                 return -1;
0174             }
0175             ret = perf_mmap__aio_bind(map, i, map->core.cpu, mp->affinity);
0176             if (ret == -1)
0177                 return -1;
0178             /*
0179              * Use cblock.aio_fildes value different from -1
0180              * to denote started aio write operation on the
0181              * cblock so it requires explicit record__aio_sync()
0182              * call prior the cblock may be reused again.
0183              */
0184             map->aio.cblocks[i].aio_fildes = -1;
0185             /*
0186              * Allocate cblocks with priority delta to have
0187              * faster aio write system calls because queued requests
0188              * are kept in separate per-prio queues and adding
0189              * a new request will iterate thru shorter per-prio
0190              * list. Blocks with numbers higher than
0191              *  _SC_AIO_PRIO_DELTA_MAX go with priority 0.
0192              */
0193             prio = delta_max - i;
0194             map->aio.cblocks[i].aio_reqprio = prio >= 0 ? prio : 0;
0195         }
0196     }
0197
0198     return 0;
0199 }
0200
0201 static void perf_mmap__aio_munmap(struct mmap *map)
0202 {
0203     int i;
0204
0205     for (i = 0; i < map->aio.nr_cblocks; ++i)
0206         perf_mmap__aio_free(map, i);
0207     if (map->aio.data)
0208         zfree(&map->aio.data);
0209     zfree(&map->aio.cblocks);
0210     zfree(&map->aio.aiocb);
0211 }
0212 #else /* !HAVE_AIO_SUPPORT */
0213 static int perf_mmap__aio_enabled(struct mmap *map __maybe_unused)
0214 {
0215     return 0;
0216 }
0217
0218 static int perf_mmap__aio_mmap(struct mmap *map __maybe_unused,
0219                    struct mmap_params *mp __maybe_unused)
0220 {
0221     return 0;
0222 }
0223
0224 static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused)
0225 {
0226 }
0227 #endif
0228
0229 void mmap__munmap(struct mmap *map)
0230 {
0231     bitmap_free(map->affinity_mask.bits);
0232
0233 #ifndef PYTHON_PERF
0234     zstd_fini(&map->zstd_data);
0235 #endif
0236
0237     perf_mmap__aio_munmap(map);
0238     if (map->data != NULL) {
0239         munmap(map->data, mmap__mmap_len(map));
0240         map->data = NULL;
0241     }
0242     auxtrace_mmap__munmap(&map->auxtrace_mmap);
0243 }
0244
0245 static void build_node_mask(int node, struct mmap_cpu_mask *mask)
0246 {
0247     int idx, nr_cpus;
0248     struct perf_cpu cpu;
0249     const struct perf_cpu_map *cpu_map = NULL;
0250
0251     cpu_map = cpu_map__online();
0252     if (!cpu_map)
0253         return;
0254
0255     nr_cpus = perf_cpu_map__nr(cpu_map);
0256     for (idx = 0; idx < nr_cpus; idx++) {
0257         cpu = perf_cpu_map__cpu(cpu_map, idx); /* map c index to online cpu index */
0258         if (cpu__get_node(cpu) == node)
0259             set_bit(cpu.cpu, mask->bits);
0260     }
0261 }
0262
0263 static int perf_mmap__setup_affinity_mask(struct mmap *map, struct mmap_params *mp)
0264 {
0265     map->affinity_mask.nbits = cpu__max_cpu().cpu;
0266     map->affinity_mask.bits = bitmap_zalloc(map->affinity_mask.nbits);
0267     if (!map->affinity_mask.bits)
0268         return -1;
0269
0270     if (mp->affinity == PERF_AFFINITY_NODE && cpu__max_node() > 1)
0271         build_node_mask(cpu__get_node(map->core.cpu), &map->affinity_mask);
0272     else if (mp->affinity == PERF_AFFINITY_CPU)
0273         set_bit(map->core.cpu.cpu, map->affinity_mask.bits);
0274
0275     return 0;
0276 }
0277
0278 int mmap__mmap(struct mmap *map, struct mmap_params *mp, int fd, struct perf_cpu cpu)
0279 {
0280     if (perf_mmap__mmap(&map->core, &mp->core, fd, cpu)) {
0281         pr_debug2("failed to mmap perf event ring buffer, error %d\n",
0282               errno);
0283         return -1;
0284     }
0285
0286     if (mp->affinity != PERF_AFFINITY_SYS &&
0287         perf_mmap__setup_affinity_mask(map, mp)) {
0288         pr_debug2("failed to alloc mmap affinity mask, error %d\n",
0289               errno);
0290         return -1;
0291     }
0292
0293     if (verbose == 2)
0294         mmap_cpu_mask__scnprintf(&map->affinity_mask, "mmap");
0295
0296     map->core.flush = mp->flush;
0297
0298     map->comp_level = mp->comp_level;
0299 #ifndef PYTHON_PERF
0300     if (zstd_init(&map->zstd_data, map->comp_level)) {
0301         pr_debug2("failed to init mmap compressor, error %d\n", errno);
0302         return -1;
0303     }
0304 #endif
0305
0306     if (map->comp_level && !perf_mmap__aio_enabled(map)) {
0307         map->data = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE,
0308                  MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
0309         if (map->data == MAP_FAILED) {
0310             pr_debug2("failed to mmap data buffer, error %d\n",
0311                     errno);
0312             map->data = NULL;
0313             return -1;
0314         }
0315     }
0316
0317     if (auxtrace_mmap__mmap(&map->auxtrace_mmap,
0318                 &mp->auxtrace_mp, map->core.base, fd))
0319         return -1;
0320
0321     return perf_mmap__aio_mmap(map, mp);
0322 }
0323
0324 int perf_mmap__push(struct mmap *md, void *to,
0325             int push(struct mmap *map, void *to, void *buf, size_t size))
0326 {
0327     u64 head = perf_mmap__read_head(&md->core);
0328     unsigned char *data = md->core.base + page_size;
0329     unsigned long size;
0330     void *buf;
0331     int rc = 0;
0332
0333     rc = perf_mmap__read_init(&md->core);
0334     if (rc < 0)
0335         return (rc == -EAGAIN) ? 1 : -1;
0336
0337     size = md->core.end - md->core.start;
0338
0339     if ((md->core.start & md->core.mask) + size != (md->core.end & md->core.mask)) {
0340         buf = &data[md->core.start & md->core.mask];
0341         size = md->core.mask + 1 - (md->core.start & md->core.mask);
0342         md->core.start += size;
0343
0344         if (push(md, to, buf, size) < 0) {
0345             rc = -1;
0346             goto out;
0347         }
0348     }
0349
0350     buf = &data[md->core.start & md->core.mask];
0351     size = md->core.end - md->core.start;
0352     md->core.start += size;
0353
0354     if (push(md, to, buf, size) < 0) {
0355         rc = -1;
0356         goto out;
0357     }
0358
0359     md->core.prev = head;
0360     perf_mmap__consume(&md->core);
0361 out:
0362     return rc;
0363 }
0364
0365 int mmap_cpu_mask__duplicate(struct mmap_cpu_mask *original, struct mmap_cpu_mask *clone)
0366 {
0367     clone->nbits = original->nbits;
0368     clone->bits  = bitmap_zalloc(original->nbits);
0369     if (!clone->bits)
0370         return -ENOMEM;
0371
0372     memcpy(clone->bits, original->bits, MMAP_CPU_MASK_BYTES(original));
0373     return 0;
0374 }