0001
0002 #include <perf/evlist.h>
0003 #include <perf/evsel.h>
0004 #include <linux/bitops.h>
0005 #include <linux/list.h>
0006 #include <linux/hash.h>
0007 #include <sys/ioctl.h>
0008 #include <internal/evlist.h>
0009 #include <internal/evsel.h>
0010 #include <internal/xyarray.h>
0011 #include <internal/mmap.h>
0012 #include <internal/cpumap.h>
0013 #include <internal/threadmap.h>
0014 #include <internal/lib.h>
0015 #include <linux/zalloc.h>
0016 #include <stdlib.h>
0017 #include <errno.h>
0018 #include <unistd.h>
0019 #include <fcntl.h>
0020 #include <signal.h>
0021 #include <poll.h>
0022 #include <sys/mman.h>
0023 #include <perf/cpumap.h>
0024 #include <perf/threadmap.h>
0025 #include <api/fd/array.h>
0026 #include "internal.h"
0027
0028 void perf_evlist__init(struct perf_evlist *evlist)
0029 {
0030 INIT_LIST_HEAD(&evlist->entries);
0031 evlist->nr_entries = 0;
0032 fdarray__init(&evlist->pollfd, 64);
0033 perf_evlist__reset_id_hash(evlist);
0034 }
0035
0036 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
0037 struct perf_evsel *evsel)
0038 {
0039
0040
0041
0042
0043 if (!evsel->own_cpus ||
0044 (!evsel->system_wide && evlist->has_user_cpus) ||
0045 (!evsel->system_wide &&
0046 !evsel->requires_cpu &&
0047 perf_cpu_map__empty(evlist->user_requested_cpus))) {
0048 perf_cpu_map__put(evsel->cpus);
0049 evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus);
0050 } else if (evsel->cpus != evsel->own_cpus) {
0051 perf_cpu_map__put(evsel->cpus);
0052 evsel->cpus = perf_cpu_map__get(evsel->own_cpus);
0053 }
0054
0055 if (!evsel->system_wide) {
0056 perf_thread_map__put(evsel->threads);
0057 evsel->threads = perf_thread_map__get(evlist->threads);
0058 }
0059
0060 evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus);
0061 }
0062
0063 static void perf_evlist__propagate_maps(struct perf_evlist *evlist)
0064 {
0065 struct perf_evsel *evsel;
0066
0067
0068 perf_cpu_map__put(evlist->all_cpus);
0069 evlist->all_cpus = NULL;
0070
0071 perf_evlist__for_each_evsel(evlist, evsel)
0072 __perf_evlist__propagate_maps(evlist, evsel);
0073 }
0074
0075 void perf_evlist__add(struct perf_evlist *evlist,
0076 struct perf_evsel *evsel)
0077 {
0078 evsel->idx = evlist->nr_entries;
0079 list_add_tail(&evsel->node, &evlist->entries);
0080 evlist->nr_entries += 1;
0081 __perf_evlist__propagate_maps(evlist, evsel);
0082 }
0083
0084 void perf_evlist__remove(struct perf_evlist *evlist,
0085 struct perf_evsel *evsel)
0086 {
0087 list_del_init(&evsel->node);
0088 evlist->nr_entries -= 1;
0089 }
0090
0091 struct perf_evlist *perf_evlist__new(void)
0092 {
0093 struct perf_evlist *evlist = zalloc(sizeof(*evlist));
0094
0095 if (evlist != NULL)
0096 perf_evlist__init(evlist);
0097
0098 return evlist;
0099 }
0100
0101 struct perf_evsel *
0102 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev)
0103 {
0104 struct perf_evsel *next;
0105
0106 if (!prev) {
0107 next = list_first_entry(&evlist->entries,
0108 struct perf_evsel,
0109 node);
0110 } else {
0111 next = list_next_entry(prev, node);
0112 }
0113
0114
0115 if (&next->node == &evlist->entries)
0116 return NULL;
0117
0118 return next;
0119 }
0120
0121 static void perf_evlist__purge(struct perf_evlist *evlist)
0122 {
0123 struct perf_evsel *pos, *n;
0124
0125 perf_evlist__for_each_entry_safe(evlist, n, pos) {
0126 list_del_init(&pos->node);
0127 perf_evsel__delete(pos);
0128 }
0129
0130 evlist->nr_entries = 0;
0131 }
0132
0133 void perf_evlist__exit(struct perf_evlist *evlist)
0134 {
0135 perf_cpu_map__put(evlist->user_requested_cpus);
0136 perf_cpu_map__put(evlist->all_cpus);
0137 perf_thread_map__put(evlist->threads);
0138 evlist->user_requested_cpus = NULL;
0139 evlist->all_cpus = NULL;
0140 evlist->threads = NULL;
0141 fdarray__exit(&evlist->pollfd);
0142 }
0143
0144 void perf_evlist__delete(struct perf_evlist *evlist)
0145 {
0146 if (evlist == NULL)
0147 return;
0148
0149 perf_evlist__munmap(evlist);
0150 perf_evlist__close(evlist);
0151 perf_evlist__purge(evlist);
0152 perf_evlist__exit(evlist);
0153 free(evlist);
0154 }
0155
0156 void perf_evlist__set_maps(struct perf_evlist *evlist,
0157 struct perf_cpu_map *cpus,
0158 struct perf_thread_map *threads)
0159 {
0160
0161
0162
0163
0164
0165
0166
0167 if (cpus != evlist->user_requested_cpus) {
0168 perf_cpu_map__put(evlist->user_requested_cpus);
0169 evlist->user_requested_cpus = perf_cpu_map__get(cpus);
0170 }
0171
0172 if (threads != evlist->threads) {
0173 perf_thread_map__put(evlist->threads);
0174 evlist->threads = perf_thread_map__get(threads);
0175 }
0176
0177 if (!evlist->all_cpus && cpus)
0178 evlist->all_cpus = perf_cpu_map__get(cpus);
0179
0180 perf_evlist__propagate_maps(evlist);
0181 }
0182
0183 int perf_evlist__open(struct perf_evlist *evlist)
0184 {
0185 struct perf_evsel *evsel;
0186 int err;
0187
0188 perf_evlist__for_each_entry(evlist, evsel) {
0189 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads);
0190 if (err < 0)
0191 goto out_err;
0192 }
0193
0194 return 0;
0195
0196 out_err:
0197 perf_evlist__close(evlist);
0198 return err;
0199 }
0200
0201 void perf_evlist__close(struct perf_evlist *evlist)
0202 {
0203 struct perf_evsel *evsel;
0204
0205 perf_evlist__for_each_entry_reverse(evlist, evsel)
0206 perf_evsel__close(evsel);
0207 }
0208
0209 void perf_evlist__enable(struct perf_evlist *evlist)
0210 {
0211 struct perf_evsel *evsel;
0212
0213 perf_evlist__for_each_entry(evlist, evsel)
0214 perf_evsel__enable(evsel);
0215 }
0216
0217 void perf_evlist__disable(struct perf_evlist *evlist)
0218 {
0219 struct perf_evsel *evsel;
0220
0221 perf_evlist__for_each_entry(evlist, evsel)
0222 perf_evsel__disable(evsel);
0223 }
0224
0225 u64 perf_evlist__read_format(struct perf_evlist *evlist)
0226 {
0227 struct perf_evsel *first = perf_evlist__first(evlist);
0228
0229 return first->attr.read_format;
0230 }
0231
0232 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
0233
0234 static void perf_evlist__id_hash(struct perf_evlist *evlist,
0235 struct perf_evsel *evsel,
0236 int cpu, int thread, u64 id)
0237 {
0238 int hash;
0239 struct perf_sample_id *sid = SID(evsel, cpu, thread);
0240
0241 sid->id = id;
0242 sid->evsel = evsel;
0243 hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);
0244 hlist_add_head(&sid->node, &evlist->heads[hash]);
0245 }
0246
0247 void perf_evlist__reset_id_hash(struct perf_evlist *evlist)
0248 {
0249 int i;
0250
0251 for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i)
0252 INIT_HLIST_HEAD(&evlist->heads[i]);
0253 }
0254
0255 void perf_evlist__id_add(struct perf_evlist *evlist,
0256 struct perf_evsel *evsel,
0257 int cpu, int thread, u64 id)
0258 {
0259 perf_evlist__id_hash(evlist, evsel, cpu, thread, id);
0260 evsel->id[evsel->ids++] = id;
0261 }
0262
0263 int perf_evlist__id_add_fd(struct perf_evlist *evlist,
0264 struct perf_evsel *evsel,
0265 int cpu, int thread, int fd)
0266 {
0267 u64 read_data[4] = { 0, };
0268 int id_idx = 1;
0269 u64 id;
0270 int ret;
0271
0272 ret = ioctl(fd, PERF_EVENT_IOC_ID, &id);
0273 if (!ret)
0274 goto add;
0275
0276 if (errno != ENOTTY)
0277 return -1;
0278
0279
0280
0281
0282
0283
0284
0285 if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP)
0286 return -1;
0287
0288 if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||
0289 read(fd, &read_data, sizeof(read_data)) == -1)
0290 return -1;
0291
0292 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
0293 ++id_idx;
0294 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
0295 ++id_idx;
0296
0297 id = read_data[id_idx];
0298
0299 add:
0300 perf_evlist__id_add(evlist, evsel, cpu, thread, id);
0301 return 0;
0302 }
0303
0304 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist)
0305 {
0306 int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);
0307 int nr_threads = perf_thread_map__nr(evlist->threads);
0308 int nfds = 0;
0309 struct perf_evsel *evsel;
0310
0311 perf_evlist__for_each_entry(evlist, evsel) {
0312 if (evsel->system_wide)
0313 nfds += nr_cpus;
0314 else
0315 nfds += nr_cpus * nr_threads;
0316 }
0317
0318 if (fdarray__available_entries(&evlist->pollfd) < nfds &&
0319 fdarray__grow(&evlist->pollfd, nfds) < 0)
0320 return -ENOMEM;
0321
0322 return 0;
0323 }
0324
0325 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd,
0326 void *ptr, short revent, enum fdarray_flags flags)
0327 {
0328 int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags);
0329
0330 if (pos >= 0) {
0331 evlist->pollfd.priv[pos].ptr = ptr;
0332 fcntl(fd, F_SETFL, O_NONBLOCK);
0333 }
0334
0335 return pos;
0336 }
0337
0338 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd,
0339 void *arg __maybe_unused)
0340 {
0341 struct perf_mmap *map = fda->priv[fd].ptr;
0342
0343 if (map)
0344 perf_mmap__put(map);
0345 }
0346
0347 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask)
0348 {
0349 return fdarray__filter(&evlist->pollfd, revents_and_mask,
0350 perf_evlist__munmap_filtered, NULL);
0351 }
0352
0353 int perf_evlist__poll(struct perf_evlist *evlist, int timeout)
0354 {
0355 return fdarray__poll(&evlist->pollfd, timeout);
0356 }
0357
0358 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite)
0359 {
0360 int i;
0361 struct perf_mmap *map;
0362
0363 map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));
0364 if (!map)
0365 return NULL;
0366
0367 for (i = 0; i < evlist->nr_mmaps; i++) {
0368 struct perf_mmap *prev = i ? &map[i - 1] : NULL;
0369
0370
0371
0372
0373
0374
0375
0376
0377
0378
0379 perf_mmap__init(&map[i], prev, overwrite, NULL);
0380 }
0381
0382 return map;
0383 }
0384
0385 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread)
0386 {
0387 struct perf_sample_id *sid = SID(evsel, cpu, thread);
0388
0389 sid->idx = idx;
0390 sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu);
0391 sid->tid = perf_thread_map__pid(evsel->threads, thread);
0392 }
0393
0394 static struct perf_mmap*
0395 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx)
0396 {
0397 struct perf_mmap *maps;
0398
0399 maps = overwrite ? evlist->mmap_ovw : evlist->mmap;
0400
0401 if (!maps) {
0402 maps = perf_evlist__alloc_mmap(evlist, overwrite);
0403 if (!maps)
0404 return NULL;
0405
0406 if (overwrite)
0407 evlist->mmap_ovw = maps;
0408 else
0409 evlist->mmap = maps;
0410 }
0411
0412 return &maps[idx];
0413 }
0414
0415 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y))
0416
0417 static int
0418 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp,
0419 int output, struct perf_cpu cpu)
0420 {
0421 return perf_mmap__mmap(map, mp, output, cpu);
0422 }
0423
0424 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map,
0425 bool overwrite)
0426 {
0427 if (overwrite)
0428 evlist->mmap_ovw_first = map;
0429 else
0430 evlist->mmap_first = map;
0431 }
0432
0433 static int
0434 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
0435 int idx, struct perf_mmap_param *mp, int cpu_idx,
0436 int thread, int *_output, int *_output_overwrite, int *nr_mmaps)
0437 {
0438 struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->all_cpus, cpu_idx);
0439 struct perf_evsel *evsel;
0440 int revent;
0441
0442 perf_evlist__for_each_entry(evlist, evsel) {
0443 bool overwrite = evsel->attr.write_backward;
0444 enum fdarray_flags flgs;
0445 struct perf_mmap *map;
0446 int *output, fd, cpu;
0447
0448 if (evsel->system_wide && thread)
0449 continue;
0450
0451 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu);
0452 if (cpu == -1)
0453 continue;
0454
0455 map = ops->get(evlist, overwrite, idx);
0456 if (map == NULL)
0457 return -ENOMEM;
0458
0459 if (overwrite) {
0460 mp->prot = PROT_READ;
0461 output = _output_overwrite;
0462 } else {
0463 mp->prot = PROT_READ | PROT_WRITE;
0464 output = _output;
0465 }
0466
0467 fd = FD(evsel, cpu, thread);
0468
0469 if (*output == -1) {
0470 *output = fd;
0471
0472
0473
0474
0475
0476
0477
0478
0479
0480
0481
0482
0483
0484
0485 refcount_set(&map->refcnt, 2);
0486
0487 if (ops->idx)
0488 ops->idx(evlist, evsel, mp, idx);
0489
0490 pr_debug("idx %d: mmapping fd %d\n", idx, *output);
0491 if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
0492 return -1;
0493
0494 *nr_mmaps += 1;
0495
0496 if (!idx)
0497 perf_evlist__set_mmap_first(evlist, map, overwrite);
0498 } else {
0499 pr_debug("idx %d: set output fd %d -> %d\n", idx, fd, *output);
0500 if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
0501 return -1;
0502
0503 perf_mmap__get(map);
0504 }
0505
0506 revent = !overwrite ? POLLIN : 0;
0507
0508 flgs = evsel->system_wide ? fdarray_flag__nonfilterable : fdarray_flag__default;
0509 if (perf_evlist__add_pollfd(evlist, fd, map, revent, flgs) < 0) {
0510 perf_mmap__put(map);
0511 return -1;
0512 }
0513
0514 if (evsel->attr.read_format & PERF_FORMAT_ID) {
0515 if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,
0516 fd) < 0)
0517 return -1;
0518 perf_evsel__set_sid_idx(evsel, idx, cpu, thread);
0519 }
0520 }
0521
0522 return 0;
0523 }
0524
0525 static int
0526 mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
0527 struct perf_mmap_param *mp)
0528 {
0529 int nr_threads = perf_thread_map__nr(evlist->threads);
0530 int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);
0531 int cpu, thread, idx = 0;
0532 int nr_mmaps = 0;
0533
0534 pr_debug("%s: nr cpu values (may include -1) %d nr threads %d\n",
0535 __func__, nr_cpus, nr_threads);
0536
0537
0538 for (thread = 0; thread < nr_threads; thread++, idx++) {
0539 int output = -1;
0540 int output_overwrite = -1;
0541
0542 if (mmap_per_evsel(evlist, ops, idx, mp, 0, thread, &output,
0543 &output_overwrite, &nr_mmaps))
0544 goto out_unmap;
0545 }
0546
0547
0548 for (cpu = 1; cpu < nr_cpus; cpu++, idx++) {
0549 int output = -1;
0550 int output_overwrite = -1;
0551
0552 if (mmap_per_evsel(evlist, ops, idx, mp, cpu, 0, &output,
0553 &output_overwrite, &nr_mmaps))
0554 goto out_unmap;
0555 }
0556
0557 if (nr_mmaps != evlist->nr_mmaps)
0558 pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);
0559
0560 return 0;
0561
0562 out_unmap:
0563 perf_evlist__munmap(evlist);
0564 return -1;
0565 }
0566
0567 static int
0568 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
0569 struct perf_mmap_param *mp)
0570 {
0571 int nr_threads = perf_thread_map__nr(evlist->threads);
0572 int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);
0573 int nr_mmaps = 0;
0574 int cpu, thread;
0575
0576 pr_debug("%s: nr cpu values %d nr threads %d\n", __func__, nr_cpus, nr_threads);
0577
0578 for (cpu = 0; cpu < nr_cpus; cpu++) {
0579 int output = -1;
0580 int output_overwrite = -1;
0581
0582 for (thread = 0; thread < nr_threads; thread++) {
0583 if (mmap_per_evsel(evlist, ops, cpu, mp, cpu,
0584 thread, &output, &output_overwrite, &nr_mmaps))
0585 goto out_unmap;
0586 }
0587 }
0588
0589 if (nr_mmaps != evlist->nr_mmaps)
0590 pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);
0591
0592 return 0;
0593
0594 out_unmap:
0595 perf_evlist__munmap(evlist);
0596 return -1;
0597 }
0598
0599 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist)
0600 {
0601 int nr_mmaps;
0602
0603
0604 nr_mmaps = perf_cpu_map__nr(evlist->all_cpus);
0605 if (perf_cpu_map__empty(evlist->all_cpus)) {
0606
0607 nr_mmaps += perf_thread_map__nr(evlist->threads);
0608
0609 nr_mmaps -= 1;
0610 }
0611
0612 return nr_mmaps;
0613 }
0614
0615 int perf_evlist__mmap_ops(struct perf_evlist *evlist,
0616 struct perf_evlist_mmap_ops *ops,
0617 struct perf_mmap_param *mp)
0618 {
0619 const struct perf_cpu_map *cpus = evlist->all_cpus;
0620 struct perf_evsel *evsel;
0621
0622 if (!ops || !ops->get || !ops->mmap)
0623 return -EINVAL;
0624
0625 mp->mask = evlist->mmap_len - page_size - 1;
0626
0627 evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist);
0628
0629 perf_evlist__for_each_entry(evlist, evsel) {
0630 if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
0631 evsel->sample_id == NULL &&
0632 perf_evsel__alloc_id(evsel, evsel->fd->max_x, evsel->fd->max_y) < 0)
0633 return -ENOMEM;
0634 }
0635
0636 if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
0637 return -ENOMEM;
0638
0639 if (perf_cpu_map__empty(cpus))
0640 return mmap_per_thread(evlist, ops, mp);
0641
0642 return mmap_per_cpu(evlist, ops, mp);
0643 }
0644
0645 int perf_evlist__mmap(struct perf_evlist *evlist, int pages)
0646 {
0647 struct perf_mmap_param mp;
0648 struct perf_evlist_mmap_ops ops = {
0649 .get = perf_evlist__mmap_cb_get,
0650 .mmap = perf_evlist__mmap_cb_mmap,
0651 };
0652
0653 evlist->mmap_len = (pages + 1) * page_size;
0654
0655 return perf_evlist__mmap_ops(evlist, &ops, &mp);
0656 }
0657
0658 void perf_evlist__munmap(struct perf_evlist *evlist)
0659 {
0660 int i;
0661
0662 if (evlist->mmap) {
0663 for (i = 0; i < evlist->nr_mmaps; i++)
0664 perf_mmap__munmap(&evlist->mmap[i]);
0665 }
0666
0667 if (evlist->mmap_ovw) {
0668 for (i = 0; i < evlist->nr_mmaps; i++)
0669 perf_mmap__munmap(&evlist->mmap_ovw[i]);
0670 }
0671
0672 zfree(&evlist->mmap);
0673 zfree(&evlist->mmap_ovw);
0674 }
0675
0676 struct perf_mmap*
0677 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map,
0678 bool overwrite)
0679 {
0680 if (map)
0681 return map->next;
0682
0683 return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first;
0684 }
0685
0686 void __perf_evlist__set_leader(struct list_head *list, struct perf_evsel *leader)
0687 {
0688 struct perf_evsel *first, *last, *evsel;
0689
0690 first = list_first_entry(list, struct perf_evsel, node);
0691 last = list_last_entry(list, struct perf_evsel, node);
0692
0693 leader->nr_members = last->idx - first->idx + 1;
0694
0695 __perf_evlist__for_each_entry(list, evsel)
0696 evsel->leader = leader;
0697 }
0698
0699 void perf_evlist__set_leader(struct perf_evlist *evlist)
0700 {
0701 if (evlist->nr_entries) {
0702 struct perf_evsel *first = list_entry(evlist->entries.next,
0703 struct perf_evsel, node);
0704
0705 evlist->nr_groups = evlist->nr_entries > 1 ? 1 : 0;
0706 __perf_evlist__set_leader(&evlist->entries, first);
0707 }
0708 }