0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024 #include <sys/types.h>
0025 #include <sys/mman.h>
0026 #include <sys/stat.h>
0027 #include <getopt.h>
0028 #include <elf.h>
0029 #include <fcntl.h>
0030 #include <stdio.h>
0031 #include <stdlib.h>
0032 #include <string.h>
0033 #include <unistd.h>
0034
0035 #ifndef EM_AARCH64
0036 #define EM_AARCH64 183
0037 #define R_AARCH64_NONE 0
0038 #define R_AARCH64_ABS64 257
0039 #endif
0040
0041 #define R_ARM_PC24 1
0042 #define R_ARM_THM_CALL 10
0043 #define R_ARM_CALL 28
0044
0045 #define R_AARCH64_CALL26 283
0046
0047 static int fd_map;
0048 static int mmap_failed;
0049 static char gpfx;
0050 static struct stat sb;
0051 static const char *altmcount;
0052 static int warn_on_notrace_sect;
0053 static void *file_map;
0054 static void *file_end;
0055 static int file_updated;
0056 static void *file_ptr;
0057
0058 static void *file_append;
0059 static size_t file_append_size;
0060
0061
0062 static void file_append_cleanup(void)
0063 {
0064 free(file_append);
0065 file_append = NULL;
0066 file_append_size = 0;
0067 file_updated = 0;
0068 }
0069
0070 static void mmap_cleanup(void)
0071 {
0072 if (!mmap_failed)
0073 munmap(file_map, sb.st_size);
0074 else
0075 free(file_map);
0076 file_map = NULL;
0077 }
0078
0079
0080
0081 static off_t ulseek(off_t const offset, int const whence)
0082 {
0083 switch (whence) {
0084 case SEEK_SET:
0085 file_ptr = file_map + offset;
0086 break;
0087 case SEEK_CUR:
0088 file_ptr += offset;
0089 break;
0090 case SEEK_END:
0091 file_ptr = file_map + (sb.st_size - offset);
0092 break;
0093 }
0094 if (file_ptr < file_map) {
0095 fprintf(stderr, "lseek: seek before file\n");
0096 return -1;
0097 }
0098 return file_ptr - file_map;
0099 }
0100
0101 static ssize_t uwrite(void const *const buf, size_t const count)
0102 {
0103 size_t cnt = count;
0104 off_t idx = 0;
0105
0106 file_updated = 1;
0107
0108 if (file_ptr + count >= file_end) {
0109 off_t aoffset = (file_ptr + count) - file_end;
0110
0111 if (aoffset > file_append_size) {
0112 file_append = realloc(file_append, aoffset);
0113 file_append_size = aoffset;
0114 }
0115 if (!file_append) {
0116 perror("write");
0117 file_append_cleanup();
0118 mmap_cleanup();
0119 return -1;
0120 }
0121 if (file_ptr < file_end) {
0122 cnt = file_end - file_ptr;
0123 } else {
0124 cnt = 0;
0125 idx = aoffset - count;
0126 }
0127 }
0128
0129 if (cnt)
0130 memcpy(file_ptr, buf, cnt);
0131
0132 if (cnt < count)
0133 memcpy(file_append + idx, buf + cnt, count - cnt);
0134
0135 file_ptr += count;
0136 return count;
0137 }
0138
0139 static void * umalloc(size_t size)
0140 {
0141 void *const addr = malloc(size);
0142 if (addr == 0) {
0143 fprintf(stderr, "malloc failed: %zu bytes\n", size);
0144 file_append_cleanup();
0145 mmap_cleanup();
0146 return NULL;
0147 }
0148 return addr;
0149 }
0150
0151
0152
0153
0154
0155
0156
0157
0158
0159
0160
0161
0162
0163
0164 static void *mmap_file(char const *fname)
0165 {
0166
0167 fd_map = -1;
0168 mmap_failed = 1;
0169 file_map = NULL;
0170 file_ptr = NULL;
0171 file_updated = 0;
0172 sb.st_size = 0;
0173
0174 fd_map = open(fname, O_RDONLY);
0175 if (fd_map < 0) {
0176 perror(fname);
0177 return NULL;
0178 }
0179 if (fstat(fd_map, &sb) < 0) {
0180 perror(fname);
0181 goto out;
0182 }
0183 if (!S_ISREG(sb.st_mode)) {
0184 fprintf(stderr, "not a regular file: %s\n", fname);
0185 goto out;
0186 }
0187 file_map = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE,
0188 fd_map, 0);
0189 if (file_map == MAP_FAILED) {
0190 mmap_failed = 1;
0191 file_map = umalloc(sb.st_size);
0192 if (!file_map) {
0193 perror(fname);
0194 goto out;
0195 }
0196 if (read(fd_map, file_map, sb.st_size) != sb.st_size) {
0197 perror(fname);
0198 free(file_map);
0199 file_map = NULL;
0200 goto out;
0201 }
0202 } else
0203 mmap_failed = 0;
0204 out:
0205 close(fd_map);
0206 fd_map = -1;
0207
0208 file_end = file_map + sb.st_size;
0209
0210 return file_map;
0211 }
0212
0213
0214 static unsigned char ideal_nop5_x86_64[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
0215 static unsigned char ideal_nop5_x86_32[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
0216 static unsigned char *ideal_nop;
0217
0218 static char rel_type_nop;
0219
0220 static int (*make_nop)(void *map, size_t const offset);
0221
0222 static int make_nop_x86(void *map, size_t const offset)
0223 {
0224 uint32_t *ptr;
0225 unsigned char *op;
0226
0227
0228 ptr = map + offset;
0229 if (*ptr != 0)
0230 return -1;
0231
0232 op = map + offset - 1;
0233 if (*op != 0xe8)
0234 return -1;
0235
0236
0237 if (ulseek(offset - 1, SEEK_SET) < 0)
0238 return -1;
0239 if (uwrite(ideal_nop, 5) < 0)
0240 return -1;
0241 return 0;
0242 }
0243
0244 static unsigned char ideal_nop4_arm_le[4] = { 0x00, 0x00, 0xa0, 0xe1 };
0245 static unsigned char ideal_nop4_arm_be[4] = { 0xe1, 0xa0, 0x00, 0x00 };
0246 static unsigned char *ideal_nop4_arm;
0247
0248 static unsigned char bl_mcount_arm_le[4] = { 0xfe, 0xff, 0xff, 0xeb };
0249 static unsigned char bl_mcount_arm_be[4] = { 0xeb, 0xff, 0xff, 0xfe };
0250 static unsigned char *bl_mcount_arm;
0251
0252 static unsigned char push_arm_le[4] = { 0x04, 0xe0, 0x2d, 0xe5 };
0253 static unsigned char push_arm_be[4] = { 0xe5, 0x2d, 0xe0, 0x04 };
0254 static unsigned char *push_arm;
0255
0256 static unsigned char ideal_nop2_thumb_le[2] = { 0x00, 0xbf };
0257 static unsigned char ideal_nop2_thumb_be[2] = { 0xbf, 0x00 };
0258 static unsigned char *ideal_nop2_thumb;
0259
0260 static unsigned char push_bl_mcount_thumb_le[6] = { 0x00, 0xb5, 0xff, 0xf7, 0xfe, 0xff };
0261 static unsigned char push_bl_mcount_thumb_be[6] = { 0xb5, 0x00, 0xf7, 0xff, 0xff, 0xfe };
0262 static unsigned char *push_bl_mcount_thumb;
0263
0264 static int make_nop_arm(void *map, size_t const offset)
0265 {
0266 char *ptr;
0267 int cnt = 1;
0268 int nop_size;
0269 size_t off = offset;
0270
0271 ptr = map + offset;
0272 if (memcmp(ptr, bl_mcount_arm, 4) == 0) {
0273 if (memcmp(ptr - 4, push_arm, 4) == 0) {
0274 off -= 4;
0275 cnt = 2;
0276 }
0277 ideal_nop = ideal_nop4_arm;
0278 nop_size = 4;
0279 } else if (memcmp(ptr - 2, push_bl_mcount_thumb, 6) == 0) {
0280 cnt = 3;
0281 nop_size = 2;
0282 off -= 2;
0283 ideal_nop = ideal_nop2_thumb;
0284 } else
0285 return -1;
0286
0287
0288 if (ulseek(off, SEEK_SET) < 0)
0289 return -1;
0290
0291 do {
0292 if (uwrite(ideal_nop, nop_size) < 0)
0293 return -1;
0294 } while (--cnt > 0);
0295
0296 return 0;
0297 }
0298
0299 static unsigned char ideal_nop4_arm64[4] = {0x1f, 0x20, 0x03, 0xd5};
0300 static int make_nop_arm64(void *map, size_t const offset)
0301 {
0302 uint32_t *ptr;
0303
0304 ptr = map + offset;
0305
0306 if (*ptr != 0x94000000)
0307 return -1;
0308
0309
0310 if (ulseek(offset, SEEK_SET) < 0)
0311 return -1;
0312 if (uwrite(ideal_nop, 4) < 0)
0313 return -1;
0314 return 0;
0315 }
0316
0317 static int write_file(const char *fname)
0318 {
0319 char tmp_file[strlen(fname) + 4];
0320 size_t n;
0321
0322 if (!file_updated)
0323 return 0;
0324
0325 sprintf(tmp_file, "%s.rc", fname);
0326
0327
0328
0329
0330
0331
0332 fd_map = open(tmp_file, O_WRONLY | O_TRUNC | O_CREAT, sb.st_mode);
0333 if (fd_map < 0) {
0334 perror(fname);
0335 return -1;
0336 }
0337 n = write(fd_map, file_map, sb.st_size);
0338 if (n != sb.st_size) {
0339 perror("write");
0340 close(fd_map);
0341 return -1;
0342 }
0343 if (file_append_size) {
0344 n = write(fd_map, file_append, file_append_size);
0345 if (n != file_append_size) {
0346 perror("write");
0347 close(fd_map);
0348 return -1;
0349 }
0350 }
0351 close(fd_map);
0352 if (rename(tmp_file, fname) < 0) {
0353 perror(fname);
0354 return -1;
0355 }
0356 return 0;
0357 }
0358
0359
0360
0361 static uint64_t w8rev(uint64_t const x)
0362 {
0363 return ((0xff & (x >> (0 * 8))) << (7 * 8))
0364 | ((0xff & (x >> (1 * 8))) << (6 * 8))
0365 | ((0xff & (x >> (2 * 8))) << (5 * 8))
0366 | ((0xff & (x >> (3 * 8))) << (4 * 8))
0367 | ((0xff & (x >> (4 * 8))) << (3 * 8))
0368 | ((0xff & (x >> (5 * 8))) << (2 * 8))
0369 | ((0xff & (x >> (6 * 8))) << (1 * 8))
0370 | ((0xff & (x >> (7 * 8))) << (0 * 8));
0371 }
0372
0373 static uint32_t w4rev(uint32_t const x)
0374 {
0375 return ((0xff & (x >> (0 * 8))) << (3 * 8))
0376 | ((0xff & (x >> (1 * 8))) << (2 * 8))
0377 | ((0xff & (x >> (2 * 8))) << (1 * 8))
0378 | ((0xff & (x >> (3 * 8))) << (0 * 8));
0379 }
0380
0381 static uint32_t w2rev(uint16_t const x)
0382 {
0383 return ((0xff & (x >> (0 * 8))) << (1 * 8))
0384 | ((0xff & (x >> (1 * 8))) << (0 * 8));
0385 }
0386
0387 static uint64_t w8nat(uint64_t const x)
0388 {
0389 return x;
0390 }
0391
0392 static uint32_t w4nat(uint32_t const x)
0393 {
0394 return x;
0395 }
0396
0397 static uint32_t w2nat(uint16_t const x)
0398 {
0399 return x;
0400 }
0401
0402 static uint64_t (*w8)(uint64_t);
0403 static uint32_t (*w)(uint32_t);
0404 static uint32_t (*w2)(uint16_t);
0405
0406
0407 static int is_mcounted_section_name(char const *const txtname)
0408 {
0409 return strncmp(".text", txtname, 5) == 0 ||
0410 strcmp(".init.text", txtname) == 0 ||
0411 strcmp(".ref.text", txtname) == 0 ||
0412 strcmp(".sched.text", txtname) == 0 ||
0413 strcmp(".spinlock.text", txtname) == 0 ||
0414 strcmp(".irqentry.text", txtname) == 0 ||
0415 strcmp(".softirqentry.text", txtname) == 0 ||
0416 strcmp(".kprobes.text", txtname) == 0 ||
0417 strcmp(".cpuidle.text", txtname) == 0;
0418 }
0419
0420 static char const *already_has_rel_mcount = "success";
0421
0422
0423 #include "recordmcount.h"
0424 #define RECORD_MCOUNT_64
0425 #include "recordmcount.h"
0426
0427 static int arm_is_fake_mcount(Elf32_Rel const *rp)
0428 {
0429 switch (ELF32_R_TYPE(w(rp->r_info))) {
0430 case R_ARM_THM_CALL:
0431 case R_ARM_CALL:
0432 case R_ARM_PC24:
0433 return 0;
0434 }
0435
0436 return 1;
0437 }
0438
0439 static int arm64_is_fake_mcount(Elf64_Rel const *rp)
0440 {
0441 return ELF64_R_TYPE(w8(rp->r_info)) != R_AARCH64_CALL26;
0442 }
0443
0444
0445
0446
0447
0448
0449
0450
0451
0452 typedef uint8_t myElf64_Byte;
0453
0454 union mips_r_info {
0455 Elf64_Xword r_info;
0456 struct {
0457 Elf64_Word r_sym;
0458 myElf64_Byte r_ssym;
0459 myElf64_Byte r_type3;
0460 myElf64_Byte r_type2;
0461 myElf64_Byte r_type;
0462 } r_mips;
0463 };
0464
0465 static uint64_t MIPS64_r_sym(Elf64_Rel const *rp)
0466 {
0467 return w(((union mips_r_info){ .r_info = rp->r_info }).r_mips.r_sym);
0468 }
0469
0470 static void MIPS64_r_info(Elf64_Rel *const rp, unsigned sym, unsigned type)
0471 {
0472 rp->r_info = ((union mips_r_info){
0473 .r_mips = { .r_sym = w(sym), .r_type = type }
0474 }).r_info;
0475 }
0476
0477 static int do_file(char const *const fname)
0478 {
0479 unsigned int reltype = 0;
0480 Elf32_Ehdr *ehdr;
0481 int rc = -1;
0482
0483 ehdr = mmap_file(fname);
0484 if (!ehdr)
0485 goto out;
0486
0487 w = w4nat;
0488 w2 = w2nat;
0489 w8 = w8nat;
0490 switch (ehdr->e_ident[EI_DATA]) {
0491 static unsigned int const endian = 1;
0492 default:
0493 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
0494 ehdr->e_ident[EI_DATA], fname);
0495 goto out;
0496 case ELFDATA2LSB:
0497 if (*(unsigned char const *)&endian != 1) {
0498
0499 w = w4rev;
0500 w2 = w2rev;
0501 w8 = w8rev;
0502 }
0503 ideal_nop4_arm = ideal_nop4_arm_le;
0504 bl_mcount_arm = bl_mcount_arm_le;
0505 push_arm = push_arm_le;
0506 ideal_nop2_thumb = ideal_nop2_thumb_le;
0507 push_bl_mcount_thumb = push_bl_mcount_thumb_le;
0508 break;
0509 case ELFDATA2MSB:
0510 if (*(unsigned char const *)&endian != 0) {
0511
0512 w = w4rev;
0513 w2 = w2rev;
0514 w8 = w8rev;
0515 }
0516 ideal_nop4_arm = ideal_nop4_arm_be;
0517 bl_mcount_arm = bl_mcount_arm_be;
0518 push_arm = push_arm_be;
0519 ideal_nop2_thumb = ideal_nop2_thumb_be;
0520 push_bl_mcount_thumb = push_bl_mcount_thumb_be;
0521 break;
0522 }
0523 if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0 ||
0524 w2(ehdr->e_type) != ET_REL ||
0525 ehdr->e_ident[EI_VERSION] != EV_CURRENT) {
0526 fprintf(stderr, "unrecognized ET_REL file %s\n", fname);
0527 goto out;
0528 }
0529
0530 gpfx = '_';
0531 switch (w2(ehdr->e_machine)) {
0532 default:
0533 fprintf(stderr, "unrecognized e_machine %u %s\n",
0534 w2(ehdr->e_machine), fname);
0535 goto out;
0536 case EM_386:
0537 reltype = R_386_32;
0538 rel_type_nop = R_386_NONE;
0539 make_nop = make_nop_x86;
0540 ideal_nop = ideal_nop5_x86_32;
0541 mcount_adjust_32 = -1;
0542 gpfx = 0;
0543 break;
0544 case EM_ARM:
0545 reltype = R_ARM_ABS32;
0546 altmcount = "__gnu_mcount_nc";
0547 make_nop = make_nop_arm;
0548 rel_type_nop = R_ARM_NONE;
0549 is_fake_mcount32 = arm_is_fake_mcount;
0550 gpfx = 0;
0551 break;
0552 case EM_AARCH64:
0553 reltype = R_AARCH64_ABS64;
0554 make_nop = make_nop_arm64;
0555 rel_type_nop = R_AARCH64_NONE;
0556 ideal_nop = ideal_nop4_arm64;
0557 is_fake_mcount64 = arm64_is_fake_mcount;
0558 break;
0559 case EM_IA_64: reltype = R_IA64_IMM64; break;
0560 case EM_MIPS: break;
0561 case EM_PPC: reltype = R_PPC_ADDR32; break;
0562 case EM_PPC64: reltype = R_PPC64_ADDR64; break;
0563 case EM_S390: break;
0564 case EM_SH: reltype = R_SH_DIR32; gpfx = 0; break;
0565 case EM_SPARCV9: reltype = R_SPARC_64; break;
0566 case EM_X86_64:
0567 make_nop = make_nop_x86;
0568 ideal_nop = ideal_nop5_x86_64;
0569 reltype = R_X86_64_64;
0570 rel_type_nop = R_X86_64_NONE;
0571 mcount_adjust_64 = -1;
0572 gpfx = 0;
0573 break;
0574 }
0575
0576 switch (ehdr->e_ident[EI_CLASS]) {
0577 default:
0578 fprintf(stderr, "unrecognized ELF class %d %s\n",
0579 ehdr->e_ident[EI_CLASS], fname);
0580 goto out;
0581 case ELFCLASS32:
0582 if (w2(ehdr->e_ehsize) != sizeof(Elf32_Ehdr)
0583 || w2(ehdr->e_shentsize) != sizeof(Elf32_Shdr)) {
0584 fprintf(stderr,
0585 "unrecognized ET_REL file: %s\n", fname);
0586 goto out;
0587 }
0588 if (w2(ehdr->e_machine) == EM_MIPS) {
0589 reltype = R_MIPS_32;
0590 is_fake_mcount32 = MIPS32_is_fake_mcount;
0591 }
0592 if (do32(ehdr, fname, reltype) < 0)
0593 goto out;
0594 break;
0595 case ELFCLASS64: {
0596 Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr;
0597 if (w2(ghdr->e_ehsize) != sizeof(Elf64_Ehdr)
0598 || w2(ghdr->e_shentsize) != sizeof(Elf64_Shdr)) {
0599 fprintf(stderr,
0600 "unrecognized ET_REL file: %s\n", fname);
0601 goto out;
0602 }
0603 if (w2(ghdr->e_machine) == EM_S390) {
0604 reltype = R_390_64;
0605 mcount_adjust_64 = -14;
0606 }
0607 if (w2(ghdr->e_machine) == EM_MIPS) {
0608 reltype = R_MIPS_64;
0609 Elf64_r_sym = MIPS64_r_sym;
0610 Elf64_r_info = MIPS64_r_info;
0611 is_fake_mcount64 = MIPS64_is_fake_mcount;
0612 }
0613 if (do64(ghdr, fname, reltype) < 0)
0614 goto out;
0615 break;
0616 }
0617 }
0618
0619 rc = write_file(fname);
0620 out:
0621 file_append_cleanup();
0622 mmap_cleanup();
0623 return rc;
0624 }
0625
0626 int main(int argc, char *argv[])
0627 {
0628 const char ftrace[] = "/ftrace.o";
0629 int ftrace_size = sizeof(ftrace) - 1;
0630 int n_error = 0;
0631 int c;
0632 int i;
0633
0634 while ((c = getopt(argc, argv, "w")) >= 0) {
0635 switch (c) {
0636 case 'w':
0637 warn_on_notrace_sect = 1;
0638 break;
0639 default:
0640 fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
0641 return 0;
0642 }
0643 }
0644
0645 if ((argc - optind) < 1) {
0646 fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
0647 return 0;
0648 }
0649
0650
0651 for (i = optind; i < argc; i++) {
0652 char *file = argv[i];
0653 int len;
0654
0655
0656
0657
0658
0659
0660 len = strlen(file);
0661 if (len >= ftrace_size &&
0662 strcmp(file + (len - ftrace_size), ftrace) == 0)
0663 continue;
0664
0665 if (do_file(file)) {
0666 fprintf(stderr, "%s: failed\n", file);
0667 ++n_error;
0668 }
0669 }
0670 return !!n_error;
0671 }