util/intel-pt-decoder/intel-pt-decoder.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * intel_pt_decoder.c: Intel Processor Trace support
0004  * Copyright (c) 2013-2014, Intel Corporation.
0005  */
0006
0007 #ifndef _GNU_SOURCE
0008 #define _GNU_SOURCE
0009 #endif
0010 #include <stdlib.h>
0011 #include <stdbool.h>
0012 #include <string.h>
0013 #include <errno.h>
0014 #include <stdint.h>
0015 #include <inttypes.h>
0016 #include <linux/compiler.h>
0017 #include <linux/string.h>
0018 #include <linux/zalloc.h>
0019
0020 #include "../auxtrace.h"
0021
0022 #include "intel-pt-insn-decoder.h"
0023 #include "intel-pt-pkt-decoder.h"
0024 #include "intel-pt-decoder.h"
0025 #include "intel-pt-log.h"
0026
0027 #define BITULL(x) (1ULL << (x))
0028
0029 /* IA32_RTIT_CTL MSR bits */
0030 #define INTEL_PT_CYC_ENABLE     BITULL(1)
0031 #define INTEL_PT_CYC_THRESHOLD      (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
0032 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
0033
0034 #define INTEL_PT_BLK_SIZE 1024
0035
0036 #define BIT63 (((uint64_t)1 << 63))
0037
0038 #define SEVEN_BYTES 0xffffffffffffffULL
0039
0040 #define NO_VMCS 0xffffffffffULL
0041
0042 #define INTEL_PT_RETURN 1
0043
0044 /*
0045  * Default maximum number of loops with no packets consumed i.e. stuck in a
0046  * loop.
0047  */
0048 #define INTEL_PT_MAX_LOOPS 100000
0049
0050 struct intel_pt_blk {
0051     struct intel_pt_blk *prev;
0052     uint64_t ip[INTEL_PT_BLK_SIZE];
0053 };
0054
0055 struct intel_pt_stack {
0056     struct intel_pt_blk *blk;
0057     struct intel_pt_blk *spare;
0058     int pos;
0059 };
0060
0061 enum intel_pt_p_once {
0062     INTEL_PT_PRT_ONCE_UNK_VMCS,
0063     INTEL_PT_PRT_ONCE_ERANGE,
0064 };
0065
0066 enum intel_pt_pkt_state {
0067     INTEL_PT_STATE_NO_PSB,
0068     INTEL_PT_STATE_NO_IP,
0069     INTEL_PT_STATE_ERR_RESYNC,
0070     INTEL_PT_STATE_IN_SYNC,
0071     INTEL_PT_STATE_TNT_CONT,
0072     INTEL_PT_STATE_TNT,
0073     INTEL_PT_STATE_TIP,
0074     INTEL_PT_STATE_TIP_PGD,
0075     INTEL_PT_STATE_FUP,
0076     INTEL_PT_STATE_FUP_NO_TIP,
0077     INTEL_PT_STATE_FUP_IN_PSB,
0078     INTEL_PT_STATE_RESAMPLE,
0079     INTEL_PT_STATE_VM_TIME_CORRELATION,
0080 };
0081
0082 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
0083 {
0084     switch (pkt_state) {
0085     case INTEL_PT_STATE_NO_PSB:
0086     case INTEL_PT_STATE_NO_IP:
0087     case INTEL_PT_STATE_ERR_RESYNC:
0088     case INTEL_PT_STATE_IN_SYNC:
0089     case INTEL_PT_STATE_TNT_CONT:
0090     case INTEL_PT_STATE_RESAMPLE:
0091     case INTEL_PT_STATE_VM_TIME_CORRELATION:
0092         return true;
0093     case INTEL_PT_STATE_TNT:
0094     case INTEL_PT_STATE_TIP:
0095     case INTEL_PT_STATE_TIP_PGD:
0096     case INTEL_PT_STATE_FUP:
0097     case INTEL_PT_STATE_FUP_NO_TIP:
0098     case INTEL_PT_STATE_FUP_IN_PSB:
0099         return false;
0100     default:
0101         return true;
0102     };
0103 }
0104
0105 #ifdef INTEL_PT_STRICT
0106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
0107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
0108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
0109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
0110 #else
0111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
0112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
0113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
0114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
0115 #endif
0116
0117 struct intel_pt_decoder {
0118     int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
0119     int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
0120              uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
0121              uint64_t max_insn_cnt, void *data);
0122     bool (*pgd_ip)(uint64_t ip, void *data);
0123     int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
0124     struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
0125     void *data;
0126     struct intel_pt_state state;
0127     const unsigned char *buf;
0128     size_t len;
0129     bool return_compression;
0130     bool branch_enable;
0131     bool mtc_insn;
0132     bool pge;
0133     bool have_tma;
0134     bool have_cyc;
0135     bool fixup_last_mtc;
0136     bool have_last_ip;
0137     bool in_psb;
0138     bool hop;
0139     bool leap;
0140     bool emulated_ptwrite;
0141     bool vm_time_correlation;
0142     bool vm_tm_corr_dry_run;
0143     bool vm_tm_corr_reliable;
0144     bool vm_tm_corr_same_buf;
0145     bool vm_tm_corr_continuous;
0146     bool nr;
0147     bool next_nr;
0148     bool iflag;
0149     bool next_iflag;
0150     enum intel_pt_param_flags flags;
0151     uint64_t pos;
0152     uint64_t last_ip;
0153     uint64_t ip;
0154     uint64_t pip_payload;
0155     uint64_t timestamp;
0156     uint64_t tsc_timestamp;
0157     uint64_t ref_timestamp;
0158     uint64_t buf_timestamp;
0159     uint64_t sample_timestamp;
0160     uint64_t ret_addr;
0161     uint64_t ctc_timestamp;
0162     uint64_t ctc_delta;
0163     uint64_t cycle_cnt;
0164     uint64_t cyc_ref_timestamp;
0165     uint64_t first_timestamp;
0166     uint64_t last_reliable_timestamp;
0167     uint64_t vmcs;
0168     uint64_t print_once;
0169     uint64_t last_ctc;
0170     uint32_t last_mtc;
0171     uint32_t tsc_ctc_ratio_n;
0172     uint32_t tsc_ctc_ratio_d;
0173     uint32_t tsc_ctc_mult;
0174     uint32_t tsc_slip;
0175     uint32_t ctc_rem_mask;
0176     int mtc_shift;
0177     struct intel_pt_stack stack;
0178     enum intel_pt_pkt_state pkt_state;
0179     enum intel_pt_pkt_ctx pkt_ctx;
0180     enum intel_pt_pkt_ctx prev_pkt_ctx;
0181     enum intel_pt_blk_type blk_type;
0182     int blk_type_pos;
0183     struct intel_pt_pkt packet;
0184     struct intel_pt_pkt tnt;
0185     int pkt_step;
0186     int pkt_len;
0187     int last_packet_type;
0188     unsigned int cbr;
0189     unsigned int cbr_seen;
0190     unsigned int max_non_turbo_ratio;
0191     double max_non_turbo_ratio_fp;
0192     double cbr_cyc_to_tsc;
0193     double calc_cyc_to_tsc;
0194     bool have_calc_cyc_to_tsc;
0195     int exec_mode;
0196     unsigned int insn_bytes;
0197     uint64_t period;
0198     enum intel_pt_period_type period_type;
0199     uint64_t tot_insn_cnt;
0200     uint64_t period_insn_cnt;
0201     uint64_t period_mask;
0202     uint64_t period_ticks;
0203     uint64_t last_masked_timestamp;
0204     uint64_t tot_cyc_cnt;
0205     uint64_t sample_tot_cyc_cnt;
0206     uint64_t base_cyc_cnt;
0207     uint64_t cyc_cnt_timestamp;
0208     uint64_t ctl;
0209     uint64_t cyc_threshold;
0210     double tsc_to_cyc;
0211     bool continuous_period;
0212     bool overflow;
0213     bool set_fup_tx_flags;
0214     bool set_fup_ptw;
0215     bool set_fup_mwait;
0216     bool set_fup_pwre;
0217     bool set_fup_exstop;
0218     bool set_fup_bep;
0219     bool set_fup_cfe_ip;
0220     bool set_fup_cfe;
0221     bool set_fup_mode_exec;
0222     bool sample_cyc;
0223     unsigned int fup_tx_flags;
0224     unsigned int tx_flags;
0225     uint64_t fup_ptw_payload;
0226     uint64_t fup_mwait_payload;
0227     uint64_t fup_pwre_payload;
0228     uint64_t cbr_payload;
0229     uint64_t timestamp_insn_cnt;
0230     uint64_t sample_insn_cnt;
0231     uint64_t stuck_ip;
0232     struct intel_pt_pkt fup_cfe_pkt;
0233     int max_loops;
0234     int no_progress;
0235     int stuck_ip_prd;
0236     int stuck_ip_cnt;
0237     uint64_t psb_ip;
0238     const unsigned char *next_buf;
0239     size_t next_len;
0240     unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
0241     int evd_cnt;
0242     struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];
0243 };
0244
0245 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
0246 {
0247     int i;
0248
0249     for (i = 0; x != 1; i++)
0250         x >>= 1;
0251
0252     return x << i;
0253 }
0254
0255 __printf(1, 2)
0256 static void p_log(const char *fmt, ...)
0257 {
0258     char buf[512];
0259     va_list args;
0260
0261     va_start(args, fmt);
0262     vsnprintf(buf, sizeof(buf), fmt, args);
0263     va_end(args);
0264
0265     fprintf(stderr, "%s\n", buf);
0266     intel_pt_log("%s\n", buf);
0267 }
0268
0269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
0270                 enum intel_pt_p_once id)
0271 {
0272     uint64_t bit = 1ULL << id;
0273
0274     if (decoder->print_once & bit)
0275         return false;
0276     decoder->print_once |= bit;
0277     return true;
0278 }
0279
0280 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
0281 {
0282     if (!(ctl & INTEL_PT_CYC_ENABLE))
0283         return 0;
0284
0285     return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
0286 }
0287
0288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
0289 {
0290     if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
0291         uint64_t period;
0292
0293         period = intel_pt_lower_power_of_2(decoder->period);
0294         decoder->period_mask  = ~(period - 1);
0295         decoder->period_ticks = period;
0296     }
0297 }
0298
0299 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
0300 {
0301     if (!d)
0302         return 0;
0303     return (t / d) * n + ((t % d) * n) / d;
0304 }
0305
0306 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
0307 {
0308     struct intel_pt_decoder *decoder;
0309
0310     if (!params->get_trace || !params->walk_insn)
0311         return NULL;
0312
0313     decoder = zalloc(sizeof(struct intel_pt_decoder));
0314     if (!decoder)
0315         return NULL;
0316
0317     decoder->get_trace          = params->get_trace;
0318     decoder->walk_insn          = params->walk_insn;
0319     decoder->pgd_ip             = params->pgd_ip;
0320     decoder->lookahead          = params->lookahead;
0321     decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
0322     decoder->data               = params->data;
0323     decoder->return_compression = params->return_compression;
0324     decoder->branch_enable      = params->branch_enable;
0325     decoder->hop                = params->quick >= 1;
0326     decoder->leap               = params->quick >= 2;
0327     decoder->vm_time_correlation = params->vm_time_correlation;
0328     decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
0329     decoder->first_timestamp    = params->first_timestamp;
0330     decoder->last_reliable_timestamp = params->first_timestamp;
0331     decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
0332
0333     decoder->flags              = params->flags;
0334
0335     decoder->ctl                = params->ctl;
0336     decoder->period             = params->period;
0337     decoder->period_type        = params->period_type;
0338
0339     decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
0340     decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
0341
0342     decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
0343
0344     intel_pt_setup_period(decoder);
0345
0346     decoder->mtc_shift = params->mtc_period;
0347     decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
0348
0349     decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
0350     decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
0351
0352     if (!decoder->tsc_ctc_ratio_n)
0353         decoder->tsc_ctc_ratio_d = 0;
0354
0355     if (decoder->tsc_ctc_ratio_d) {
0356         if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
0357             decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
0358                         decoder->tsc_ctc_ratio_d;
0359     }
0360
0361     /*
0362      * A TSC packet can slip past MTC packets so that the timestamp appears
0363      * to go backwards. One estimate is that can be up to about 40 CPU
0364      * cycles, which is certainly less than 0x1000 TSC ticks, but accept
0365      * slippage an order of magnitude more to be on the safe side.
0366      */
0367     decoder->tsc_slip = 0x10000;
0368
0369     intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
0370     intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
0371     intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
0372     intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
0373     intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
0374
0375     if (decoder->hop)
0376         intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
0377
0378     return decoder;
0379 }
0380
0381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
0382                   uint64_t first_timestamp)
0383 {
0384     decoder->first_timestamp = first_timestamp;
0385 }
0386
0387 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
0388 {
0389     struct intel_pt_blk *blk = stack->blk;
0390
0391     stack->blk = blk->prev;
0392     if (!stack->spare)
0393         stack->spare = blk;
0394     else
0395         free(blk);
0396 }
0397
0398 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
0399 {
0400     if (!stack->pos) {
0401         if (!stack->blk)
0402             return 0;
0403         intel_pt_pop_blk(stack);
0404         if (!stack->blk)
0405             return 0;
0406         stack->pos = INTEL_PT_BLK_SIZE;
0407     }
0408     return stack->blk->ip[--stack->pos];
0409 }
0410
0411 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
0412 {
0413     struct intel_pt_blk *blk;
0414
0415     if (stack->spare) {
0416         blk = stack->spare;
0417         stack->spare = NULL;
0418     } else {
0419         blk = malloc(sizeof(struct intel_pt_blk));
0420         if (!blk)
0421             return -ENOMEM;
0422     }
0423
0424     blk->prev = stack->blk;
0425     stack->blk = blk;
0426     stack->pos = 0;
0427     return 0;
0428 }
0429
0430 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
0431 {
0432     int err;
0433
0434     if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
0435         err = intel_pt_alloc_blk(stack);
0436         if (err)
0437             return err;
0438     }
0439
0440     stack->blk->ip[stack->pos++] = ip;
0441     return 0;
0442 }
0443
0444 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
0445 {
0446     while (stack->blk)
0447         intel_pt_pop_blk(stack);
0448     stack->pos = 0;
0449 }
0450
0451 static void intel_pt_free_stack(struct intel_pt_stack *stack)
0452 {
0453     intel_pt_clear_stack(stack);
0454     zfree(&stack->blk);
0455     zfree(&stack->spare);
0456 }
0457
0458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
0459 {
0460     intel_pt_free_stack(&decoder->stack);
0461     free(decoder);
0462 }
0463
0464 static int intel_pt_ext_err(int code)
0465 {
0466     switch (code) {
0467     case -ENOMEM:
0468         return INTEL_PT_ERR_NOMEM;
0469     case -ENOSYS:
0470         return INTEL_PT_ERR_INTERN;
0471     case -EBADMSG:
0472         return INTEL_PT_ERR_BADPKT;
0473     case -ENODATA:
0474         return INTEL_PT_ERR_NODATA;
0475     case -EILSEQ:
0476         return INTEL_PT_ERR_NOINSN;
0477     case -ENOENT:
0478         return INTEL_PT_ERR_MISMAT;
0479     case -EOVERFLOW:
0480         return INTEL_PT_ERR_OVR;
0481     case -ENOSPC:
0482         return INTEL_PT_ERR_LOST;
0483     case -ELOOP:
0484         return INTEL_PT_ERR_NELOOP;
0485     case -ECONNRESET:
0486         return INTEL_PT_ERR_EPTW;
0487     default:
0488         return INTEL_PT_ERR_UNK;
0489     }
0490 }
0491
0492 static const char *intel_pt_err_msgs[] = {
0493     [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
0494     [INTEL_PT_ERR_INTERN] = "Internal error",
0495     [INTEL_PT_ERR_BADPKT] = "Bad packet",
0496     [INTEL_PT_ERR_NODATA] = "No more data",
0497     [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
0498     [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
0499     [INTEL_PT_ERR_OVR]    = "Overflow packet",
0500     [INTEL_PT_ERR_LOST]   = "Lost trace data",
0501     [INTEL_PT_ERR_UNK]    = "Unknown error!",
0502     [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
0503     [INTEL_PT_ERR_EPTW]   = "Broken emulated ptwrite",
0504 };
0505
0506 int intel_pt__strerror(int code, char *buf, size_t buflen)
0507 {
0508     if (code < 1 || code >= INTEL_PT_ERR_MAX)
0509         code = INTEL_PT_ERR_UNK;
0510     strlcpy(buf, intel_pt_err_msgs[code], buflen);
0511     return 0;
0512 }
0513
0514 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
0515                  uint64_t last_ip)
0516 {
0517     uint64_t ip;
0518
0519     switch (packet->count) {
0520     case 1:
0521         ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
0522              packet->payload;
0523         break;
0524     case 2:
0525         ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
0526              packet->payload;
0527         break;
0528     case 3:
0529         ip = packet->payload;
0530         /* Sign-extend 6-byte ip */
0531         if (ip & (uint64_t)0x800000000000ULL)
0532             ip |= (uint64_t)0xffff000000000000ULL;
0533         break;
0534     case 4:
0535         ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
0536              packet->payload;
0537         break;
0538     case 6:
0539         ip = packet->payload;
0540         break;
0541     default:
0542         return 0;
0543     }
0544
0545     return ip;
0546 }
0547
0548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
0549 {
0550     decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
0551     decoder->have_last_ip = true;
0552 }
0553
0554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
0555 {
0556     intel_pt_set_last_ip(decoder);
0557     decoder->ip = decoder->last_ip;
0558 }
0559
0560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
0561 {
0562     intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
0563                 decoder->buf);
0564 }
0565
0566 static int intel_pt_bug(struct intel_pt_decoder *decoder)
0567 {
0568     intel_pt_log("ERROR: Internal error\n");
0569     decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
0570     return -ENOSYS;
0571 }
0572
0573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
0574 {
0575     decoder->tx_flags = 0;
0576 }
0577
0578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
0579 {
0580     decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
0581 }
0582
0583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
0584 {
0585     decoder->pip_payload = decoder->packet.payload;
0586 }
0587
0588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
0589 {
0590     decoder->next_nr = decoder->pip_payload & 1;
0591 }
0592
0593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
0594 {
0595     decoder->nr = decoder->pip_payload & 1;
0596     decoder->next_nr = decoder->nr;
0597 }
0598
0599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
0600 {
0601     intel_pt_update_pip(decoder);
0602     intel_pt_set_nr(decoder);
0603 }
0604
0605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
0606 {
0607     intel_pt_clear_tx_flags(decoder);
0608     decoder->have_tma = false;
0609     decoder->pkt_len = 1;
0610     decoder->pkt_step = 1;
0611     intel_pt_decoder_log_packet(decoder);
0612     if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
0613         intel_pt_log("ERROR: Bad packet\n");
0614         decoder->pkt_state = INTEL_PT_STATE_ERR1;
0615     }
0616     return -EBADMSG;
0617 }
0618
0619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
0620 {
0621     decoder->sample_timestamp = decoder->timestamp;
0622     decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
0623     decoder->state.cycles = decoder->tot_cyc_cnt;
0624 }
0625
0626 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
0627 {
0628     decoder->ip = 0;
0629     decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
0630     decoder->timestamp = 0;
0631     decoder->have_tma = false;
0632 }
0633
0634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
0635 {
0636     struct intel_pt_buffer buffer = { .buf = 0, };
0637     int ret;
0638
0639     decoder->pkt_step = 0;
0640
0641     intel_pt_log("Getting more data\n");
0642     ret = decoder->get_trace(&buffer, decoder->data);
0643     if (ret)
0644         return ret;
0645     decoder->buf = buffer.buf;
0646     decoder->len = buffer.len;
0647     if (!decoder->len) {
0648         intel_pt_log("No more data\n");
0649         return -ENODATA;
0650     }
0651     decoder->buf_timestamp = buffer.ref_timestamp;
0652     if (!buffer.consecutive || reposition) {
0653         intel_pt_reposition(decoder);
0654         decoder->ref_timestamp = buffer.ref_timestamp;
0655         decoder->state.trace_nr = buffer.trace_nr;
0656         decoder->vm_tm_corr_same_buf = false;
0657         intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
0658                  decoder->ref_timestamp);
0659         return -ENOLINK;
0660     }
0661
0662     return 0;
0663 }
0664
0665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
0666                   bool reposition)
0667 {
0668     if (!decoder->next_buf)
0669         return intel_pt_get_data(decoder, reposition);
0670
0671     decoder->buf = decoder->next_buf;
0672     decoder->len = decoder->next_len;
0673     decoder->next_buf = 0;
0674     decoder->next_len = 0;
0675     return 0;
0676 }
0677
0678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
0679 {
0680     unsigned char *buf = decoder->temp_buf;
0681     size_t old_len, len, n;
0682     int ret;
0683
0684     old_len = decoder->len;
0685     len = decoder->len;
0686     memcpy(buf, decoder->buf, len);
0687
0688     ret = intel_pt_get_data(decoder, false);
0689     if (ret) {
0690         decoder->pos += old_len;
0691         return ret < 0 ? ret : -EINVAL;
0692     }
0693
0694     n = INTEL_PT_PKT_MAX_SZ - len;
0695     if (n > decoder->len)
0696         n = decoder->len;
0697     memcpy(buf + len, decoder->buf, n);
0698     len += n;
0699
0700     decoder->prev_pkt_ctx = decoder->pkt_ctx;
0701     ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
0702     if (ret < (int)old_len) {
0703         decoder->next_buf = decoder->buf;
0704         decoder->next_len = decoder->len;
0705         decoder->buf = buf;
0706         decoder->len = old_len;
0707         return intel_pt_bad_packet(decoder);
0708     }
0709
0710     decoder->next_buf = decoder->buf + (ret - old_len);
0711     decoder->next_len = decoder->len - (ret - old_len);
0712
0713     decoder->buf = buf;
0714     decoder->len = ret;
0715
0716     return ret;
0717 }
0718
0719 struct intel_pt_pkt_info {
0720     struct intel_pt_decoder   *decoder;
0721     struct intel_pt_pkt       packet;
0722     uint64_t                  pos;
0723     int                       pkt_len;
0724     int                       last_packet_type;
0725     void                      *data;
0726 };
0727
0728 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
0729
0730 /* Lookahead packets in current buffer */
0731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
0732                   intel_pt_pkt_cb_t cb, void *data)
0733 {
0734     struct intel_pt_pkt_info pkt_info;
0735     const unsigned char *buf = decoder->buf;
0736     enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
0737     size_t len = decoder->len;
0738     int ret;
0739
0740     pkt_info.decoder          = decoder;
0741     pkt_info.pos              = decoder->pos;
0742     pkt_info.pkt_len          = decoder->pkt_step;
0743     pkt_info.last_packet_type = decoder->last_packet_type;
0744     pkt_info.data             = data;
0745
0746     while (1) {
0747         do {
0748             pkt_info.pos += pkt_info.pkt_len;
0749             buf          += pkt_info.pkt_len;
0750             len          -= pkt_info.pkt_len;
0751
0752             if (!len)
0753                 return INTEL_PT_NEED_MORE_BYTES;
0754
0755             ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
0756                           &pkt_ctx);
0757             if (!ret)
0758                 return INTEL_PT_NEED_MORE_BYTES;
0759             if (ret < 0)
0760                 return ret;
0761
0762             pkt_info.pkt_len = ret;
0763         } while (pkt_info.packet.type == INTEL_PT_PAD);
0764
0765         ret = cb(&pkt_info);
0766         if (ret)
0767             return 0;
0768
0769         pkt_info.last_packet_type = pkt_info.packet.type;
0770     }
0771 }
0772
0773 struct intel_pt_calc_cyc_to_tsc_info {
0774     uint64_t        cycle_cnt;
0775     unsigned int    cbr;
0776     uint32_t        last_mtc;
0777     uint64_t        ctc_timestamp;
0778     uint64_t        ctc_delta;
0779     uint64_t        tsc_timestamp;
0780     uint64_t        timestamp;
0781     bool            have_tma;
0782     bool            fixup_last_mtc;
0783     bool            from_mtc;
0784     double          cbr_cyc_to_tsc;
0785 };
0786
0787 /*
0788  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
0789  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
0790  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
0791  * packet by copying the missing bits from the current MTC assuming the least
0792  * difference between the two, and that the current MTC comes after last_mtc.
0793  */
0794 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
0795                     uint32_t *last_mtc)
0796 {
0797     uint32_t first_missing_bit = 1U << (16 - mtc_shift);
0798     uint32_t mask = ~(first_missing_bit - 1);
0799
0800     *last_mtc |= mtc & mask;
0801     if (*last_mtc >= mtc) {
0802         *last_mtc -= first_missing_bit;
0803         *last_mtc &= 0xff;
0804     }
0805 }
0806
0807 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
0808 {
0809     struct intel_pt_decoder *decoder = pkt_info->decoder;
0810     struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
0811     uint64_t timestamp;
0812     double cyc_to_tsc;
0813     unsigned int cbr;
0814     uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
0815
0816     switch (pkt_info->packet.type) {
0817     case INTEL_PT_TNT:
0818     case INTEL_PT_TIP_PGE:
0819     case INTEL_PT_TIP:
0820     case INTEL_PT_FUP:
0821     case INTEL_PT_PSB:
0822     case INTEL_PT_PIP:
0823     case INTEL_PT_MODE_EXEC:
0824     case INTEL_PT_MODE_TSX:
0825     case INTEL_PT_PSBEND:
0826     case INTEL_PT_PAD:
0827     case INTEL_PT_VMCS:
0828     case INTEL_PT_MNT:
0829     case INTEL_PT_PTWRITE:
0830     case INTEL_PT_PTWRITE_IP:
0831     case INTEL_PT_BBP:
0832     case INTEL_PT_BIP:
0833     case INTEL_PT_BEP:
0834     case INTEL_PT_BEP_IP:
0835     case INTEL_PT_CFE:
0836     case INTEL_PT_CFE_IP:
0837     case INTEL_PT_EVD:
0838         return 0;
0839
0840     case INTEL_PT_MTC:
0841         if (!data->have_tma)
0842             return 0;
0843
0844         mtc = pkt_info->packet.payload;
0845         if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
0846             data->fixup_last_mtc = false;
0847             intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
0848                         &data->last_mtc);
0849         }
0850         if (mtc > data->last_mtc)
0851             mtc_delta = mtc - data->last_mtc;
0852         else
0853             mtc_delta = mtc + 256 - data->last_mtc;
0854         data->ctc_delta += mtc_delta << decoder->mtc_shift;
0855         data->last_mtc = mtc;
0856
0857         if (decoder->tsc_ctc_mult) {
0858             timestamp = data->ctc_timestamp +
0859                 data->ctc_delta * decoder->tsc_ctc_mult;
0860         } else {
0861             timestamp = data->ctc_timestamp +
0862                 multdiv(data->ctc_delta,
0863                     decoder->tsc_ctc_ratio_n,
0864                     decoder->tsc_ctc_ratio_d);
0865         }
0866
0867         if (timestamp < data->timestamp)
0868             return 1;
0869
0870         if (pkt_info->last_packet_type != INTEL_PT_CYC) {
0871             data->timestamp = timestamp;
0872             return 0;
0873         }
0874
0875         break;
0876
0877     case INTEL_PT_TSC:
0878         /*
0879          * For now, do not support using TSC packets - refer
0880          * intel_pt_calc_cyc_to_tsc().
0881          */
0882         if (data->from_mtc)
0883             return 1;
0884         timestamp = pkt_info->packet.payload |
0885                 (data->timestamp & (0xffULL << 56));
0886         if (data->from_mtc && timestamp < data->timestamp &&
0887             data->timestamp - timestamp < decoder->tsc_slip)
0888             return 1;
0889         if (timestamp < data->timestamp)
0890             timestamp += (1ULL << 56);
0891         if (pkt_info->last_packet_type != INTEL_PT_CYC) {
0892             if (data->from_mtc)
0893                 return 1;
0894             data->tsc_timestamp = timestamp;
0895             data->timestamp = timestamp;
0896             return 0;
0897         }
0898         break;
0899
0900     case INTEL_PT_TMA:
0901         if (data->from_mtc)
0902             return 1;
0903
0904         if (!decoder->tsc_ctc_ratio_d)
0905             return 0;
0906
0907         ctc = pkt_info->packet.payload;
0908         fc = pkt_info->packet.count;
0909         ctc_rem = ctc & decoder->ctc_rem_mask;
0910
0911         data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
0912
0913         data->ctc_timestamp = data->tsc_timestamp - fc;
0914         if (decoder->tsc_ctc_mult) {
0915             data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
0916         } else {
0917             data->ctc_timestamp -=
0918                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
0919                     decoder->tsc_ctc_ratio_d);
0920         }
0921
0922         data->ctc_delta = 0;
0923         data->have_tma = true;
0924         data->fixup_last_mtc = true;
0925
0926         return 0;
0927
0928     case INTEL_PT_CYC:
0929         data->cycle_cnt += pkt_info->packet.payload;
0930         return 0;
0931
0932     case INTEL_PT_CBR:
0933         cbr = pkt_info->packet.payload;
0934         if (data->cbr && data->cbr != cbr)
0935             return 1;
0936         data->cbr = cbr;
0937         data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
0938         return 0;
0939
0940     case INTEL_PT_TIP_PGD:
0941     case INTEL_PT_TRACESTOP:
0942     case INTEL_PT_EXSTOP:
0943     case INTEL_PT_EXSTOP_IP:
0944     case INTEL_PT_MWAIT:
0945     case INTEL_PT_PWRE:
0946     case INTEL_PT_PWRX:
0947     case INTEL_PT_OVF:
0948     case INTEL_PT_BAD: /* Does not happen */
0949     default:
0950         return 1;
0951     }
0952
0953     if (!data->cbr && decoder->cbr) {
0954         data->cbr = decoder->cbr;
0955         data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
0956     }
0957
0958     if (!data->cycle_cnt)
0959         return 1;
0960
0961     cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
0962
0963     if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
0964         cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
0965         intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
0966                  cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
0967         return 1;
0968     }
0969
0970     decoder->calc_cyc_to_tsc = cyc_to_tsc;
0971     decoder->have_calc_cyc_to_tsc = true;
0972
0973     if (data->cbr) {
0974         intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
0975                  cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
0976     } else {
0977         intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
0978                  cyc_to_tsc, pkt_info->pos);
0979     }
0980
0981     return 1;
0982 }
0983
0984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
0985                      bool from_mtc)
0986 {
0987     struct intel_pt_calc_cyc_to_tsc_info data = {
0988         .cycle_cnt      = 0,
0989         .cbr            = 0,
0990         .last_mtc       = decoder->last_mtc,
0991         .ctc_timestamp  = decoder->ctc_timestamp,
0992         .ctc_delta      = decoder->ctc_delta,
0993         .tsc_timestamp  = decoder->tsc_timestamp,
0994         .timestamp      = decoder->timestamp,
0995         .have_tma       = decoder->have_tma,
0996         .fixup_last_mtc = decoder->fixup_last_mtc,
0997         .from_mtc       = from_mtc,
0998         .cbr_cyc_to_tsc = 0,
0999     };
1000
1001     /*
1002      * For now, do not support using TSC packets for at least the reasons:
1003      * 1) timing might have stopped
1004      * 2) TSC packets within PSB+ can slip against CYC packets
1005      */
1006     if (!from_mtc)
1007         return;
1008
1009     intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
1010 }
1011
1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
1013 {
1014     int ret;
1015
1016     decoder->last_packet_type = decoder->packet.type;
1017
1018     do {
1019         decoder->pos += decoder->pkt_step;
1020         decoder->buf += decoder->pkt_step;
1021         decoder->len -= decoder->pkt_step;
1022
1023         if (!decoder->len) {
1024             ret = intel_pt_get_next_data(decoder, false);
1025             if (ret)
1026                 return ret;
1027         }
1028
1029         decoder->prev_pkt_ctx = decoder->pkt_ctx;
1030         ret = intel_pt_get_packet(decoder->buf, decoder->len,
1031                       &decoder->packet, &decoder->pkt_ctx);
1032         if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1033             decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1034             ret = intel_pt_get_split_packet(decoder);
1035             if (ret < 0)
1036                 return ret;
1037         }
1038         if (ret <= 0)
1039             return intel_pt_bad_packet(decoder);
1040
1041         decoder->pkt_len = ret;
1042         decoder->pkt_step = ret;
1043         intel_pt_decoder_log_packet(decoder);
1044     } while (decoder->packet.type == INTEL_PT_PAD);
1045
1046     return 0;
1047 }
1048
1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1050 {
1051     uint64_t timestamp, masked_timestamp;
1052
1053     timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1054     masked_timestamp = timestamp & decoder->period_mask;
1055     if (decoder->continuous_period) {
1056         if (masked_timestamp > decoder->last_masked_timestamp)
1057             return 1;
1058     } else {
1059         timestamp += 1;
1060         masked_timestamp = timestamp & decoder->period_mask;
1061         if (masked_timestamp > decoder->last_masked_timestamp) {
1062             decoder->last_masked_timestamp = masked_timestamp;
1063             decoder->continuous_period = true;
1064         }
1065     }
1066
1067     if (masked_timestamp < decoder->last_masked_timestamp)
1068         return decoder->period_ticks;
1069
1070     return decoder->period_ticks - (timestamp - masked_timestamp);
1071 }
1072
1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1074 {
1075     switch (decoder->period_type) {
1076     case INTEL_PT_PERIOD_INSTRUCTIONS:
1077         return decoder->period - decoder->period_insn_cnt;
1078     case INTEL_PT_PERIOD_TICKS:
1079         return intel_pt_next_period(decoder);
1080     case INTEL_PT_PERIOD_NONE:
1081     case INTEL_PT_PERIOD_MTC:
1082     default:
1083         return 0;
1084     }
1085 }
1086
1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1088 {
1089     uint64_t timestamp, masked_timestamp;
1090
1091     switch (decoder->period_type) {
1092     case INTEL_PT_PERIOD_INSTRUCTIONS:
1093         decoder->period_insn_cnt = 0;
1094         break;
1095     case INTEL_PT_PERIOD_TICKS:
1096         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1097         masked_timestamp = timestamp & decoder->period_mask;
1098         if (masked_timestamp > decoder->last_masked_timestamp)
1099             decoder->last_masked_timestamp = masked_timestamp;
1100         else
1101             decoder->last_masked_timestamp += decoder->period_ticks;
1102         break;
1103     case INTEL_PT_PERIOD_NONE:
1104     case INTEL_PT_PERIOD_MTC:
1105     default:
1106         break;
1107     }
1108
1109     decoder->state.type |= INTEL_PT_INSTRUCTION;
1110 }
1111
1112 /*
1113  * Sample FUP instruction at the same time as reporting the FUP event, so the
1114  * instruction sample gets the same flags as the FUP event.
1115  */
1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)
1117 {
1118     struct intel_pt_insn intel_pt_insn;
1119     uint64_t max_insn_cnt, insn_cnt = 0;
1120     int err;
1121
1122     decoder->state.insn_op = INTEL_PT_OP_OTHER;
1123     decoder->state.insn_len = 0;
1124
1125     if (!decoder->branch_enable || !decoder->pge || decoder->hop ||
1126         decoder->ip != decoder->last_ip)
1127         return;
1128
1129     if (!decoder->mtc_insn)
1130         decoder->mtc_insn = true;
1131
1132     max_insn_cnt = intel_pt_next_sample(decoder);
1133     if (max_insn_cnt != 1)
1134         return;
1135
1136     err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,
1137                  0, max_insn_cnt, decoder->data);
1138     /* Ignore error, it will be reported next walk anyway */
1139     if (err)
1140         return;
1141
1142     if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) {
1143         intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);
1144         return;
1145     }
1146
1147     decoder->tot_insn_cnt += insn_cnt;
1148     decoder->timestamp_insn_cnt += insn_cnt;
1149     decoder->sample_insn_cnt += insn_cnt;
1150     decoder->period_insn_cnt += insn_cnt;
1151
1152     intel_pt_sample_insn(decoder);
1153
1154     decoder->state.type |= INTEL_PT_INSTRUCTION;
1155     decoder->ip += intel_pt_insn.length;
1156 }
1157
1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1159                   struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1160 {
1161     uint64_t max_insn_cnt, insn_cnt = 0;
1162     int err;
1163
1164     if (!decoder->mtc_insn)
1165         decoder->mtc_insn = true;
1166
1167     max_insn_cnt = intel_pt_next_sample(decoder);
1168
1169     err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1170                  max_insn_cnt, decoder->data);
1171
1172     decoder->tot_insn_cnt += insn_cnt;
1173     decoder->timestamp_insn_cnt += insn_cnt;
1174     decoder->sample_insn_cnt += insn_cnt;
1175     decoder->period_insn_cnt += insn_cnt;
1176
1177     if (err) {
1178         decoder->no_progress = 0;
1179         decoder->pkt_state = INTEL_PT_STATE_ERR2;
1180         intel_pt_log_at("ERROR: Failed to get instruction",
1181                 decoder->ip);
1182         if (err == -ENOENT)
1183             return -ENOLINK;
1184         return -EILSEQ;
1185     }
1186
1187     if (ip && decoder->ip == ip) {
1188         err = -EAGAIN;
1189         goto out;
1190     }
1191
1192     if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1193         intel_pt_sample_insn(decoder);
1194
1195     if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1196         decoder->state.type = INTEL_PT_INSTRUCTION;
1197         decoder->state.from_ip = decoder->ip;
1198         decoder->state.to_ip = 0;
1199         decoder->ip += intel_pt_insn->length;
1200         err = INTEL_PT_RETURN;
1201         goto out;
1202     }
1203
1204     if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1205         /* Zero-length calls are excluded */
1206         if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1207             intel_pt_insn->rel) {
1208             err = intel_pt_push(&decoder->stack, decoder->ip +
1209                         intel_pt_insn->length);
1210             if (err)
1211                 goto out;
1212         }
1213     } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1214         decoder->ret_addr = intel_pt_pop(&decoder->stack);
1215     }
1216
1217     if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1218         int cnt = decoder->no_progress++;
1219
1220         decoder->state.from_ip = decoder->ip;
1221         decoder->ip += intel_pt_insn->length +
1222                 intel_pt_insn->rel;
1223         decoder->state.to_ip = decoder->ip;
1224         err = INTEL_PT_RETURN;
1225
1226         /*
1227          * Check for being stuck in a loop.  This can happen if a
1228          * decoder error results in the decoder erroneously setting the
1229          * ip to an address that is itself in an infinite loop that
1230          * consumes no packets.  When that happens, there must be an
1231          * unconditional branch.
1232          */
1233         if (cnt) {
1234             if (cnt == 1) {
1235                 decoder->stuck_ip = decoder->state.to_ip;
1236                 decoder->stuck_ip_prd = 1;
1237                 decoder->stuck_ip_cnt = 1;
1238             } else if (cnt > decoder->max_loops ||
1239                    decoder->state.to_ip == decoder->stuck_ip) {
1240                 intel_pt_log_at("ERROR: Never-ending loop",
1241                         decoder->state.to_ip);
1242                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1243                 err = -ELOOP;
1244                 goto out;
1245             } else if (!--decoder->stuck_ip_cnt) {
1246                 decoder->stuck_ip_prd += 1;
1247                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1248                 decoder->stuck_ip = decoder->state.to_ip;
1249             }
1250         }
1251         goto out_no_progress;
1252     }
1253 out:
1254     decoder->no_progress = 0;
1255 out_no_progress:
1256     decoder->state.insn_op = intel_pt_insn->op;
1257     decoder->state.insn_len = intel_pt_insn->length;
1258     memcpy(decoder->state.insn, intel_pt_insn->buf,
1259            INTEL_PT_INSN_BUF_SZ);
1260
1261     if (decoder->tx_flags & INTEL_PT_IN_TX)
1262         decoder->state.flags |= INTEL_PT_IN_TX;
1263
1264     return err;
1265 }
1266
1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)
1268 {
1269     bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1270
1271     decoder->exec_mode = decoder->packet.payload;
1272     decoder->iflag = iflag;
1273     decoder->next_iflag = iflag;
1274     decoder->state.from_iflag = iflag;
1275     decoder->state.to_iflag = iflag;
1276 }
1277
1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)
1279 {
1280     bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1281
1282     decoder->exec_mode = decoder->packet.payload;
1283     decoder->next_iflag = iflag;
1284 }
1285
1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)
1287 {
1288     decoder->state.type |= INTEL_PT_IFLAG_CHG;
1289     decoder->state.from_iflag = decoder->iflag;
1290     decoder->state.to_iflag = decoder->next_iflag;
1291     decoder->iflag = decoder->next_iflag;
1292 }
1293
1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)
1295 {
1296     if (decoder->iflag != decoder->next_iflag)
1297         intel_pt_sample_iflag(decoder);
1298 }
1299
1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)
1301 {
1302     decoder->set_fup_tx_flags = false;
1303     decoder->set_fup_ptw = false;
1304     decoder->set_fup_mwait = false;
1305     decoder->set_fup_pwre = false;
1306     decoder->set_fup_exstop = false;
1307     decoder->set_fup_bep = false;
1308     decoder->set_fup_cfe_ip = false;
1309     decoder->set_fup_cfe = false;
1310     decoder->evd_cnt = 0;
1311     decoder->set_fup_mode_exec = false;
1312     decoder->iflag = decoder->next_iflag;
1313 }
1314
1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)
1316 {
1317     enum intel_pt_sample_type type = decoder->state.type;
1318     bool sample_fup_insn = false;
1319     bool ret = false;
1320
1321     decoder->state.type &= ~INTEL_PT_BRANCH;
1322
1323     if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {
1324         bool ip = decoder->set_fup_cfe_ip;
1325
1326         decoder->set_fup_cfe_ip = false;
1327         decoder->set_fup_cfe = false;
1328         decoder->state.type |= INTEL_PT_EVT;
1329         if (!ip && decoder->pge)
1330             decoder->state.type |= INTEL_PT_BRANCH;
1331         decoder->state.cfe_type = decoder->fup_cfe_pkt.count;
1332         decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;
1333         decoder->state.evd_cnt = decoder->evd_cnt;
1334         decoder->state.evd = decoder->evd;
1335         decoder->evd_cnt = 0;
1336         if (ip || decoder->pge)
1337             decoder->state.flags |= INTEL_PT_FUP_IP;
1338         ret = true;
1339     }
1340     if (decoder->set_fup_mode_exec) {
1341         decoder->set_fup_mode_exec = false;
1342         intel_pt_sample_iflag(decoder);
1343         sample_fup_insn = no_tip;
1344         ret = true;
1345     }
1346     if (decoder->set_fup_tx_flags) {
1347         decoder->set_fup_tx_flags = false;
1348         decoder->tx_flags = decoder->fup_tx_flags;
1349         decoder->state.type |= INTEL_PT_TRANSACTION;
1350         if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1351             decoder->state.type |= INTEL_PT_BRANCH;
1352         decoder->state.flags = decoder->fup_tx_flags;
1353         ret = true;
1354     }
1355     if (decoder->set_fup_ptw) {
1356         decoder->set_fup_ptw = false;
1357         decoder->state.type |= INTEL_PT_PTW;
1358         decoder->state.flags |= INTEL_PT_FUP_IP;
1359         decoder->state.ptw_payload = decoder->fup_ptw_payload;
1360         ret = true;
1361     }
1362     if (decoder->set_fup_mwait) {
1363         decoder->set_fup_mwait = false;
1364         decoder->state.type |= INTEL_PT_MWAIT_OP;
1365         decoder->state.mwait_payload = decoder->fup_mwait_payload;
1366         ret = true;
1367     }
1368     if (decoder->set_fup_pwre) {
1369         decoder->set_fup_pwre = false;
1370         decoder->state.type |= INTEL_PT_PWR_ENTRY;
1371         decoder->state.pwre_payload = decoder->fup_pwre_payload;
1372         ret = true;
1373     }
1374     if (decoder->set_fup_exstop) {
1375         decoder->set_fup_exstop = false;
1376         decoder->state.type |= INTEL_PT_EX_STOP;
1377         decoder->state.flags |= INTEL_PT_FUP_IP;
1378         ret = true;
1379     }
1380     if (decoder->set_fup_bep) {
1381         decoder->set_fup_bep = false;
1382         decoder->state.type |= INTEL_PT_BLK_ITEMS;
1383         ret = true;
1384     }
1385     if (decoder->overflow) {
1386         decoder->overflow = false;
1387         if (!ret && !decoder->pge) {
1388             if (decoder->hop) {
1389                 decoder->state.type = 0;
1390                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1391             }
1392             decoder->pge = true;
1393             decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
1394             decoder->state.from_ip = 0;
1395             decoder->state.to_ip = decoder->ip;
1396             return true;
1397         }
1398     }
1399     if (ret) {
1400         decoder->state.from_ip = decoder->ip;
1401         decoder->state.to_ip = 0;
1402         if (sample_fup_insn)
1403             intel_pt_sample_fup_insn(decoder);
1404     } else {
1405         decoder->state.type = type;
1406     }
1407     return ret;
1408 }
1409
1410 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1411                       struct intel_pt_insn *intel_pt_insn,
1412                       uint64_t ip, int err)
1413 {
1414     return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1415            intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1416            ip == decoder->ip + intel_pt_insn->length;
1417 }
1418
1419 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1420 {
1421     struct intel_pt_insn intel_pt_insn;
1422     uint64_t ip;
1423     int err;
1424
1425     ip = decoder->last_ip;
1426
1427     while (1) {
1428         err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1429         if (err == INTEL_PT_RETURN)
1430             return 0;
1431         if (err == -EAGAIN ||
1432             intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1433             bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1434
1435             decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1436             if (intel_pt_fup_event(decoder, no_tip) && no_tip)
1437                 return 0;
1438             return -EAGAIN;
1439         }
1440         decoder->set_fup_tx_flags = false;
1441         if (err)
1442             return err;
1443
1444         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1445             intel_pt_log_at("ERROR: Unexpected indirect branch",
1446                     decoder->ip);
1447             decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1448             return -ENOENT;
1449         }
1450
1451         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1452             intel_pt_log_at("ERROR: Unexpected conditional branch",
1453                     decoder->ip);
1454             decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1455             return -ENOENT;
1456         }
1457
1458         intel_pt_bug(decoder);
1459     }
1460 }
1461
1462 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1463 {
1464     struct intel_pt_insn intel_pt_insn;
1465     int err;
1466
1467     err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1468     if (err == INTEL_PT_RETURN &&
1469         decoder->pgd_ip &&
1470         decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1471         (decoder->state.type & INTEL_PT_BRANCH) &&
1472         decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1473         /* Unconditional branch leaving filter region */
1474         decoder->no_progress = 0;
1475         decoder->pge = false;
1476         decoder->continuous_period = false;
1477         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1478         decoder->state.type |= INTEL_PT_TRACE_END;
1479         intel_pt_update_nr(decoder);
1480         return 0;
1481     }
1482     if (err == INTEL_PT_RETURN)
1483         return 0;
1484     if (err)
1485         return err;
1486
1487     intel_pt_update_nr(decoder);
1488     intel_pt_sample_iflag_chg(decoder);
1489
1490     if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1491         if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1492             decoder->pge = false;
1493             decoder->continuous_period = false;
1494             decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1495             decoder->state.from_ip = decoder->ip;
1496             if (decoder->packet.count == 0) {
1497                 decoder->state.to_ip = 0;
1498             } else {
1499                 decoder->state.to_ip = decoder->last_ip;
1500                 decoder->ip = decoder->last_ip;
1501             }
1502             decoder->state.type |= INTEL_PT_TRACE_END;
1503         } else {
1504             decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1505             decoder->state.from_ip = decoder->ip;
1506             if (decoder->packet.count == 0) {
1507                 decoder->state.to_ip = 0;
1508             } else {
1509                 decoder->state.to_ip = decoder->last_ip;
1510                 decoder->ip = decoder->last_ip;
1511             }
1512         }
1513         return 0;
1514     }
1515
1516     if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1517         uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1518                  intel_pt_insn.rel;
1519
1520         if (decoder->pgd_ip &&
1521             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1522             decoder->pgd_ip(to_ip, decoder->data)) {
1523             /* Conditional branch leaving filter region */
1524             decoder->pge = false;
1525             decoder->continuous_period = false;
1526             decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1527             decoder->ip = to_ip;
1528             decoder->state.from_ip = decoder->ip;
1529             decoder->state.to_ip = to_ip;
1530             decoder->state.type |= INTEL_PT_TRACE_END;
1531             return 0;
1532         }
1533         intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1534                 decoder->ip);
1535         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1536         return -ENOENT;
1537     }
1538
1539     return intel_pt_bug(decoder);
1540 }
1541
1542 struct eptw_data {
1543     int bit_countdown;
1544     uint64_t payload;
1545 };
1546
1547 static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
1548 {
1549     struct eptw_data *data = pkt_info->data;
1550     int nr_bits;
1551
1552     switch (pkt_info->packet.type) {
1553     case INTEL_PT_PAD:
1554     case INTEL_PT_MNT:
1555     case INTEL_PT_MODE_EXEC:
1556     case INTEL_PT_MODE_TSX:
1557     case INTEL_PT_MTC:
1558     case INTEL_PT_FUP:
1559     case INTEL_PT_CYC:
1560     case INTEL_PT_CBR:
1561     case INTEL_PT_TSC:
1562     case INTEL_PT_TMA:
1563     case INTEL_PT_PIP:
1564     case INTEL_PT_VMCS:
1565     case INTEL_PT_PSB:
1566     case INTEL_PT_PSBEND:
1567     case INTEL_PT_PTWRITE:
1568     case INTEL_PT_PTWRITE_IP:
1569     case INTEL_PT_EXSTOP:
1570     case INTEL_PT_EXSTOP_IP:
1571     case INTEL_PT_MWAIT:
1572     case INTEL_PT_PWRE:
1573     case INTEL_PT_PWRX:
1574     case INTEL_PT_BBP:
1575     case INTEL_PT_BIP:
1576     case INTEL_PT_BEP:
1577     case INTEL_PT_BEP_IP:
1578     case INTEL_PT_CFE:
1579     case INTEL_PT_CFE_IP:
1580     case INTEL_PT_EVD:
1581         break;
1582
1583     case INTEL_PT_TNT:
1584         nr_bits = data->bit_countdown;
1585         if (nr_bits > pkt_info->packet.count)
1586             nr_bits = pkt_info->packet.count;
1587         data->payload <<= nr_bits;
1588         data->payload |= pkt_info->packet.payload >> (64 - nr_bits);
1589         data->bit_countdown -= nr_bits;
1590         return !data->bit_countdown;
1591
1592     case INTEL_PT_TIP_PGE:
1593     case INTEL_PT_TIP_PGD:
1594     case INTEL_PT_TIP:
1595     case INTEL_PT_BAD:
1596     case INTEL_PT_OVF:
1597     case INTEL_PT_TRACESTOP:
1598     default:
1599         return 1;
1600     }
1601
1602     return 0;
1603 }
1604
1605 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)
1606 {
1607     int n = 64 - decoder->tnt.count;
1608     struct eptw_data data = {
1609         .bit_countdown = n,
1610         .payload = decoder->tnt.payload >> n,
1611     };
1612
1613     decoder->emulated_ptwrite = false;
1614     intel_pt_log("Emulated ptwrite detected\n");
1615
1616     intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);
1617     if (data.bit_countdown)
1618         return -ECONNRESET;
1619
1620     decoder->state.type = INTEL_PT_PTW;
1621     decoder->state.from_ip = decoder->ip;
1622     decoder->state.to_ip = 0;
1623     decoder->state.ptw_payload = data.payload;
1624     return 0;
1625 }
1626
1627 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1628 {
1629     struct intel_pt_insn intel_pt_insn;
1630     int err;
1631
1632     while (1) {
1633         if (decoder->emulated_ptwrite)
1634             return intel_pt_emulated_ptwrite(decoder);
1635         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1636         if (err == INTEL_PT_RETURN) {
1637             decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;
1638             return 0;
1639         }
1640         if (err) {
1641             decoder->emulated_ptwrite = false;
1642             return err;
1643         }
1644
1645         if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1646             if (!decoder->return_compression) {
1647                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1648                         decoder->ip);
1649                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1650                 return -ENOENT;
1651             }
1652             if (!decoder->ret_addr) {
1653                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1654                         decoder->ip);
1655                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1656                 return -ENOENT;
1657             }
1658             if (!(decoder->tnt.payload & BIT63)) {
1659                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1660                         decoder->ip);
1661                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1662                 return -ENOENT;
1663             }
1664             decoder->tnt.count -= 1;
1665             if (decoder->tnt.count)
1666                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1667             else
1668                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1669             decoder->tnt.payload <<= 1;
1670             decoder->state.from_ip = decoder->ip;
1671             decoder->ip = decoder->ret_addr;
1672             decoder->state.to_ip = decoder->ip;
1673             return 0;
1674         }
1675
1676         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1677             /* Handle deferred TIPs */
1678             err = intel_pt_get_next_packet(decoder);
1679             if (err)
1680                 return err;
1681             if (decoder->packet.type != INTEL_PT_TIP ||
1682                 decoder->packet.count == 0) {
1683                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1684                         decoder->ip);
1685                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1686                 decoder->pkt_step = 0;
1687                 return -ENOENT;
1688             }
1689             intel_pt_set_last_ip(decoder);
1690             decoder->state.from_ip = decoder->ip;
1691             decoder->state.to_ip = decoder->last_ip;
1692             decoder->ip = decoder->last_ip;
1693             intel_pt_update_nr(decoder);
1694             intel_pt_sample_iflag_chg(decoder);
1695             return 0;
1696         }
1697
1698         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1699             decoder->tnt.count -= 1;
1700             if (decoder->tnt.count)
1701                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1702             else
1703                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1704             if (decoder->tnt.payload & BIT63) {
1705                 decoder->tnt.payload <<= 1;
1706                 decoder->state.from_ip = decoder->ip;
1707                 decoder->ip += intel_pt_insn.length +
1708                            intel_pt_insn.rel;
1709                 decoder->state.to_ip = decoder->ip;
1710                 return 0;
1711             }
1712             /* Instruction sample for a non-taken branch */
1713             if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1714                 decoder->tnt.payload <<= 1;
1715                 decoder->state.type = INTEL_PT_INSTRUCTION;
1716                 decoder->state.from_ip = decoder->ip;
1717                 decoder->state.to_ip = 0;
1718                 decoder->ip += intel_pt_insn.length;
1719                 return 0;
1720             }
1721             decoder->sample_cyc = false;
1722             decoder->ip += intel_pt_insn.length;
1723             if (!decoder->tnt.count) {
1724                 intel_pt_update_sample_time(decoder);
1725                 return -EAGAIN;
1726             }
1727             decoder->tnt.payload <<= 1;
1728             continue;
1729         }
1730
1731         return intel_pt_bug(decoder);
1732     }
1733 }
1734
1735 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1736 {
1737     unsigned int fup_tx_flags;
1738     int err;
1739
1740     fup_tx_flags = decoder->packet.payload &
1741                (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1742     err = intel_pt_get_next_packet(decoder);
1743     if (err)
1744         return err;
1745     if (decoder->packet.type == INTEL_PT_FUP) {
1746         decoder->fup_tx_flags = fup_tx_flags;
1747         decoder->set_fup_tx_flags = true;
1748         if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1749             *no_tip = true;
1750     } else {
1751         intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1752                 decoder->pos);
1753         intel_pt_update_in_tx(decoder);
1754     }
1755     return 0;
1756 }
1757
1758 static int intel_pt_evd(struct intel_pt_decoder *decoder)
1759 {
1760     if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {
1761         intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);
1762         return -ENOSYS;
1763     }
1764     decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){
1765         .type = decoder->packet.count,
1766         .payload = decoder->packet.payload,
1767     };
1768     return 0;
1769 }
1770
1771 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1772 {
1773     timestamp |= (ref_timestamp & (0xffULL << 56));
1774
1775     if (timestamp < ref_timestamp) {
1776         if (ref_timestamp - timestamp > (1ULL << 55))
1777             timestamp += (1ULL << 56);
1778     } else {
1779         if (timestamp - ref_timestamp > (1ULL << 55))
1780             timestamp -= (1ULL << 56);
1781     }
1782
1783     return timestamp;
1784 }
1785
1786 /* For use only when decoder->vm_time_correlation is true */
1787 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1788                    uint64_t timestamp)
1789 {
1790     uint64_t max_timestamp = decoder->buf_timestamp;
1791
1792     if (!max_timestamp) {
1793         max_timestamp = decoder->last_reliable_timestamp +
1794                 0x400000000ULL;
1795     }
1796     return timestamp >= decoder->last_reliable_timestamp &&
1797            timestamp < decoder->buf_timestamp;
1798 }
1799
1800 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1801 {
1802     uint64_t timestamp;
1803     bool bad = false;
1804
1805     decoder->have_tma = false;
1806
1807     if (decoder->ref_timestamp) {
1808         timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1809                         decoder->ref_timestamp);
1810         decoder->tsc_timestamp = timestamp;
1811         decoder->timestamp = timestamp;
1812         decoder->ref_timestamp = 0;
1813         decoder->timestamp_insn_cnt = 0;
1814     } else if (decoder->timestamp) {
1815         timestamp = decoder->packet.payload |
1816                 (decoder->timestamp & (0xffULL << 56));
1817         decoder->tsc_timestamp = timestamp;
1818         if (timestamp < decoder->timestamp &&
1819             decoder->timestamp - timestamp < decoder->tsc_slip) {
1820             intel_pt_log_to("Suppressing backwards timestamp",
1821                     timestamp);
1822             timestamp = decoder->timestamp;
1823         }
1824         if (timestamp < decoder->timestamp) {
1825             if (!decoder->buf_timestamp ||
1826                 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1827                 intel_pt_log_to("Wraparound timestamp", timestamp);
1828                 timestamp += (1ULL << 56);
1829                 decoder->tsc_timestamp = timestamp;
1830             } else {
1831                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1832                 timestamp = decoder->timestamp;
1833                 bad = true;
1834             }
1835         }
1836         if (decoder->vm_time_correlation &&
1837             (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1838             intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1839             p_log("Timestamp out of range");
1840         decoder->timestamp = timestamp;
1841         decoder->timestamp_insn_cnt = 0;
1842     }
1843
1844     if (decoder->last_packet_type == INTEL_PT_CYC) {
1845         decoder->cyc_ref_timestamp = decoder->timestamp;
1846         decoder->cycle_cnt = 0;
1847         decoder->have_calc_cyc_to_tsc = false;
1848         intel_pt_calc_cyc_to_tsc(decoder, false);
1849     }
1850
1851     intel_pt_log_to("Setting timestamp", decoder->timestamp);
1852 }
1853
1854 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1855 {
1856     intel_pt_log("ERROR: Buffer overflow\n");
1857     intel_pt_clear_tx_flags(decoder);
1858     intel_pt_set_nr(decoder);
1859     decoder->timestamp_insn_cnt = 0;
1860     decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1861     decoder->state.from_ip = decoder->ip;
1862     decoder->ip = 0;
1863     decoder->pge = false;
1864     intel_pt_clear_fup_event(decoder);
1865     decoder->overflow = true;
1866     return -EOVERFLOW;
1867 }
1868
1869 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1870 {
1871     if (decoder->have_cyc)
1872         return;
1873
1874     decoder->cyc_cnt_timestamp = decoder->timestamp;
1875     decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1876 }
1877
1878 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1879 {
1880     decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1881
1882     if (decoder->pge)
1883         intel_pt_mtc_cyc_cnt_pge(decoder);
1884 }
1885
1886 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1887 {
1888     uint64_t tot_cyc_cnt, tsc_delta;
1889
1890     if (decoder->have_cyc)
1891         return;
1892
1893     decoder->sample_cyc = true;
1894
1895     if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1896         return;
1897
1898     tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1899     tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1900
1901     if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1902         decoder->tot_cyc_cnt = tot_cyc_cnt;
1903 }
1904
1905 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1906 {
1907     uint32_t ctc = decoder->packet.payload;
1908     uint32_t fc = decoder->packet.count;
1909     uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1910
1911     if (!decoder->tsc_ctc_ratio_d)
1912         return;
1913
1914     if (decoder->pge && !decoder->in_psb)
1915         intel_pt_mtc_cyc_cnt_pge(decoder);
1916     else
1917         intel_pt_mtc_cyc_cnt_upd(decoder);
1918
1919     decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1920     decoder->last_ctc = ctc - ctc_rem;
1921     decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1922     if (decoder->tsc_ctc_mult) {
1923         decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1924     } else {
1925         decoder->ctc_timestamp -= multdiv(ctc_rem,
1926                           decoder->tsc_ctc_ratio_n,
1927                           decoder->tsc_ctc_ratio_d);
1928     }
1929     decoder->ctc_delta = 0;
1930     decoder->have_tma = true;
1931     decoder->fixup_last_mtc = true;
1932     intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1933              decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1934 }
1935
1936 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1937 {
1938     uint64_t timestamp;
1939     uint32_t mtc, mtc_delta;
1940
1941     if (!decoder->have_tma)
1942         return;
1943
1944     mtc = decoder->packet.payload;
1945
1946     if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1947         decoder->fixup_last_mtc = false;
1948         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1949                     &decoder->last_mtc);
1950     }
1951
1952     if (mtc > decoder->last_mtc)
1953         mtc_delta = mtc - decoder->last_mtc;
1954     else
1955         mtc_delta = mtc + 256 - decoder->last_mtc;
1956
1957     decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1958
1959     if (decoder->tsc_ctc_mult) {
1960         timestamp = decoder->ctc_timestamp +
1961                 decoder->ctc_delta * decoder->tsc_ctc_mult;
1962     } else {
1963         timestamp = decoder->ctc_timestamp +
1964                 multdiv(decoder->ctc_delta,
1965                     decoder->tsc_ctc_ratio_n,
1966                     decoder->tsc_ctc_ratio_d);
1967     }
1968
1969     if (timestamp < decoder->timestamp)
1970         intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1971                  timestamp, decoder->timestamp);
1972     else
1973         decoder->timestamp = timestamp;
1974
1975     intel_pt_mtc_cyc_cnt_upd(decoder);
1976
1977     decoder->timestamp_insn_cnt = 0;
1978     decoder->last_mtc = mtc;
1979
1980     if (decoder->last_packet_type == INTEL_PT_CYC) {
1981         decoder->cyc_ref_timestamp = decoder->timestamp;
1982         decoder->cycle_cnt = 0;
1983         decoder->have_calc_cyc_to_tsc = false;
1984         intel_pt_calc_cyc_to_tsc(decoder, true);
1985     }
1986
1987     intel_pt_log_to("Setting timestamp", decoder->timestamp);
1988 }
1989
1990 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1991 {
1992     unsigned int cbr = decoder->packet.payload & 0xff;
1993
1994     decoder->cbr_payload = decoder->packet.payload;
1995
1996     if (decoder->cbr == cbr)
1997         return;
1998
1999     decoder->cbr = cbr;
2000     decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
2001
2002     intel_pt_mtc_cyc_cnt_cbr(decoder);
2003 }
2004
2005 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
2006 {
2007     uint64_t timestamp = decoder->cyc_ref_timestamp;
2008
2009     decoder->have_cyc = true;
2010
2011     decoder->cycle_cnt += decoder->packet.payload;
2012     if (decoder->pge)
2013         decoder->tot_cyc_cnt += decoder->packet.payload;
2014     decoder->sample_cyc = true;
2015
2016     if (!decoder->cyc_ref_timestamp)
2017         return;
2018
2019     if (decoder->have_calc_cyc_to_tsc)
2020         timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
2021     else if (decoder->cbr)
2022         timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
2023     else
2024         return;
2025
2026     if (timestamp < decoder->timestamp)
2027         intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
2028                  timestamp, decoder->timestamp);
2029     else
2030         decoder->timestamp = timestamp;
2031
2032     decoder->timestamp_insn_cnt = 0;
2033
2034     intel_pt_log_to("Setting timestamp", decoder->timestamp);
2035 }
2036
2037 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
2038 {
2039     if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
2040         memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
2041         decoder->state.items.is_32_bit = false;
2042     }
2043     decoder->blk_type = decoder->packet.payload;
2044     decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
2045     if (decoder->blk_type == INTEL_PT_GP_REGS)
2046         decoder->state.items.is_32_bit = decoder->packet.count;
2047     if (decoder->blk_type_pos < 0) {
2048         intel_pt_log("WARNING: Unknown block type %u\n",
2049                  decoder->blk_type);
2050     } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
2051         intel_pt_log("WARNING: Duplicate block type %u\n",
2052                  decoder->blk_type);
2053     }
2054 }
2055
2056 static void intel_pt_bip(struct intel_pt_decoder *decoder)
2057 {
2058     uint32_t id = decoder->packet.count;
2059     uint32_t bit = 1 << id;
2060     int pos = decoder->blk_type_pos;
2061
2062     if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
2063         intel_pt_log("WARNING: Unknown block item %u type %d\n",
2064                  id, decoder->blk_type);
2065         return;
2066     }
2067
2068     if (decoder->state.items.mask[pos] & bit) {
2069         intel_pt_log("WARNING: Duplicate block item %u type %d\n",
2070                  id, decoder->blk_type);
2071     }
2072
2073     decoder->state.items.mask[pos] |= bit;
2074     decoder->state.items.val[pos][id] = decoder->packet.payload;
2075 }
2076
2077 /* Walk PSB+ packets when already in sync. */
2078 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
2079 {
2080     int err;
2081
2082     decoder->in_psb = true;
2083
2084     while (1) {
2085         err = intel_pt_get_next_packet(decoder);
2086         if (err)
2087             goto out;
2088
2089         switch (decoder->packet.type) {
2090         case INTEL_PT_PSBEND:
2091             err = 0;
2092             goto out;
2093
2094         case INTEL_PT_TIP_PGD:
2095         case INTEL_PT_TIP_PGE:
2096         case INTEL_PT_TIP:
2097         case INTEL_PT_TNT:
2098         case INTEL_PT_TRACESTOP:
2099         case INTEL_PT_BAD:
2100         case INTEL_PT_PSB:
2101         case INTEL_PT_PTWRITE:
2102         case INTEL_PT_PTWRITE_IP:
2103         case INTEL_PT_EXSTOP:
2104         case INTEL_PT_EXSTOP_IP:
2105         case INTEL_PT_MWAIT:
2106         case INTEL_PT_PWRE:
2107         case INTEL_PT_PWRX:
2108         case INTEL_PT_BBP:
2109         case INTEL_PT_BIP:
2110         case INTEL_PT_BEP:
2111         case INTEL_PT_BEP_IP:
2112         case INTEL_PT_CFE:
2113         case INTEL_PT_CFE_IP:
2114         case INTEL_PT_EVD:
2115             decoder->have_tma = false;
2116             intel_pt_log("ERROR: Unexpected packet\n");
2117             err = -EAGAIN;
2118             goto out;
2119
2120         case INTEL_PT_OVF:
2121             err = intel_pt_overflow(decoder);
2122             goto out;
2123
2124         case INTEL_PT_TSC:
2125             intel_pt_calc_tsc_timestamp(decoder);
2126             break;
2127
2128         case INTEL_PT_TMA:
2129             intel_pt_calc_tma(decoder);
2130             break;
2131
2132         case INTEL_PT_CBR:
2133             intel_pt_calc_cbr(decoder);
2134             break;
2135
2136         case INTEL_PT_MODE_EXEC:
2137             intel_pt_mode_exec_status(decoder);
2138             break;
2139
2140         case INTEL_PT_PIP:
2141             intel_pt_set_pip(decoder);
2142             break;
2143
2144         case INTEL_PT_FUP:
2145             decoder->pge = true;
2146             if (decoder->packet.count) {
2147                 intel_pt_set_last_ip(decoder);
2148                 decoder->psb_ip = decoder->last_ip;
2149             }
2150             break;
2151
2152         case INTEL_PT_MODE_TSX:
2153             intel_pt_update_in_tx(decoder);
2154             break;
2155
2156         case INTEL_PT_MTC:
2157             intel_pt_calc_mtc_timestamp(decoder);
2158             if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2159                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2160             break;
2161
2162         case INTEL_PT_CYC:
2163             intel_pt_calc_cyc_timestamp(decoder);
2164             break;
2165
2166         case INTEL_PT_VMCS:
2167         case INTEL_PT_MNT:
2168         case INTEL_PT_PAD:
2169         default:
2170             break;
2171         }
2172     }
2173 out:
2174     decoder->in_psb = false;
2175
2176     return err;
2177 }
2178
2179 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
2180 {
2181     int err;
2182
2183     if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
2184         decoder->tx_flags = 0;
2185         decoder->state.flags &= ~INTEL_PT_IN_TX;
2186         decoder->state.flags |= INTEL_PT_ABORT_TX;
2187     } else {
2188         decoder->state.flags |= INTEL_PT_ASYNC;
2189     }
2190
2191     while (1) {
2192         err = intel_pt_get_next_packet(decoder);
2193         if (err)
2194             return err;
2195
2196         switch (decoder->packet.type) {
2197         case INTEL_PT_TNT:
2198         case INTEL_PT_FUP:
2199         case INTEL_PT_TRACESTOP:
2200         case INTEL_PT_PSB:
2201         case INTEL_PT_TSC:
2202         case INTEL_PT_TMA:
2203         case INTEL_PT_MODE_TSX:
2204         case INTEL_PT_BAD:
2205         case INTEL_PT_PSBEND:
2206         case INTEL_PT_PTWRITE:
2207         case INTEL_PT_PTWRITE_IP:
2208         case INTEL_PT_EXSTOP:
2209         case INTEL_PT_EXSTOP_IP:
2210         case INTEL_PT_MWAIT:
2211         case INTEL_PT_PWRE:
2212         case INTEL_PT_PWRX:
2213         case INTEL_PT_BBP:
2214         case INTEL_PT_BIP:
2215         case INTEL_PT_BEP:
2216         case INTEL_PT_BEP_IP:
2217         case INTEL_PT_CFE:
2218         case INTEL_PT_CFE_IP:
2219         case INTEL_PT_EVD:
2220             intel_pt_log("ERROR: Missing TIP after FUP\n");
2221             decoder->pkt_state = INTEL_PT_STATE_ERR3;
2222             decoder->pkt_step = 0;
2223             return -ENOENT;
2224
2225         case INTEL_PT_CBR:
2226             intel_pt_calc_cbr(decoder);
2227             break;
2228
2229         case INTEL_PT_OVF:
2230             return intel_pt_overflow(decoder);
2231
2232         case INTEL_PT_TIP_PGD:
2233             decoder->state.from_ip = decoder->ip;
2234             if (decoder->packet.count == 0) {
2235                 decoder->state.to_ip = 0;
2236             } else {
2237                 intel_pt_set_ip(decoder);
2238                 decoder->state.to_ip = decoder->ip;
2239             }
2240             decoder->pge = false;
2241             decoder->continuous_period = false;
2242             decoder->state.type |= INTEL_PT_TRACE_END;
2243             intel_pt_update_nr(decoder);
2244             return 0;
2245
2246         case INTEL_PT_TIP_PGE:
2247             decoder->pge = true;
2248             intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2249                      decoder->ip);
2250             decoder->state.from_ip = 0;
2251             if (decoder->packet.count == 0) {
2252                 decoder->state.to_ip = 0;
2253             } else {
2254                 intel_pt_set_ip(decoder);
2255                 decoder->state.to_ip = decoder->ip;
2256             }
2257             decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2258             intel_pt_mtc_cyc_cnt_pge(decoder);
2259             intel_pt_set_nr(decoder);
2260             return 0;
2261
2262         case INTEL_PT_TIP:
2263             decoder->state.from_ip = decoder->ip;
2264             if (decoder->packet.count == 0) {
2265                 decoder->state.to_ip = 0;
2266             } else {
2267                 intel_pt_set_ip(decoder);
2268                 decoder->state.to_ip = decoder->ip;
2269             }
2270             intel_pt_update_nr(decoder);
2271             intel_pt_sample_iflag_chg(decoder);
2272             return 0;
2273
2274         case INTEL_PT_PIP:
2275             intel_pt_update_pip(decoder);
2276             break;
2277
2278         case INTEL_PT_MTC:
2279             intel_pt_calc_mtc_timestamp(decoder);
2280             if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2281                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2282             break;
2283
2284         case INTEL_PT_CYC:
2285             intel_pt_calc_cyc_timestamp(decoder);
2286             break;
2287
2288         case INTEL_PT_MODE_EXEC:
2289             intel_pt_mode_exec(decoder);
2290             break;
2291
2292         case INTEL_PT_VMCS:
2293         case INTEL_PT_MNT:
2294         case INTEL_PT_PAD:
2295             break;
2296
2297         default:
2298             return intel_pt_bug(decoder);
2299         }
2300     }
2301 }
2302
2303 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2304 {
2305     decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2306     decoder->state.type = INTEL_PT_INSTRUCTION;
2307     decoder->state.from_ip = decoder->ip;
2308     decoder->state.to_ip = 0;
2309     return 0;
2310 }
2311
2312 struct intel_pt_vm_tsc_info {
2313     struct intel_pt_pkt pip_packet;
2314     struct intel_pt_pkt vmcs_packet;
2315     struct intel_pt_pkt tma_packet;
2316     bool tsc, pip, vmcs, tma, psbend;
2317     uint64_t ctc_delta;
2318     uint64_t last_ctc;
2319     int max_lookahead;
2320 };
2321
2322 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2323 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2324 {
2325     struct intel_pt_vm_tsc_info *data = pkt_info->data;
2326
2327     switch (pkt_info->packet.type) {
2328     case INTEL_PT_PAD:
2329     case INTEL_PT_MNT:
2330     case INTEL_PT_MODE_EXEC:
2331     case INTEL_PT_MODE_TSX:
2332     case INTEL_PT_MTC:
2333     case INTEL_PT_FUP:
2334     case INTEL_PT_CYC:
2335     case INTEL_PT_CBR:
2336         break;
2337
2338     case INTEL_PT_TSC:
2339         data->tsc = true;
2340         break;
2341
2342     case INTEL_PT_TMA:
2343         data->tma_packet = pkt_info->packet;
2344         data->tma = true;
2345         break;
2346
2347     case INTEL_PT_PIP:
2348         data->pip_packet = pkt_info->packet;
2349         data->pip = true;
2350         break;
2351
2352     case INTEL_PT_VMCS:
2353         data->vmcs_packet = pkt_info->packet;
2354         data->vmcs = true;
2355         break;
2356
2357     case INTEL_PT_PSBEND:
2358         data->psbend = true;
2359         return 1;
2360
2361     case INTEL_PT_TIP_PGE:
2362     case INTEL_PT_PTWRITE:
2363     case INTEL_PT_PTWRITE_IP:
2364     case INTEL_PT_EXSTOP:
2365     case INTEL_PT_EXSTOP_IP:
2366     case INTEL_PT_MWAIT:
2367     case INTEL_PT_PWRE:
2368     case INTEL_PT_PWRX:
2369     case INTEL_PT_BBP:
2370     case INTEL_PT_BIP:
2371     case INTEL_PT_BEP:
2372     case INTEL_PT_BEP_IP:
2373     case INTEL_PT_OVF:
2374     case INTEL_PT_BAD:
2375     case INTEL_PT_TNT:
2376     case INTEL_PT_TIP_PGD:
2377     case INTEL_PT_TIP:
2378     case INTEL_PT_PSB:
2379     case INTEL_PT_TRACESTOP:
2380     case INTEL_PT_CFE:
2381     case INTEL_PT_CFE_IP:
2382     case INTEL_PT_EVD:
2383     default:
2384         return 1;
2385     }
2386
2387     return 0;
2388 }
2389
2390 struct intel_pt_ovf_fup_info {
2391     int max_lookahead;
2392     bool found;
2393 };
2394
2395 /* Lookahead to detect a FUP packet after OVF */
2396 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2397 {
2398     struct intel_pt_ovf_fup_info *data = pkt_info->data;
2399
2400     if (pkt_info->packet.type == INTEL_PT_CYC ||
2401         pkt_info->packet.type == INTEL_PT_MTC ||
2402         pkt_info->packet.type == INTEL_PT_TSC)
2403         return !--(data->max_lookahead);
2404     data->found = pkt_info->packet.type == INTEL_PT_FUP;
2405     return 1;
2406 }
2407
2408 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2409 {
2410     struct intel_pt_ovf_fup_info data = {
2411         .max_lookahead = 16,
2412         .found = false,
2413     };
2414
2415     intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2416     return data.found;
2417 }
2418
2419 /* Lookahead and get the TMA packet after TSC */
2420 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2421 {
2422     struct intel_pt_vm_tsc_info *data = pkt_info->data;
2423
2424     if (pkt_info->packet.type == INTEL_PT_CYC ||
2425         pkt_info->packet.type == INTEL_PT_MTC)
2426         return !--(data->max_lookahead);
2427
2428     if (pkt_info->packet.type == INTEL_PT_TMA) {
2429         data->tma_packet = pkt_info->packet;
2430         data->tma = true;
2431     }
2432     return 1;
2433 }
2434
2435 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2436 {
2437     if (decoder->tsc_ctc_mult)
2438         return ctc * decoder->tsc_ctc_mult;
2439     else
2440         return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2441 }
2442
2443 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2444                        uint32_t ctc,
2445                        uint32_t fc,
2446                        uint64_t last_ctc_timestamp,
2447                        uint64_t ctc_delta,
2448                        uint32_t last_ctc)
2449 {
2450     /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2451     uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2452     /*
2453      * Number of CTC ticks from there until current TMA packet. We would
2454      * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2455      * past an MTC, so a sign-extended value is used.
2456      */
2457     uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2458     /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2459     uint64_t new_ctc_delta = ctc_delta + delta;
2460     uint64_t expected_tsc;
2461
2462     /*
2463      * Convert CTC ticks to TSC ticks, add the starting point
2464      * (last_ctc_timestamp) and the fast counter from the TMA packet.
2465      */
2466     expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2467
2468     if (intel_pt_enable_logging) {
2469         intel_pt_log_x64(last_mtc_ctc);
2470         intel_pt_log_x32(last_ctc);
2471         intel_pt_log_x64(ctc_delta);
2472         intel_pt_log_x64(delta);
2473         intel_pt_log_x32(ctc);
2474         intel_pt_log_x64(new_ctc_delta);
2475         intel_pt_log_x64(last_ctc_timestamp);
2476         intel_pt_log_x32(fc);
2477         intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2478         intel_pt_log_x64(expected_tsc);
2479     }
2480
2481     return expected_tsc;
2482 }
2483
2484 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2485                       struct intel_pt_vm_tsc_info *data)
2486 {
2487     uint32_t ctc = data->tma_packet.payload;
2488     uint32_t fc = data->tma_packet.count;
2489
2490     return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2491                       decoder->ctc_timestamp,
2492                       data->ctc_delta, data->last_ctc);
2493 }
2494
2495 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2496                       struct intel_pt_vmcs_info *vmcs_info)
2497 {
2498     uint64_t payload = decoder->packet.payload;
2499
2500     /* VMX adds the TSC Offset, so subtract to get host TSC */
2501     decoder->packet.payload -= vmcs_info->tsc_offset;
2502     /* TSC packet has only 7 bytes */
2503     decoder->packet.payload &= SEVEN_BYTES;
2504
2505     /*
2506      * The buffer is mmapped from the data file, so this also updates the
2507      * data file.
2508      */
2509     if (!decoder->vm_tm_corr_dry_run)
2510         memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2511
2512     intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2513              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2514              payload, decoder->packet.payload, vmcs_info->vmcs,
2515              vmcs_info->tsc_offset);
2516 }
2517
2518 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2519                          uint64_t tsc_offset)
2520 {
2521     struct intel_pt_vmcs_info vmcs_info = {
2522         .vmcs = NO_VMCS,
2523         .tsc_offset = tsc_offset
2524     };
2525
2526     intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2527 }
2528
2529 static inline bool in_vm(uint64_t pip_payload)
2530 {
2531     return pip_payload & 1;
2532 }
2533
2534 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2535 {
2536     return pip_packet->payload & 1;
2537 }
2538
2539 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2540 {
2541     p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2542           vmcs_info->vmcs, vmcs_info->tsc_offset);
2543 }
2544
2545 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2546                     struct intel_pt_vm_tsc_info *data)
2547 {
2548     memset(data, 0, sizeof(*data));
2549     data->ctc_delta = decoder->ctc_delta;
2550     data->last_ctc = decoder->last_ctc;
2551     intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2552     if (data->tsc && !data->psbend)
2553         p_log("ERROR: PSB without PSBEND");
2554     decoder->in_psb = data->psbend;
2555 }
2556
2557 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2558                       struct intel_pt_vm_tsc_info *data,
2559                       struct intel_pt_vmcs_info *vmcs_info,
2560                       uint64_t host_tsc)
2561 {
2562     if (!decoder->in_psb) {
2563         /* Can't happen */
2564         p_log("ERROR: First TSC is not in PSB+");
2565     }
2566
2567     if (data->pip) {
2568         if (pip_in_vm(&data->pip_packet)) { /* Guest */
2569             if (vmcs_info && vmcs_info->tsc_offset) {
2570                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2571                 decoder->vm_tm_corr_reliable = true;
2572             } else {
2573                 p_log("ERROR: First TSC, unknown TSC Offset");
2574             }
2575         } else { /* Host */
2576             decoder->vm_tm_corr_reliable = true;
2577         }
2578     } else { /* Host or Guest */
2579         decoder->vm_tm_corr_reliable = false;
2580         if (intel_pt_time_in_range(decoder, host_tsc)) {
2581             /* Assume Host */
2582         } else {
2583             /* Assume Guest */
2584             if (vmcs_info && vmcs_info->tsc_offset)
2585                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2586             else
2587                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2588         }
2589     }
2590 }
2591
2592 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2593                     struct intel_pt_vm_tsc_info *data)
2594 {
2595     struct intel_pt_vmcs_info *vmcs_info;
2596     uint64_t tsc_offset = 0;
2597     uint64_t vmcs;
2598     bool reliable = true;
2599     uint64_t expected_tsc;
2600     uint64_t host_tsc;
2601     uint64_t ref_timestamp;
2602
2603     bool assign = false;
2604     bool assign_reliable = false;
2605
2606     /* Already have 'data' for the in_psb case */
2607     if (!decoder->in_psb) {
2608         memset(data, 0, sizeof(*data));
2609         data->ctc_delta = decoder->ctc_delta;
2610         data->last_ctc = decoder->last_ctc;
2611         data->max_lookahead = 16;
2612         intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2613         if (decoder->pge) {
2614             data->pip = true;
2615             data->pip_packet.payload = decoder->pip_payload;
2616         }
2617     }
2618
2619     /* Calculations depend on having TMA packets */
2620     if (!data->tma) {
2621         p_log("ERROR: TSC without TMA");
2622         return;
2623     }
2624
2625     vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2626     if (vmcs == NO_VMCS)
2627         vmcs = 0;
2628
2629     vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2630
2631     ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2632     host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2633
2634     if (!decoder->ctc_timestamp) {
2635         intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2636         return;
2637     }
2638
2639     expected_tsc = intel_pt_expected_tsc(decoder, data);
2640
2641     tsc_offset = host_tsc - expected_tsc;
2642
2643     /* Determine if TSC is from Host or Guest */
2644     if (data->pip) {
2645         if (pip_in_vm(&data->pip_packet)) { /* Guest */
2646             if (!vmcs_info) {
2647                 /* PIP NR=1 without VMCS cannot happen */
2648                 p_log("ERROR: Missing VMCS");
2649                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2650                 decoder->vm_tm_corr_reliable = false;
2651                 return;
2652             }
2653         } else { /* Host */
2654             decoder->last_reliable_timestamp = host_tsc;
2655             decoder->vm_tm_corr_reliable = true;
2656             return;
2657         }
2658     } else { /* Host or Guest */
2659         reliable = false; /* Host/Guest is a guess, so not reliable */
2660         if (decoder->in_psb) {
2661             if (!tsc_offset)
2662                 return; /* Zero TSC Offset, assume Host */
2663             /*
2664              * TSC packet has only 7 bytes of TSC. We have no
2665              * information about the Guest's 8th byte, but it
2666              * doesn't matter because we only need 7 bytes.
2667              * Here, since the 8th byte is unreliable and
2668              * irrelevant, compare only 7 byes.
2669              */
2670             if (vmcs_info &&
2671                 (tsc_offset & SEVEN_BYTES) ==
2672                 (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2673                 /* Same TSC Offset as last VMCS, assume Guest */
2674                 goto guest;
2675             }
2676         }
2677         /*
2678          * Check if the host_tsc is within the expected range.
2679          * Note, we could narrow the range more by looking ahead for
2680          * the next host TSC in the same buffer, but we don't bother to
2681          * do that because this is probably good enough.
2682          */
2683         if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2684             /* Within expected range for Host TSC, assume Host */
2685             decoder->vm_tm_corr_reliable = false;
2686             return;
2687         }
2688     }
2689
2690 guest: /* Assuming Guest */
2691
2692     /* Determine whether to assign TSC Offset */
2693     if (vmcs_info && vmcs_info->vmcs) {
2694         if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2695             assign = false;
2696         } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2697                decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2698             /* Continuous tracing, TSC in a PSB is not a time loss */
2699             assign = true;
2700             assign_reliable = true;
2701         } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2702             /*
2703              * Unlikely to be a time loss TSC in a PSB which is not
2704              * at the start of a buffer.
2705              */
2706             assign = true;
2707             assign_reliable = false;
2708         }
2709     }
2710
2711     /* Record VMCS TSC Offset */
2712     if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2713                vmcs_info->reliable != assign_reliable)) {
2714         bool print = vmcs_info->tsc_offset != tsc_offset;
2715
2716         vmcs_info->tsc_offset = tsc_offset;
2717         vmcs_info->reliable = assign_reliable;
2718         if (print)
2719             intel_pt_print_vmcs_info(vmcs_info);
2720     }
2721
2722     /* Determine what TSC Offset to use */
2723     if (vmcs_info && vmcs_info->tsc_offset) {
2724         if (!vmcs_info->reliable)
2725             reliable = false;
2726         intel_pt_translate_vm_tsc(decoder, vmcs_info);
2727     } else {
2728         reliable = false;
2729         if (vmcs_info) {
2730             if (!vmcs_info->error_printed) {
2731                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2732                       vmcs_info->vmcs);
2733                 vmcs_info->error_printed = true;
2734             }
2735         } else {
2736             if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2737                 p_log("ERROR: Unknown VMCS");
2738         }
2739         intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2740     }
2741
2742     decoder->vm_tm_corr_reliable = reliable;
2743 }
2744
2745 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2746 {
2747     uint64_t host_tsc = decoder->packet.payload;
2748     uint64_t guest_tsc = decoder->packet.payload;
2749     struct intel_pt_vmcs_info *vmcs_info;
2750     uint64_t vmcs;
2751
2752     vmcs = decoder->vmcs;
2753     if (vmcs == NO_VMCS)
2754         vmcs = 0;
2755
2756     vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2757
2758     if (decoder->pge) {
2759         if (in_vm(decoder->pip_payload)) { /* Guest */
2760             if (!vmcs_info) {
2761                 /* PIP NR=1 without VMCS cannot happen */
2762                 p_log("ERROR: Missing VMCS");
2763             }
2764         } else { /* Host */
2765             return;
2766         }
2767     } else { /* Host or Guest */
2768         if (intel_pt_time_in_range(decoder, host_tsc)) {
2769             /* Within expected range for Host TSC, assume Host */
2770             return;
2771         }
2772     }
2773
2774     if (vmcs_info) {
2775         /* Translate Guest TSC to Host TSC */
2776         host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2777         host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2778         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2779                  "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2780                  guest_tsc, host_tsc, vmcs_info->vmcs,
2781                  vmcs_info->tsc_offset);
2782         if (!intel_pt_time_in_range(decoder, host_tsc) &&
2783             intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2784             p_log("Timestamp out of range");
2785     } else {
2786         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2787             p_log("ERROR: Unknown VMCS");
2788         host_tsc = decoder->timestamp;
2789     }
2790
2791     decoder->packet.payload = host_tsc;
2792
2793     if (!decoder->vm_tm_corr_dry_run)
2794         memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2795 }
2796
2797 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2798 {
2799     struct intel_pt_vm_tsc_info data = { .psbend = false };
2800     bool pge;
2801     int err;
2802
2803     if (decoder->in_psb)
2804         intel_pt_vm_tm_corr_psb(decoder, &data);
2805
2806     while (1) {
2807         err = intel_pt_get_next_packet(decoder);
2808         if (err == -ENOLINK)
2809             continue;
2810         if (err)
2811             break;
2812
2813         switch (decoder->packet.type) {
2814         case INTEL_PT_TIP_PGD:
2815             decoder->pge = false;
2816             decoder->vm_tm_corr_continuous = false;
2817             break;
2818
2819         case INTEL_PT_TNT:
2820         case INTEL_PT_TIP:
2821         case INTEL_PT_TIP_PGE:
2822             decoder->pge = true;
2823             break;
2824
2825         case INTEL_PT_OVF:
2826             decoder->in_psb = false;
2827             pge = decoder->pge;
2828             decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2829             if (pge != decoder->pge)
2830                 intel_pt_log("Surprising PGE change in OVF!");
2831             if (!decoder->pge)
2832                 decoder->vm_tm_corr_continuous = false;
2833             break;
2834
2835         case INTEL_PT_FUP:
2836             if (decoder->in_psb)
2837                 decoder->pge = true;
2838             break;
2839
2840         case INTEL_PT_TRACESTOP:
2841             decoder->pge = false;
2842             decoder->vm_tm_corr_continuous = false;
2843             decoder->have_tma = false;
2844             break;
2845
2846         case INTEL_PT_PSB:
2847             intel_pt_vm_tm_corr_psb(decoder, &data);
2848             break;
2849
2850         case INTEL_PT_PIP:
2851             decoder->pip_payload = decoder->packet.payload;
2852             break;
2853
2854         case INTEL_PT_MTC:
2855             intel_pt_calc_mtc_timestamp(decoder);
2856             break;
2857
2858         case INTEL_PT_TSC:
2859             intel_pt_vm_tm_corr_tsc(decoder, &data);
2860             intel_pt_calc_tsc_timestamp(decoder);
2861             decoder->vm_tm_corr_same_buf = true;
2862             decoder->vm_tm_corr_continuous = decoder->pge;
2863             break;
2864
2865         case INTEL_PT_TMA:
2866             intel_pt_calc_tma(decoder);
2867             break;
2868
2869         case INTEL_PT_CYC:
2870             intel_pt_calc_cyc_timestamp(decoder);
2871             break;
2872
2873         case INTEL_PT_CBR:
2874             intel_pt_calc_cbr(decoder);
2875             break;
2876
2877         case INTEL_PT_PSBEND:
2878             decoder->in_psb = false;
2879             data.psbend = false;
2880             break;
2881
2882         case INTEL_PT_VMCS:
2883             if (decoder->packet.payload != NO_VMCS)
2884                 decoder->vmcs = decoder->packet.payload;
2885             break;
2886
2887         case INTEL_PT_BBP:
2888             decoder->blk_type = decoder->packet.payload;
2889             break;
2890
2891         case INTEL_PT_BIP:
2892             if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2893                 decoder->packet.count == 2)
2894                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2895             break;
2896
2897         case INTEL_PT_BEP:
2898         case INTEL_PT_BEP_IP:
2899             decoder->blk_type = 0;
2900             break;
2901
2902         case INTEL_PT_CFE:
2903         case INTEL_PT_CFE_IP:
2904         case INTEL_PT_EVD:
2905         case INTEL_PT_MODE_EXEC:
2906         case INTEL_PT_MODE_TSX:
2907         case INTEL_PT_MNT:
2908         case INTEL_PT_PAD:
2909         case INTEL_PT_PTWRITE_IP:
2910         case INTEL_PT_PTWRITE:
2911         case INTEL_PT_MWAIT:
2912         case INTEL_PT_PWRE:
2913         case INTEL_PT_EXSTOP_IP:
2914         case INTEL_PT_EXSTOP:
2915         case INTEL_PT_PWRX:
2916         case INTEL_PT_BAD: /* Does not happen */
2917         default:
2918             break;
2919         }
2920     }
2921
2922     return err;
2923 }
2924
2925 #define HOP_PROCESS 0
2926 #define HOP_IGNORE  1
2927 #define HOP_RETURN  2
2928 #define HOP_AGAIN   3
2929
2930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2931
2932 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2933 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2934 {
2935     *err = 0;
2936
2937     /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2938     if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2939         *err = intel_pt_scan_for_psb(decoder);
2940         if (*err)
2941             return HOP_RETURN;
2942     }
2943
2944     switch (decoder->packet.type) {
2945     case INTEL_PT_TNT:
2946         return HOP_IGNORE;
2947
2948     case INTEL_PT_TIP_PGD:
2949         decoder->pge = false;
2950         if (!decoder->packet.count) {
2951             intel_pt_set_nr(decoder);
2952             return HOP_IGNORE;
2953         }
2954         intel_pt_set_ip(decoder);
2955         decoder->state.type |= INTEL_PT_TRACE_END;
2956         decoder->state.from_ip = 0;
2957         decoder->state.to_ip = decoder->ip;
2958         intel_pt_update_nr(decoder);
2959         return HOP_RETURN;
2960
2961     case INTEL_PT_TIP:
2962         if (!decoder->packet.count) {
2963             intel_pt_set_nr(decoder);
2964             return HOP_IGNORE;
2965         }
2966         intel_pt_set_ip(decoder);
2967         decoder->state.type = INTEL_PT_INSTRUCTION;
2968         decoder->state.from_ip = decoder->ip;
2969         decoder->state.to_ip = 0;
2970         intel_pt_update_nr(decoder);
2971         intel_pt_sample_iflag_chg(decoder);
2972         return HOP_RETURN;
2973
2974     case INTEL_PT_FUP:
2975         if (!decoder->packet.count)
2976             return HOP_IGNORE;
2977         intel_pt_set_ip(decoder);
2978         if (decoder->set_fup_mwait || decoder->set_fup_pwre)
2979             *no_tip = true;
2980         if (!decoder->branch_enable || !decoder->pge)
2981             *no_tip = true;
2982         if (*no_tip) {
2983             decoder->state.type = INTEL_PT_INSTRUCTION;
2984             decoder->state.from_ip = decoder->ip;
2985             decoder->state.to_ip = 0;
2986             intel_pt_fup_event(decoder, *no_tip);
2987             return HOP_RETURN;
2988         }
2989         intel_pt_fup_event(decoder, *no_tip);
2990         decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;
2991         *err = intel_pt_walk_fup_tip(decoder);
2992         if (!*err && decoder->state.to_ip)
2993             decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2994         return HOP_RETURN;
2995
2996     case INTEL_PT_PSB:
2997         decoder->state.psb_offset = decoder->pos;
2998         decoder->psb_ip = 0;
2999         decoder->last_ip = 0;
3000         decoder->have_last_ip = true;
3001         *err = intel_pt_walk_psbend(decoder);
3002         if (*err == -EAGAIN)
3003             return HOP_AGAIN;
3004         if (*err)
3005             return HOP_RETURN;
3006         decoder->state.type = INTEL_PT_PSB_EVT;
3007         if (decoder->psb_ip) {
3008             decoder->state.type |= INTEL_PT_INSTRUCTION;
3009             decoder->ip = decoder->psb_ip;
3010         }
3011         decoder->state.from_ip = decoder->psb_ip;
3012         decoder->state.to_ip = 0;
3013         return HOP_RETURN;
3014
3015     case INTEL_PT_BAD:
3016     case INTEL_PT_PAD:
3017     case INTEL_PT_TIP_PGE:
3018     case INTEL_PT_TSC:
3019     case INTEL_PT_TMA:
3020     case INTEL_PT_MODE_EXEC:
3021     case INTEL_PT_MODE_TSX:
3022     case INTEL_PT_MTC:
3023     case INTEL_PT_CYC:
3024     case INTEL_PT_VMCS:
3025     case INTEL_PT_PSBEND:
3026     case INTEL_PT_CBR:
3027     case INTEL_PT_TRACESTOP:
3028     case INTEL_PT_PIP:
3029     case INTEL_PT_OVF:
3030     case INTEL_PT_MNT:
3031     case INTEL_PT_PTWRITE:
3032     case INTEL_PT_PTWRITE_IP:
3033     case INTEL_PT_EXSTOP:
3034     case INTEL_PT_EXSTOP_IP:
3035     case INTEL_PT_MWAIT:
3036     case INTEL_PT_PWRE:
3037     case INTEL_PT_PWRX:
3038     case INTEL_PT_BBP:
3039     case INTEL_PT_BIP:
3040     case INTEL_PT_BEP:
3041     case INTEL_PT_BEP_IP:
3042     case INTEL_PT_CFE:
3043     case INTEL_PT_CFE_IP:
3044     case INTEL_PT_EVD:
3045     default:
3046         return HOP_PROCESS;
3047     }
3048 }
3049
3050 struct intel_pt_psb_info {
3051     struct intel_pt_pkt fup_packet;
3052     bool fup;
3053     int after_psbend;
3054 };
3055
3056 /* Lookahead and get the FUP packet from PSB+ */
3057 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
3058 {
3059     struct intel_pt_psb_info *data = pkt_info->data;
3060
3061     switch (pkt_info->packet.type) {
3062     case INTEL_PT_PAD:
3063     case INTEL_PT_MNT:
3064     case INTEL_PT_TSC:
3065     case INTEL_PT_TMA:
3066     case INTEL_PT_MODE_EXEC:
3067     case INTEL_PT_MODE_TSX:
3068     case INTEL_PT_MTC:
3069     case INTEL_PT_CYC:
3070     case INTEL_PT_VMCS:
3071     case INTEL_PT_CBR:
3072     case INTEL_PT_PIP:
3073         if (data->after_psbend) {
3074             data->after_psbend -= 1;
3075             if (!data->after_psbend)
3076                 return 1;
3077         }
3078         break;
3079
3080     case INTEL_PT_FUP:
3081         if (data->after_psbend)
3082             return 1;
3083         if (data->fup || pkt_info->packet.count == 0)
3084             return 1;
3085         data->fup_packet = pkt_info->packet;
3086         data->fup = true;
3087         break;
3088
3089     case INTEL_PT_PSBEND:
3090         if (!data->fup)
3091             return 1;
3092         /* Keep going to check for a TIP.PGE */
3093         data->after_psbend = 6;
3094         break;
3095
3096     case INTEL_PT_TIP_PGE:
3097         /* Ignore FUP in PSB+ if followed by TIP.PGE */
3098         if (data->after_psbend)
3099             data->fup = false;
3100         return 1;
3101
3102     case INTEL_PT_PTWRITE:
3103     case INTEL_PT_PTWRITE_IP:
3104     case INTEL_PT_EXSTOP:
3105     case INTEL_PT_EXSTOP_IP:
3106     case INTEL_PT_MWAIT:
3107     case INTEL_PT_PWRE:
3108     case INTEL_PT_PWRX:
3109     case INTEL_PT_BBP:
3110     case INTEL_PT_BIP:
3111     case INTEL_PT_BEP:
3112     case INTEL_PT_BEP_IP:
3113     case INTEL_PT_CFE:
3114     case INTEL_PT_CFE_IP:
3115     case INTEL_PT_EVD:
3116         if (data->after_psbend) {
3117             data->after_psbend -= 1;
3118             if (!data->after_psbend)
3119                 return 1;
3120             break;
3121         }
3122         return 1;
3123
3124     case INTEL_PT_OVF:
3125     case INTEL_PT_BAD:
3126     case INTEL_PT_TNT:
3127     case INTEL_PT_TIP_PGD:
3128     case INTEL_PT_TIP:
3129     case INTEL_PT_PSB:
3130     case INTEL_PT_TRACESTOP:
3131     default:
3132         return 1;
3133     }
3134
3135     return 0;
3136 }
3137
3138 static int intel_pt_psb(struct intel_pt_decoder *decoder)
3139 {
3140     int err;
3141
3142     decoder->last_ip = 0;
3143     decoder->psb_ip = 0;
3144     decoder->have_last_ip = true;
3145     intel_pt_clear_stack(&decoder->stack);
3146     err = intel_pt_walk_psbend(decoder);
3147     if (err)
3148         return err;
3149     decoder->state.type = INTEL_PT_PSB_EVT;
3150     decoder->state.from_ip = decoder->psb_ip;
3151     decoder->state.to_ip = 0;
3152     return 0;
3153 }
3154
3155 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
3156 {
3157     int err;
3158
3159     if (decoder->ip != decoder->last_ip) {
3160         err = intel_pt_walk_fup(decoder);
3161         if (!err || err != -EAGAIN)
3162             return err;
3163     }
3164
3165     decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3166     err = intel_pt_psb(decoder);
3167     if (err) {
3168         decoder->pkt_state = INTEL_PT_STATE_ERR3;
3169         return -ENOENT;
3170     }
3171
3172     return 0;
3173 }
3174
3175 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
3176 {
3177     struct intel_pt_psb_info data = { .fup = false };
3178
3179     if (!decoder->branch_enable)
3180         return false;
3181
3182     intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
3183     if (!data.fup)
3184         return false;
3185
3186     decoder->packet = data.fup_packet;
3187     intel_pt_set_last_ip(decoder);
3188     decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
3189
3190     *err = intel_pt_fup_in_psb(decoder);
3191
3192     return true;
3193 }
3194
3195 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
3196 {
3197     int last_packet_type = INTEL_PT_PAD;
3198     bool no_tip = false;
3199     int err;
3200
3201     while (1) {
3202         err = intel_pt_get_next_packet(decoder);
3203         if (err)
3204             return err;
3205 next:
3206         err = 0;
3207         if (decoder->cyc_threshold) {
3208             if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
3209                 decoder->sample_cyc = false;
3210             last_packet_type = decoder->packet.type;
3211         }
3212
3213         if (decoder->hop) {
3214             switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
3215             case HOP_IGNORE:
3216                 continue;
3217             case HOP_RETURN:
3218                 return err;
3219             case HOP_AGAIN:
3220                 goto next;
3221             default:
3222                 break;
3223             }
3224         }
3225
3226         switch (decoder->packet.type) {
3227         case INTEL_PT_TNT:
3228             if (!decoder->packet.count)
3229                 break;
3230             decoder->tnt = decoder->packet;
3231             decoder->pkt_state = INTEL_PT_STATE_TNT;
3232             err = intel_pt_walk_tnt(decoder);
3233             if (err == -EAGAIN)
3234                 break;
3235             return err;
3236
3237         case INTEL_PT_TIP_PGD:
3238             if (decoder->packet.count != 0)
3239                 intel_pt_set_last_ip(decoder);
3240             decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
3241             return intel_pt_walk_tip(decoder);
3242
3243         case INTEL_PT_TIP_PGE: {
3244             decoder->pge = true;
3245             decoder->overflow = false;
3246             intel_pt_mtc_cyc_cnt_pge(decoder);
3247             intel_pt_set_nr(decoder);
3248             if (decoder->packet.count == 0) {
3249                 intel_pt_log_at("Skipping zero TIP.PGE",
3250                         decoder->pos);
3251                 break;
3252             }
3253             intel_pt_sample_iflag_chg(decoder);
3254             intel_pt_set_ip(decoder);
3255             decoder->state.from_ip = 0;
3256             decoder->state.to_ip = decoder->ip;
3257             decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3258             /*
3259              * In hop mode, resample to get the to_ip as an
3260              * "instruction" sample.
3261              */
3262             if (decoder->hop)
3263                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3264             return 0;
3265         }
3266
3267         case INTEL_PT_OVF:
3268             return intel_pt_overflow(decoder);
3269
3270         case INTEL_PT_TIP:
3271             if (decoder->packet.count != 0)
3272                 intel_pt_set_last_ip(decoder);
3273             decoder->pkt_state = INTEL_PT_STATE_TIP;
3274             return intel_pt_walk_tip(decoder);
3275
3276         case INTEL_PT_FUP:
3277             if (decoder->packet.count == 0) {
3278                 intel_pt_log_at("Skipping zero FUP",
3279                         decoder->pos);
3280                 no_tip = false;
3281                 break;
3282             }
3283             intel_pt_set_last_ip(decoder);
3284             if (!decoder->branch_enable || !decoder->pge) {
3285                 decoder->ip = decoder->last_ip;
3286                 if (intel_pt_fup_event(decoder, no_tip))
3287                     return 0;
3288                 no_tip = false;
3289                 break;
3290             }
3291             if (decoder->set_fup_mwait)
3292                 no_tip = true;
3293             if (no_tip)
3294                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3295             else
3296                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3297             err = intel_pt_walk_fup(decoder);
3298             if (err != -EAGAIN)
3299                 return err;
3300             if (no_tip) {
3301                 no_tip = false;
3302                 break;
3303             }
3304             return intel_pt_walk_fup_tip(decoder);
3305
3306         case INTEL_PT_TRACESTOP:
3307             decoder->pge = false;
3308             decoder->continuous_period = false;
3309             intel_pt_clear_tx_flags(decoder);
3310             decoder->have_tma = false;
3311             break;
3312
3313         case INTEL_PT_PSB:
3314             decoder->state.psb_offset = decoder->pos;
3315             decoder->psb_ip = 0;
3316             if (intel_pt_psb_with_fup(decoder, &err))
3317                 return err;
3318             err = intel_pt_psb(decoder);
3319             if (err == -EAGAIN)
3320                 goto next;
3321             return err;
3322
3323         case INTEL_PT_PIP:
3324             intel_pt_update_pip(decoder);
3325             break;
3326
3327         case INTEL_PT_MTC:
3328             intel_pt_calc_mtc_timestamp(decoder);
3329             if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3330                 break;
3331             /*
3332              * Ensure that there has been an instruction since the
3333              * last MTC.
3334              */
3335             if (!decoder->mtc_insn)
3336                 break;
3337             decoder->mtc_insn = false;
3338             /* Ensure that there is a timestamp */
3339             if (!decoder->timestamp)
3340                 break;
3341             decoder->state.type = INTEL_PT_INSTRUCTION;
3342             decoder->state.from_ip = decoder->ip;
3343             decoder->state.to_ip = 0;
3344             decoder->mtc_insn = false;
3345             return 0;
3346
3347         case INTEL_PT_TSC:
3348             intel_pt_calc_tsc_timestamp(decoder);
3349             break;
3350
3351         case INTEL_PT_TMA:
3352             intel_pt_calc_tma(decoder);
3353             break;
3354
3355         case INTEL_PT_CYC:
3356             intel_pt_calc_cyc_timestamp(decoder);
3357             break;
3358
3359         case INTEL_PT_CBR:
3360             intel_pt_calc_cbr(decoder);
3361             if (decoder->cbr != decoder->cbr_seen) {
3362                 decoder->state.type = 0;
3363                 return 0;
3364             }
3365             break;
3366
3367         case INTEL_PT_MODE_EXEC:
3368             intel_pt_mode_exec(decoder);
3369             err = intel_pt_get_next_packet(decoder);
3370             if (err)
3371                 return err;
3372             if (decoder->packet.type == INTEL_PT_FUP) {
3373                 decoder->set_fup_mode_exec = true;
3374                 no_tip = true;
3375             }
3376             goto next;
3377
3378         case INTEL_PT_MODE_TSX:
3379             /* MODE_TSX need not be followed by FUP */
3380             if (!decoder->pge || decoder->in_psb) {
3381                 intel_pt_update_in_tx(decoder);
3382                 break;
3383             }
3384             err = intel_pt_mode_tsx(decoder, &no_tip);
3385             if (err)
3386                 return err;
3387             goto next;
3388
3389         case INTEL_PT_BAD: /* Does not happen */
3390             return intel_pt_bug(decoder);
3391
3392         case INTEL_PT_PSBEND:
3393         case INTEL_PT_VMCS:
3394         case INTEL_PT_MNT:
3395         case INTEL_PT_PAD:
3396             break;
3397
3398         case INTEL_PT_PTWRITE_IP:
3399             decoder->fup_ptw_payload = decoder->packet.payload;
3400             err = intel_pt_get_next_packet(decoder);
3401             if (err)
3402                 return err;
3403             if (decoder->packet.type == INTEL_PT_FUP) {
3404                 decoder->set_fup_ptw = true;
3405                 no_tip = true;
3406             } else {
3407                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3408                         decoder->pos);
3409             }
3410             goto next;
3411
3412         case INTEL_PT_PTWRITE:
3413             decoder->state.type = INTEL_PT_PTW;
3414             decoder->state.from_ip = decoder->ip;
3415             decoder->state.to_ip = 0;
3416             decoder->state.ptw_payload = decoder->packet.payload;
3417             return 0;
3418
3419         case INTEL_PT_MWAIT:
3420             decoder->fup_mwait_payload = decoder->packet.payload;
3421             decoder->set_fup_mwait = true;
3422             break;
3423
3424         case INTEL_PT_PWRE:
3425             if (decoder->set_fup_mwait) {
3426                 decoder->fup_pwre_payload =
3427                             decoder->packet.payload;
3428                 decoder->set_fup_pwre = true;
3429                 break;
3430             }
3431             decoder->state.type = INTEL_PT_PWR_ENTRY;
3432             decoder->state.from_ip = decoder->ip;
3433             decoder->state.to_ip = 0;
3434             decoder->state.pwrx_payload = decoder->packet.payload;
3435             return 0;
3436
3437         case INTEL_PT_EXSTOP_IP:
3438             err = intel_pt_get_next_packet(decoder);
3439             if (err)
3440                 return err;
3441             if (decoder->packet.type == INTEL_PT_FUP) {
3442                 decoder->set_fup_exstop = true;
3443                 no_tip = true;
3444             } else {
3445                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3446                         decoder->pos);
3447             }
3448             goto next;
3449
3450         case INTEL_PT_EXSTOP:
3451             decoder->state.type = INTEL_PT_EX_STOP;
3452             decoder->state.from_ip = decoder->ip;
3453             decoder->state.to_ip = 0;
3454             return 0;
3455
3456         case INTEL_PT_PWRX:
3457             decoder->state.type = INTEL_PT_PWR_EXIT;
3458             decoder->state.from_ip = decoder->ip;
3459             decoder->state.to_ip = 0;
3460             decoder->state.pwrx_payload = decoder->packet.payload;
3461             return 0;
3462
3463         case INTEL_PT_BBP:
3464             intel_pt_bbp(decoder);
3465             break;
3466
3467         case INTEL_PT_BIP:
3468             intel_pt_bip(decoder);
3469             break;
3470
3471         case INTEL_PT_BEP:
3472             decoder->state.type = INTEL_PT_BLK_ITEMS;
3473             decoder->state.from_ip = decoder->ip;
3474             decoder->state.to_ip = 0;
3475             return 0;
3476
3477         case INTEL_PT_BEP_IP:
3478             err = intel_pt_get_next_packet(decoder);
3479             if (err)
3480                 return err;
3481             if (decoder->packet.type == INTEL_PT_FUP) {
3482                 decoder->set_fup_bep = true;
3483                 no_tip = true;
3484             } else {
3485                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3486                         decoder->pos);
3487             }
3488             goto next;
3489
3490         case INTEL_PT_CFE:
3491             decoder->fup_cfe_pkt = decoder->packet;
3492             decoder->set_fup_cfe = true;
3493             if (!decoder->pge) {
3494                 intel_pt_fup_event(decoder, true);
3495                 return 0;
3496             }
3497             break;
3498
3499         case INTEL_PT_CFE_IP:
3500             decoder->fup_cfe_pkt = decoder->packet;
3501             err = intel_pt_get_next_packet(decoder);
3502             if (err)
3503                 return err;
3504             if (decoder->packet.type == INTEL_PT_FUP) {
3505                 decoder->set_fup_cfe_ip = true;
3506                 no_tip = true;
3507             } else {
3508                 intel_pt_log_at("ERROR: Missing FUP after CFE",
3509                         decoder->pos);
3510             }
3511             goto next;
3512
3513         case INTEL_PT_EVD:
3514             err = intel_pt_evd(decoder);
3515             if (err)
3516                 return err;
3517             break;
3518
3519         default:
3520             return intel_pt_bug(decoder);
3521         }
3522     }
3523 }
3524
3525 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3526 {
3527     return decoder->packet.count &&
3528            (decoder->have_last_ip || decoder->packet.count == 3 ||
3529         decoder->packet.count == 6);
3530 }
3531
3532 /* Walk PSB+ packets to get in sync. */
3533 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3534 {
3535     int err;
3536
3537     decoder->in_psb = true;
3538
3539     while (1) {
3540         err = intel_pt_get_next_packet(decoder);
3541         if (err)
3542             goto out;
3543
3544         switch (decoder->packet.type) {
3545         case INTEL_PT_TIP_PGD:
3546             decoder->continuous_period = false;
3547             __fallthrough;
3548         case INTEL_PT_TIP_PGE:
3549         case INTEL_PT_TIP:
3550         case INTEL_PT_PTWRITE:
3551         case INTEL_PT_PTWRITE_IP:
3552         case INTEL_PT_EXSTOP:
3553         case INTEL_PT_EXSTOP_IP:
3554         case INTEL_PT_MWAIT:
3555         case INTEL_PT_PWRE:
3556         case INTEL_PT_PWRX:
3557         case INTEL_PT_BBP:
3558         case INTEL_PT_BIP:
3559         case INTEL_PT_BEP:
3560         case INTEL_PT_BEP_IP:
3561         case INTEL_PT_CFE:
3562         case INTEL_PT_CFE_IP:
3563         case INTEL_PT_EVD:
3564             intel_pt_log("ERROR: Unexpected packet\n");
3565             err = -ENOENT;
3566             goto out;
3567
3568         case INTEL_PT_FUP:
3569             decoder->pge = true;
3570             if (intel_pt_have_ip(decoder)) {
3571                 uint64_t current_ip = decoder->ip;
3572
3573                 intel_pt_set_ip(decoder);
3574                 decoder->psb_ip = decoder->ip;
3575                 if (current_ip)
3576                     intel_pt_log_to("Setting IP",
3577                             decoder->ip);
3578             }
3579             break;
3580
3581         case INTEL_PT_MTC:
3582             intel_pt_calc_mtc_timestamp(decoder);
3583             break;
3584
3585         case INTEL_PT_TSC:
3586             intel_pt_calc_tsc_timestamp(decoder);
3587             break;
3588
3589         case INTEL_PT_TMA:
3590             intel_pt_calc_tma(decoder);
3591             break;
3592
3593         case INTEL_PT_CYC:
3594             intel_pt_calc_cyc_timestamp(decoder);
3595             break;
3596
3597         case INTEL_PT_CBR:
3598             intel_pt_calc_cbr(decoder);
3599             break;
3600
3601         case INTEL_PT_PIP:
3602             intel_pt_set_pip(decoder);
3603             break;
3604
3605         case INTEL_PT_MODE_EXEC:
3606             intel_pt_mode_exec_status(decoder);
3607             break;
3608
3609         case INTEL_PT_MODE_TSX:
3610             intel_pt_update_in_tx(decoder);
3611             break;
3612
3613         case INTEL_PT_TRACESTOP:
3614             decoder->pge = false;
3615             decoder->continuous_period = false;
3616             intel_pt_clear_tx_flags(decoder);
3617             __fallthrough;
3618
3619         case INTEL_PT_TNT:
3620             decoder->have_tma = false;
3621             intel_pt_log("ERROR: Unexpected packet\n");
3622             if (decoder->ip)
3623                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3624             else
3625                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3626             err = -ENOENT;
3627             goto out;
3628
3629         case INTEL_PT_BAD: /* Does not happen */
3630             err = intel_pt_bug(decoder);
3631             goto out;
3632
3633         case INTEL_PT_OVF:
3634             err = intel_pt_overflow(decoder);
3635             goto out;
3636
3637         case INTEL_PT_PSBEND:
3638             err = 0;
3639             goto out;
3640
3641         case INTEL_PT_PSB:
3642         case INTEL_PT_VMCS:
3643         case INTEL_PT_MNT:
3644         case INTEL_PT_PAD:
3645         default:
3646             break;
3647         }
3648     }
3649 out:
3650     decoder->in_psb = false;
3651
3652     return err;
3653 }
3654
3655 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3656 {
3657     int err;
3658
3659     while (1) {
3660         err = intel_pt_get_next_packet(decoder);
3661         if (err)
3662             return err;
3663
3664         switch (decoder->packet.type) {
3665         case INTEL_PT_TIP_PGD:
3666             decoder->continuous_period = false;
3667             decoder->pge = false;
3668             if (intel_pt_have_ip(decoder))
3669                 intel_pt_set_ip(decoder);
3670             if (!decoder->ip)
3671                 break;
3672             decoder->state.type |= INTEL_PT_TRACE_END;
3673             return 0;
3674
3675         case INTEL_PT_TIP_PGE:
3676             decoder->pge = true;
3677             intel_pt_mtc_cyc_cnt_pge(decoder);
3678             if (intel_pt_have_ip(decoder))
3679                 intel_pt_set_ip(decoder);
3680             if (!decoder->ip)
3681                 break;
3682             decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3683             return 0;
3684
3685         case INTEL_PT_TIP:
3686             decoder->pge = true;
3687             if (intel_pt_have_ip(decoder))
3688                 intel_pt_set_ip(decoder);
3689             if (!decoder->ip)
3690                 break;
3691             return 0;
3692
3693         case INTEL_PT_FUP:
3694             if (intel_pt_have_ip(decoder))
3695                 intel_pt_set_ip(decoder);
3696             if (decoder->ip)
3697                 return 0;
3698             break;
3699
3700         case INTEL_PT_MTC:
3701             intel_pt_calc_mtc_timestamp(decoder);
3702             break;
3703
3704         case INTEL_PT_TSC:
3705             intel_pt_calc_tsc_timestamp(decoder);
3706             break;
3707
3708         case INTEL_PT_TMA:
3709             intel_pt_calc_tma(decoder);
3710             break;
3711
3712         case INTEL_PT_CYC:
3713             intel_pt_calc_cyc_timestamp(decoder);
3714             break;
3715
3716         case INTEL_PT_CBR:
3717             intel_pt_calc_cbr(decoder);
3718             break;
3719
3720         case INTEL_PT_PIP:
3721             intel_pt_set_pip(decoder);
3722             break;
3723
3724         case INTEL_PT_MODE_EXEC:
3725             intel_pt_mode_exec_status(decoder);
3726             break;
3727
3728         case INTEL_PT_MODE_TSX:
3729             intel_pt_update_in_tx(decoder);
3730             break;
3731
3732         case INTEL_PT_OVF:
3733             return intel_pt_overflow(decoder);
3734
3735         case INTEL_PT_BAD: /* Does not happen */
3736             return intel_pt_bug(decoder);
3737
3738         case INTEL_PT_TRACESTOP:
3739             decoder->pge = false;
3740             decoder->continuous_period = false;
3741             intel_pt_clear_tx_flags(decoder);
3742             decoder->have_tma = false;
3743             break;
3744
3745         case INTEL_PT_PSB:
3746             decoder->state.psb_offset = decoder->pos;
3747             decoder->psb_ip = 0;
3748             decoder->last_ip = 0;
3749             decoder->have_last_ip = true;
3750             intel_pt_clear_stack(&decoder->stack);
3751             err = intel_pt_walk_psb(decoder);
3752             if (err)
3753                 return err;
3754             decoder->state.type = INTEL_PT_PSB_EVT;
3755             decoder->state.from_ip = decoder->psb_ip;
3756             decoder->state.to_ip = 0;
3757             return 0;
3758
3759         case INTEL_PT_TNT:
3760         case INTEL_PT_PSBEND:
3761         case INTEL_PT_VMCS:
3762         case INTEL_PT_MNT:
3763         case INTEL_PT_PAD:
3764         case INTEL_PT_PTWRITE:
3765         case INTEL_PT_PTWRITE_IP:
3766         case INTEL_PT_EXSTOP:
3767         case INTEL_PT_EXSTOP_IP:
3768         case INTEL_PT_MWAIT:
3769         case INTEL_PT_PWRE:
3770         case INTEL_PT_PWRX:
3771         case INTEL_PT_BBP:
3772         case INTEL_PT_BIP:
3773         case INTEL_PT_BEP:
3774         case INTEL_PT_BEP_IP:
3775         case INTEL_PT_CFE:
3776         case INTEL_PT_CFE_IP:
3777         case INTEL_PT_EVD:
3778         default:
3779             break;
3780         }
3781     }
3782 }
3783
3784 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3785 {
3786     int err;
3787
3788     intel_pt_clear_fup_event(decoder);
3789     decoder->overflow = false;
3790
3791     if (!decoder->branch_enable) {
3792         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3793         decoder->state.type = 0; /* Do not have a sample */
3794         return 0;
3795     }
3796
3797     intel_pt_log("Scanning for full IP\n");
3798     err = intel_pt_walk_to_ip(decoder);
3799     if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3800         return err;
3801
3802     /* In hop mode, resample to get the to_ip as an "instruction" sample */
3803     if (decoder->hop)
3804         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3805     else
3806         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3807
3808     decoder->state.from_ip = 0;
3809     decoder->state.to_ip = decoder->ip;
3810     intel_pt_log_to("Setting IP", decoder->ip);
3811
3812     return 0;
3813 }
3814
3815 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3816 {
3817     const unsigned char *end = decoder->buf + decoder->len;
3818     size_t i;
3819
3820     for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3821         if (i > decoder->len)
3822             continue;
3823         if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3824             return i;
3825     }
3826     return 0;
3827 }
3828
3829 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3830 {
3831     size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3832     const char *psb = INTEL_PT_PSB_STR;
3833
3834     if (rest_psb > decoder->len ||
3835         memcmp(decoder->buf, psb + part_psb, rest_psb))
3836         return 0;
3837
3838     return rest_psb;
3839 }
3840
3841 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3842                   int part_psb)
3843 {
3844     int rest_psb, ret;
3845
3846     decoder->pos += decoder->len;
3847     decoder->len = 0;
3848
3849     ret = intel_pt_get_next_data(decoder, false);
3850     if (ret)
3851         return ret;
3852
3853     rest_psb = intel_pt_rest_psb(decoder, part_psb);
3854     if (!rest_psb)
3855         return 0;
3856
3857     decoder->pos -= part_psb;
3858     decoder->next_buf = decoder->buf + rest_psb;
3859     decoder->next_len = decoder->len - rest_psb;
3860     memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3861     decoder->buf = decoder->temp_buf;
3862     decoder->len = INTEL_PT_PSB_LEN;
3863
3864     return 0;
3865 }
3866
3867 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3868 {
3869     unsigned char *next;
3870     int ret;
3871
3872     intel_pt_log("Scanning for PSB\n");
3873     while (1) {
3874         if (!decoder->len) {
3875             ret = intel_pt_get_next_data(decoder, false);
3876             if (ret)
3877                 return ret;
3878         }
3879
3880         next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3881                   INTEL_PT_PSB_LEN);
3882         if (!next) {
3883             int part_psb;
3884
3885             part_psb = intel_pt_part_psb(decoder);
3886             if (part_psb) {
3887                 ret = intel_pt_get_split_psb(decoder, part_psb);
3888                 if (ret)
3889                     return ret;
3890             } else {
3891                 decoder->pos += decoder->len;
3892                 decoder->len = 0;
3893             }
3894             continue;
3895         }
3896
3897         decoder->pkt_step = next - decoder->buf;
3898         return intel_pt_get_next_packet(decoder);
3899     }
3900 }
3901
3902 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3903 {
3904     int err;
3905
3906     decoder->pge = false;
3907     decoder->continuous_period = false;
3908     decoder->have_last_ip = false;
3909     decoder->last_ip = 0;
3910     decoder->psb_ip = 0;
3911     decoder->ip = 0;
3912     intel_pt_clear_stack(&decoder->stack);
3913
3914     err = intel_pt_scan_for_psb(decoder);
3915     if (err)
3916         return err;
3917
3918     if (decoder->vm_time_correlation) {
3919         decoder->in_psb = true;
3920         if (!decoder->timestamp)
3921             decoder->timestamp = 1;
3922         decoder->state.type = 0;
3923         decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3924         return 0;
3925     }
3926
3927     decoder->have_last_ip = true;
3928     decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3929
3930     err = intel_pt_walk_psb(decoder);
3931     if (err)
3932         return err;
3933
3934     decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3935     decoder->state.from_ip = decoder->psb_ip;
3936     decoder->state.to_ip = 0;
3937
3938     if (decoder->ip) {
3939         /*
3940          * In hop mode, resample to get the PSB FUP ip as an
3941          * "instruction" sample.
3942          */
3943         if (decoder->hop)
3944             decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3945         else
3946             decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3947     }
3948
3949     return 0;
3950 }
3951
3952 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3953 {
3954     uint64_t est = decoder->sample_insn_cnt << 1;
3955
3956     if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3957         goto out;
3958
3959     est *= decoder->max_non_turbo_ratio;
3960     est /= decoder->cbr;
3961 out:
3962     return decoder->sample_timestamp + est;
3963 }
3964
3965 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3966 {
3967     int err;
3968
3969     do {
3970         decoder->state.type = INTEL_PT_BRANCH;
3971         decoder->state.flags = 0;
3972
3973         switch (decoder->pkt_state) {
3974         case INTEL_PT_STATE_NO_PSB:
3975             err = intel_pt_sync(decoder);
3976             break;
3977         case INTEL_PT_STATE_NO_IP:
3978             decoder->have_last_ip = false;
3979             decoder->last_ip = 0;
3980             decoder->ip = 0;
3981             __fallthrough;
3982         case INTEL_PT_STATE_ERR_RESYNC:
3983             err = intel_pt_sync_ip(decoder);
3984             break;
3985         case INTEL_PT_STATE_IN_SYNC:
3986             err = intel_pt_walk_trace(decoder);
3987             break;
3988         case INTEL_PT_STATE_TNT:
3989         case INTEL_PT_STATE_TNT_CONT:
3990             err = intel_pt_walk_tnt(decoder);
3991             if (err == -EAGAIN)
3992                 err = intel_pt_walk_trace(decoder);
3993             break;
3994         case INTEL_PT_STATE_TIP:
3995         case INTEL_PT_STATE_TIP_PGD:
3996             err = intel_pt_walk_tip(decoder);
3997             break;
3998         case INTEL_PT_STATE_FUP:
3999             err = intel_pt_walk_fup(decoder);
4000             if (err == -EAGAIN)
4001                 err = intel_pt_walk_fup_tip(decoder);
4002             break;
4003         case INTEL_PT_STATE_FUP_NO_TIP:
4004             err = intel_pt_walk_fup(decoder);
4005             if (err == -EAGAIN)
4006                 err = intel_pt_walk_trace(decoder);
4007             break;
4008         case INTEL_PT_STATE_FUP_IN_PSB:
4009             err = intel_pt_fup_in_psb(decoder);
4010             break;
4011         case INTEL_PT_STATE_RESAMPLE:
4012             err = intel_pt_resample(decoder);
4013             break;
4014         case INTEL_PT_STATE_VM_TIME_CORRELATION:
4015             err = intel_pt_vm_time_correlation(decoder);
4016             break;
4017         default:
4018             err = intel_pt_bug(decoder);
4019             break;
4020         }
4021     } while (err == -ENOLINK);
4022
4023     if (err) {
4024         decoder->state.err = intel_pt_ext_err(err);
4025         if (err != -EOVERFLOW)
4026             decoder->state.from_ip = decoder->ip;
4027         intel_pt_update_sample_time(decoder);
4028         decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4029         intel_pt_set_nr(decoder);
4030     } else {
4031         decoder->state.err = 0;
4032         if (decoder->cbr != decoder->cbr_seen) {
4033             decoder->cbr_seen = decoder->cbr;
4034             if (!decoder->state.type) {
4035                 decoder->state.from_ip = decoder->ip;
4036                 decoder->state.to_ip = 0;
4037             }
4038             decoder->state.type |= INTEL_PT_CBR_CHG;
4039             decoder->state.cbr_payload = decoder->cbr_payload;
4040             decoder->state.cbr = decoder->cbr;
4041         }
4042         if (intel_pt_sample_time(decoder->pkt_state)) {
4043             intel_pt_update_sample_time(decoder);
4044             if (decoder->sample_cyc) {
4045                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4046                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4047                 decoder->sample_cyc = false;
4048             }
4049         }
4050         /*
4051          * When using only TSC/MTC to compute cycles, IPC can be
4052          * sampled as soon as the cycle count changes.
4053          */
4054         if (!decoder->have_cyc)
4055             decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4056     }
4057
4058      /* Let PSB event always have TSC timestamp */
4059     if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
4060         decoder->sample_timestamp = decoder->tsc_timestamp;
4061
4062     decoder->state.from_nr = decoder->nr;
4063     decoder->state.to_nr = decoder->next_nr;
4064     decoder->nr = decoder->next_nr;
4065
4066     decoder->state.timestamp = decoder->sample_timestamp;
4067     decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
4068     decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
4069     decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
4070
4071     return &decoder->state;
4072 }
4073
4074 /**
4075  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
4076  * @buf: pointer to buffer pointer
4077  * @len: size of buffer
4078  *
4079  * Updates the buffer pointer to point to the start of the next PSB packet if
4080  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
4081  * @len is adjusted accordingly.
4082  *
4083  * Return: %true if a PSB packet is found, %false otherwise.
4084  */
4085 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
4086 {
4087     unsigned char *next;
4088
4089     next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4090     if (next) {
4091         *len -= next - *buf;
4092         *buf = next;
4093         return true;
4094     }
4095     return false;
4096 }
4097
4098 /**
4099  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
4100  *                     packet.
4101  * @buf: pointer to buffer pointer
4102  * @len: size of buffer
4103  *
4104  * Updates the buffer pointer to point to the start of the following PSB packet
4105  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
4106  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
4107  *
4108  * Return: %true if a PSB packet is found, %false otherwise.
4109  */
4110 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
4111 {
4112     unsigned char *next;
4113
4114     if (!*len)
4115         return false;
4116
4117     next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4118     if (next) {
4119         *len -= next - *buf;
4120         *buf = next;
4121         return true;
4122     }
4123     return false;
4124 }
4125
4126 /**
4127  * intel_pt_last_psb - find the last PSB packet in a buffer.
4128  * @buf: buffer
4129  * @len: size of buffer
4130  *
4131  * This function finds the last PSB in a buffer.
4132  *
4133  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
4134  */
4135 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
4136 {
4137     const char *n = INTEL_PT_PSB_STR;
4138     unsigned char *p;
4139     size_t k;
4140
4141     if (len < INTEL_PT_PSB_LEN)
4142         return NULL;
4143
4144     k = len - INTEL_PT_PSB_LEN + 1;
4145     while (1) {
4146         p = memrchr(buf, n[0], k);
4147         if (!p)
4148             return NULL;
4149         if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
4150             return p;
4151         k = p - buf;
4152         if (!k)
4153             return NULL;
4154     }
4155 }
4156
4157 /**
4158  * intel_pt_next_tsc - find and return next TSC.
4159  * @buf: buffer
4160  * @len: size of buffer
4161  * @tsc: TSC value returned
4162  * @rem: returns remaining size when TSC is found
4163  *
4164  * Find a TSC packet in @buf and return the TSC value.  This function assumes
4165  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
4166  * PSBEND packet is found.
4167  *
4168  * Return: %true if TSC is found, false otherwise.
4169  */
4170 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
4171                   size_t *rem)
4172 {
4173     enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
4174     struct intel_pt_pkt packet;
4175     int ret;
4176
4177     while (len) {
4178         ret = intel_pt_get_packet(buf, len, &packet, &ctx);
4179         if (ret <= 0)
4180             return false;
4181         if (packet.type == INTEL_PT_TSC) {
4182             *tsc = packet.payload;
4183             *rem = len;
4184             return true;
4185         }
4186         if (packet.type == INTEL_PT_PSBEND)
4187             return false;
4188         buf += ret;
4189         len -= ret;
4190     }
4191     return false;
4192 }
4193
4194 /**
4195  * intel_pt_tsc_cmp - compare 7-byte TSCs.
4196  * @tsc1: first TSC to compare
4197  * @tsc2: second TSC to compare
4198  *
4199  * This function compares 7-byte TSC values allowing for the possibility that
4200  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
4201  * around so for that purpose this function assumes the absolute difference is
4202  * less than half the maximum difference.
4203  *
4204  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
4205  * after @tsc2.
4206  */
4207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
4208 {
4209     const uint64_t halfway = (1ULL << 55);
4210
4211     if (tsc1 == tsc2)
4212         return 0;
4213
4214     if (tsc1 < tsc2) {
4215         if (tsc2 - tsc1 < halfway)
4216             return -1;
4217         else
4218             return 1;
4219     } else {
4220         if (tsc1 - tsc2 < halfway)
4221             return 1;
4222         else
4223             return -1;
4224     }
4225 }
4226
4227 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
4228
4229 /**
4230  * adj_for_padding - adjust overlap to account for padding.
4231  * @buf_b: second buffer
4232  * @buf_a: first buffer
4233  * @len_a: size of first buffer
4234  *
4235  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
4236  * accordingly.
4237  *
4238  * Return: A pointer into @buf_b from where non-overlapped data starts
4239  */
4240 static unsigned char *adj_for_padding(unsigned char *buf_b,
4241                       unsigned char *buf_a, size_t len_a)
4242 {
4243     unsigned char *p = buf_b - MAX_PADDING;
4244     unsigned char *q = buf_a + len_a - MAX_PADDING;
4245     int i;
4246
4247     for (i = MAX_PADDING; i; i--, p++, q++) {
4248         if (*p != *q)
4249             break;
4250     }
4251
4252     return p;
4253 }
4254
4255 /**
4256  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
4257  *                             using TSC.
4258  * @buf_a: first buffer
4259  * @len_a: size of first buffer
4260  * @buf_b: second buffer
4261  * @len_b: size of second buffer
4262  * @consecutive: returns true if there is data in buf_b that is consecutive
4263  *               to buf_a
4264  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4265  *
4266  * If the trace contains TSC we can look at the last TSC of @buf_a and the
4267  * first TSC of @buf_b in order to determine if the buffers overlap, and then
4268  * walk forward in @buf_b until a later TSC is found.  A precondition is that
4269  * @buf_a and @buf_b are positioned at a PSB.
4270  *
4271  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4272  * @buf_b + @len_b if there is no non-overlapped data.
4273  */
4274 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
4275                         size_t len_a,
4276                         unsigned char *buf_b,
4277                         size_t len_b, bool *consecutive,
4278                         bool ooo_tsc)
4279 {
4280     uint64_t tsc_a, tsc_b;
4281     unsigned char *p;
4282     size_t len, rem_a, rem_b;
4283
4284     p = intel_pt_last_psb(buf_a, len_a);
4285     if (!p)
4286         return buf_b; /* No PSB in buf_a => no overlap */
4287
4288     len = len_a - (p - buf_a);
4289     if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
4290         /* The last PSB+ in buf_a is incomplete, so go back one more */
4291         len_a -= len;
4292         p = intel_pt_last_psb(buf_a, len_a);
4293         if (!p)
4294             return buf_b; /* No full PSB+ => assume no overlap */
4295         len = len_a - (p - buf_a);
4296         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
4297             return buf_b; /* No TSC in buf_a => assume no overlap */
4298     }
4299
4300     while (1) {
4301         /* Ignore PSB+ with no TSC */
4302         if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4303             int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4304
4305             /* Same TSC, so buffers are consecutive */
4306             if (!cmp && rem_b >= rem_a) {
4307                 unsigned char *start;
4308
4309                 *consecutive = true;
4310                 start = buf_b + len_b - (rem_b - rem_a);
4311                 return adj_for_padding(start, buf_a, len_a);
4312             }
4313             if (cmp < 0 && !ooo_tsc)
4314                 return buf_b; /* tsc_a < tsc_b => no overlap */
4315         }
4316
4317         if (!intel_pt_step_psb(&buf_b, &len_b))
4318             return buf_b + len_b; /* No PSB in buf_b => no data */
4319     }
4320 }
4321
4322 /**
4323  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4324  * @buf_a: first buffer
4325  * @len_a: size of first buffer
4326  * @buf_b: second buffer
4327  * @len_b: size of second buffer
4328  * @have_tsc: can use TSC packets to detect overlap
4329  * @consecutive: returns true if there is data in buf_b that is consecutive
4330  *               to buf_a
4331  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4332  *
4333  * When trace samples or snapshots are recorded there is the possibility that
4334  * the data overlaps.  Note that, for the purposes of decoding, data is only
4335  * useful if it begins with a PSB packet.
4336  *
4337  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4338  * @buf_b + @len_b if there is no non-overlapped data.
4339  */
4340 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4341                      unsigned char *buf_b, size_t len_b,
4342                      bool have_tsc, bool *consecutive,
4343                      bool ooo_tsc)
4344 {
4345     unsigned char *found;
4346
4347     /* Buffer 'b' must start at PSB so throw away everything before that */
4348     if (!intel_pt_next_psb(&buf_b, &len_b))
4349         return buf_b + len_b; /* No PSB */
4350
4351     if (!intel_pt_next_psb(&buf_a, &len_a))
4352         return buf_b; /* No overlap */
4353
4354     if (have_tsc) {
4355         found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4356                           consecutive, ooo_tsc);
4357         if (found)
4358             return found;
4359     }
4360
4361     /*
4362      * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4363      * we can ignore the first part of buffer 'a'.
4364      */
4365     while (len_b < len_a) {
4366         if (!intel_pt_step_psb(&buf_a, &len_a))
4367             return buf_b; /* No overlap */
4368     }
4369
4370     /* Now len_b >= len_a */
4371     while (1) {
4372         /* Potential overlap so check the bytes */
4373         found = memmem(buf_a, len_a, buf_b, len_a);
4374         if (found) {
4375             *consecutive = true;
4376             return adj_for_padding(buf_b + len_a, buf_a, len_a);
4377         }
4378
4379         /* Try again at next PSB in buffer 'a' */
4380         if (!intel_pt_step_psb(&buf_a, &len_a))
4381             return buf_b; /* No overlap */
4382     }
4383 }
4384
4385 /**
4386  * struct fast_forward_data - data used by intel_pt_ff_cb().
4387  * @timestamp: timestamp to fast forward towards
4388  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4389  *                 the fast forward timestamp.
4390  */
4391 struct fast_forward_data {
4392     uint64_t timestamp;
4393     uint64_t buf_timestamp;
4394 };
4395
4396 /**
4397  * intel_pt_ff_cb - fast forward lookahead callback.
4398  * @buffer: Intel PT trace buffer
4399  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4400  *
4401  * Determine if @buffer trace is past the fast forward timestamp.
4402  *
4403  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4404  *         timestamp, and 0 otherwise.
4405  */
4406 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4407 {
4408     struct fast_forward_data *d = data;
4409     unsigned char *buf;
4410     uint64_t tsc;
4411     size_t rem;
4412     size_t len;
4413
4414     buf = (unsigned char *)buffer->buf;
4415     len = buffer->len;
4416
4417     if (!intel_pt_next_psb(&buf, &len) ||
4418         !intel_pt_next_tsc(buf, len, &tsc, &rem))
4419         return 0;
4420
4421     tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4422
4423     intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4424              tsc, buffer->ref_timestamp);
4425
4426     /*
4427      * If the buffer contains a timestamp earlier that the fast forward
4428      * timestamp, then record it, else stop.
4429      */
4430     if (tsc < d->timestamp)
4431         d->buf_timestamp = buffer->ref_timestamp;
4432     else
4433         return 1;
4434
4435     return 0;
4436 }
4437
4438 /**
4439  * intel_pt_fast_forward - reposition decoder forwards.
4440  * @decoder: Intel PT decoder
4441  * @timestamp: timestamp to fast forward towards
4442  *
4443  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4444  *
4445  * Return: 0 on success or negative error code on failure.
4446  */
4447 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4448 {
4449     struct fast_forward_data d = { .timestamp = timestamp };
4450     unsigned char *buf;
4451     size_t len;
4452     int err;
4453
4454     intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4455
4456     /* Find buffer timestamp of buffer to fast forward to */
4457     err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4458     if (err < 0)
4459         return err;
4460
4461     /* Walk to buffer with same buffer timestamp */
4462     if (d.buf_timestamp) {
4463         do {
4464             decoder->pos += decoder->len;
4465             decoder->len = 0;
4466             err = intel_pt_get_next_data(decoder, true);
4467             /* -ENOLINK means non-consecutive trace */
4468             if (err && err != -ENOLINK)
4469                 return err;
4470         } while (decoder->buf_timestamp != d.buf_timestamp);
4471     }
4472
4473     if (!decoder->buf)
4474         return 0;
4475
4476     buf = (unsigned char *)decoder->buf;
4477     len = decoder->len;
4478
4479     if (!intel_pt_next_psb(&buf, &len))
4480         return 0;
4481
4482     /*
4483      * Walk PSBs while the PSB timestamp is less than the fast forward
4484      * timestamp.
4485      */
4486     do {
4487         uint64_t tsc;
4488         size_t rem;
4489
4490         if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4491             break;
4492         tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4493         /*
4494          * A TSC packet can slip past MTC packets but, after fast
4495          * forward, decoding starts at the TSC timestamp. That means
4496          * the timestamps may not be exactly the same as the timestamps
4497          * that would have been decoded without fast forward.
4498          */
4499         if (tsc < timestamp) {
4500             intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4501             decoder->pos += decoder->len - len;
4502             decoder->buf = buf;
4503             decoder->len = len;
4504             intel_pt_reposition(decoder);
4505         } else {
4506             break;
4507         }
4508     } while (intel_pt_step_psb(&buf, &len));
4509
4510     return 0;
4511 }