drivers/xen/time.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Xen stolen ticks accounting.
0004  */
0005 #include <linux/kernel.h>
0006 #include <linux/kernel_stat.h>
0007 #include <linux/math64.h>
0008 #include <linux/gfp.h>
0009 #include <linux/slab.h>
0010 #include <linux/static_call.h>
0011
0012 #include <asm/paravirt.h>
0013 #include <asm/xen/hypervisor.h>
0014 #include <asm/xen/hypercall.h>
0015
0016 #include <xen/events.h>
0017 #include <xen/features.h>
0018 #include <xen/interface/xen.h>
0019 #include <xen/interface/vcpu.h>
0020 #include <xen/xen-ops.h>
0021
0022 /* runstate info updated by Xen */
0023 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
0024
0025 static DEFINE_PER_CPU(u64[4], old_runstate_time);
0026
0027 /* return an consistent snapshot of 64-bit time/counter value */
0028 static u64 get64(const u64 *p)
0029 {
0030     u64 ret;
0031
0032     if (BITS_PER_LONG < 64) {
0033         u32 *p32 = (u32 *)p;
0034         u32 h, l, h2;
0035
0036         /*
0037          * Read high then low, and then make sure high is
0038          * still the same; this will only loop if low wraps
0039          * and carries into high.
0040          * XXX some clean way to make this endian-proof?
0041          */
0042         do {
0043             h = READ_ONCE(p32[1]);
0044             l = READ_ONCE(p32[0]);
0045             h2 = READ_ONCE(p32[1]);
0046         } while(h2 != h);
0047
0048         ret = (((u64)h) << 32) | l;
0049     } else
0050         ret = READ_ONCE(*p);
0051
0052     return ret;
0053 }
0054
0055 static void xen_get_runstate_snapshot_cpu_delta(
0056                   struct vcpu_runstate_info *res, unsigned int cpu)
0057 {
0058     u64 state_time;
0059     struct vcpu_runstate_info *state;
0060
0061     BUG_ON(preemptible());
0062
0063     state = per_cpu_ptr(&xen_runstate, cpu);
0064
0065     do {
0066         state_time = get64(&state->state_entry_time);
0067         rmb();  /* Hypervisor might update data. */
0068         *res = __READ_ONCE(*state);
0069         rmb();  /* Hypervisor might update data. */
0070     } while (get64(&state->state_entry_time) != state_time ||
0071          (state_time & XEN_RUNSTATE_UPDATE));
0072 }
0073
0074 static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
0075                       unsigned int cpu)
0076 {
0077     int i;
0078
0079     xen_get_runstate_snapshot_cpu_delta(res, cpu);
0080
0081     for (i = 0; i < 4; i++)
0082         res->time[i] += per_cpu(old_runstate_time, cpu)[i];
0083 }
0084
0085 void xen_manage_runstate_time(int action)
0086 {
0087     static struct vcpu_runstate_info *runstate_delta;
0088     struct vcpu_runstate_info state;
0089     int cpu, i;
0090
0091     switch (action) {
0092     case -1: /* backup runstate time before suspend */
0093         if (unlikely(runstate_delta))
0094             pr_warn_once("%s: memory leak as runstate_delta is not NULL\n",
0095                     __func__);
0096
0097         runstate_delta = kmalloc_array(num_possible_cpus(),
0098                     sizeof(*runstate_delta),
0099                     GFP_ATOMIC);
0100         if (unlikely(!runstate_delta)) {
0101             pr_warn("%s: failed to allocate runstate_delta\n",
0102                     __func__);
0103             return;
0104         }
0105
0106         for_each_possible_cpu(cpu) {
0107             xen_get_runstate_snapshot_cpu_delta(&state, cpu);
0108             memcpy(runstate_delta[cpu].time, state.time,
0109                     sizeof(runstate_delta[cpu].time));
0110         }
0111
0112         break;
0113
0114     case 0: /* backup runstate time after resume */
0115         if (unlikely(!runstate_delta)) {
0116             pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n",
0117                     __func__);
0118             return;
0119         }
0120
0121         for_each_possible_cpu(cpu) {
0122             for (i = 0; i < 4; i++)
0123                 per_cpu(old_runstate_time, cpu)[i] +=
0124                     runstate_delta[cpu].time[i];
0125         }
0126
0127         break;
0128
0129     default: /* do not accumulate runstate time for checkpointing */
0130         break;
0131     }
0132
0133     if (action != -1 && runstate_delta) {
0134         kfree(runstate_delta);
0135         runstate_delta = NULL;
0136     }
0137 }
0138
0139 /*
0140  * Runstate accounting
0141  */
0142 void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
0143 {
0144     xen_get_runstate_snapshot_cpu(res, smp_processor_id());
0145 }
0146
0147 /* return true when a vcpu could run but has no real cpu to run on */
0148 bool xen_vcpu_stolen(int vcpu)
0149 {
0150     return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
0151 }
0152
0153 u64 xen_steal_clock(int cpu)
0154 {
0155     struct vcpu_runstate_info state;
0156
0157     xen_get_runstate_snapshot_cpu(&state, cpu);
0158     return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
0159 }
0160
0161 void xen_setup_runstate_info(int cpu)
0162 {
0163     struct vcpu_register_runstate_memory_area area;
0164
0165     area.addr.v = &per_cpu(xen_runstate, cpu);
0166
0167     if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
0168                    xen_vcpu_nr(cpu), &area))
0169         BUG();
0170 }
0171
0172 void __init xen_time_setup_guest(void)
0173 {
0174     bool xen_runstate_remote;
0175
0176     xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
0177                     VMASST_TYPE_runstate_update_flag);
0178
0179     static_call_update(pv_steal_clock, xen_steal_clock);
0180
0181     static_key_slow_inc(&paravirt_steal_enabled);
0182     if (xen_runstate_remote)
0183         static_key_slow_inc(&paravirt_steal_rq_enabled);
0184 }