s390/kernel/vtime.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  *    Virtual cpu timer based timer functions.
0004  *
0005  *    Copyright IBM Corp. 2004, 2012
0006  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
0007  */
0008
0009 #include <linux/kernel_stat.h>
0010 #include <linux/sched/cputime.h>
0011 #include <linux/export.h>
0012 #include <linux/kernel.h>
0013 #include <linux/timex.h>
0014 #include <linux/types.h>
0015 #include <linux/time.h>
0016 #include <asm/alternative.h>
0017 #include <asm/vtimer.h>
0018 #include <asm/vtime.h>
0019 #include <asm/cpu_mf.h>
0020 #include <asm/smp.h>
0021
0022 #include "entry.h"
0023
0024 static void virt_timer_expire(void);
0025
0026 static LIST_HEAD(virt_timer_list);
0027 static DEFINE_SPINLOCK(virt_timer_lock);
0028 static atomic64_t virt_timer_current;
0029 static atomic64_t virt_timer_elapsed;
0030
0031 DEFINE_PER_CPU(u64, mt_cycles[8]);
0032 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
0033 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
0034 static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
0035
0036 static inline u64 get_vtimer(void)
0037 {
0038     u64 timer;
0039
0040     asm volatile("stpt %0" : "=Q" (timer));
0041     return timer;
0042 }
0043
0044 static inline void set_vtimer(u64 expires)
0045 {
0046     u64 timer;
0047
0048     asm volatile(
0049         "   stpt    %0\n"   /* Store current cpu timer value */
0050         "   spt %1" /* Set new value imm. afterwards */
0051         : "=Q" (timer) : "Q" (expires));
0052     S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
0053     S390_lowcore.last_update_timer = expires;
0054 }
0055
0056 static inline int virt_timer_forward(u64 elapsed)
0057 {
0058     BUG_ON(!irqs_disabled());
0059
0060     if (list_empty(&virt_timer_list))
0061         return 0;
0062     elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
0063     return elapsed >= atomic64_read(&virt_timer_current);
0064 }
0065
0066 static void update_mt_scaling(void)
0067 {
0068     u64 cycles_new[8], *cycles_old;
0069     u64 delta, fac, mult, div;
0070     int i;
0071
0072     stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
0073     cycles_old = this_cpu_ptr(mt_cycles);
0074     fac = 1;
0075     mult = div = 0;
0076     for (i = 0; i <= smp_cpu_mtid; i++) {
0077         delta = cycles_new[i] - cycles_old[i];
0078         div += delta;
0079         mult *= i + 1;
0080         mult += delta * fac;
0081         fac *= i + 1;
0082     }
0083     div *= fac;
0084     if (div > 0) {
0085         /* Update scaling factor */
0086         __this_cpu_write(mt_scaling_mult, mult);
0087         __this_cpu_write(mt_scaling_div, div);
0088         memcpy(cycles_old, cycles_new,
0089                sizeof(u64) * (smp_cpu_mtid + 1));
0090     }
0091     __this_cpu_write(mt_scaling_jiffies, jiffies_64);
0092 }
0093
0094 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
0095 {
0096     u64 delta;
0097
0098     delta = new - *tsk_vtime;
0099     *tsk_vtime = new;
0100     return delta;
0101 }
0102
0103
0104 static inline u64 scale_vtime(u64 vtime)
0105 {
0106     u64 mult = __this_cpu_read(mt_scaling_mult);
0107     u64 div = __this_cpu_read(mt_scaling_div);
0108
0109     if (smp_cpu_mtid)
0110         return vtime * mult / div;
0111     return vtime;
0112 }
0113
0114 static void account_system_index_scaled(struct task_struct *p, u64 cputime,
0115                     enum cpu_usage_stat index)
0116 {
0117     p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
0118     account_system_index_time(p, cputime_to_nsecs(cputime), index);
0119 }
0120
0121 /*
0122  * Update process times based on virtual cpu times stored by entry.S
0123  * to the lowcore fields user_timer, system_timer & steal_clock.
0124  */
0125 static int do_account_vtime(struct task_struct *tsk)
0126 {
0127     u64 timer, clock, user, guest, system, hardirq, softirq;
0128
0129     timer = S390_lowcore.last_update_timer;
0130     clock = S390_lowcore.last_update_clock;
0131     asm volatile(
0132         "   stpt    %0\n"   /* Store current cpu timer value */
0133         "   stckf   %1" /* Store current tod clock value */
0134         : "=Q" (S390_lowcore.last_update_timer),
0135           "=Q" (S390_lowcore.last_update_clock)
0136         : : "cc");
0137     clock = S390_lowcore.last_update_clock - clock;
0138     timer -= S390_lowcore.last_update_timer;
0139
0140     if (hardirq_count())
0141         S390_lowcore.hardirq_timer += timer;
0142     else
0143         S390_lowcore.system_timer += timer;
0144
0145     /* Update MT utilization calculation */
0146     if (smp_cpu_mtid &&
0147         time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
0148         update_mt_scaling();
0149
0150     /* Calculate cputime delta */
0151     user = update_tsk_timer(&tsk->thread.user_timer,
0152                 READ_ONCE(S390_lowcore.user_timer));
0153     guest = update_tsk_timer(&tsk->thread.guest_timer,
0154                  READ_ONCE(S390_lowcore.guest_timer));
0155     system = update_tsk_timer(&tsk->thread.system_timer,
0156                   READ_ONCE(S390_lowcore.system_timer));
0157     hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
0158                    READ_ONCE(S390_lowcore.hardirq_timer));
0159     softirq = update_tsk_timer(&tsk->thread.softirq_timer,
0160                    READ_ONCE(S390_lowcore.softirq_timer));
0161     S390_lowcore.steal_timer +=
0162         clock - user - guest - system - hardirq - softirq;
0163
0164     /* Push account value */
0165     if (user) {
0166         account_user_time(tsk, cputime_to_nsecs(user));
0167         tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
0168     }
0169
0170     if (guest) {
0171         account_guest_time(tsk, cputime_to_nsecs(guest));
0172         tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
0173     }
0174
0175     if (system)
0176         account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
0177     if (hardirq)
0178         account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
0179     if (softirq)
0180         account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
0181
0182     return virt_timer_forward(user + guest + system + hardirq + softirq);
0183 }
0184
0185 void vtime_task_switch(struct task_struct *prev)
0186 {
0187     do_account_vtime(prev);
0188     prev->thread.user_timer = S390_lowcore.user_timer;
0189     prev->thread.guest_timer = S390_lowcore.guest_timer;
0190     prev->thread.system_timer = S390_lowcore.system_timer;
0191     prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
0192     prev->thread.softirq_timer = S390_lowcore.softirq_timer;
0193     S390_lowcore.user_timer = current->thread.user_timer;
0194     S390_lowcore.guest_timer = current->thread.guest_timer;
0195     S390_lowcore.system_timer = current->thread.system_timer;
0196     S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
0197     S390_lowcore.softirq_timer = current->thread.softirq_timer;
0198 }
0199
0200 /*
0201  * In s390, accounting pending user time also implies
0202  * accounting system time in order to correctly compute
0203  * the stolen time accounting.
0204  */
0205 void vtime_flush(struct task_struct *tsk)
0206 {
0207     u64 steal, avg_steal;
0208
0209     if (do_account_vtime(tsk))
0210         virt_timer_expire();
0211
0212     steal = S390_lowcore.steal_timer;
0213     avg_steal = S390_lowcore.avg_steal_timer / 2;
0214     if ((s64) steal > 0) {
0215         S390_lowcore.steal_timer = 0;
0216         account_steal_time(cputime_to_nsecs(steal));
0217         avg_steal += steal;
0218     }
0219     S390_lowcore.avg_steal_timer = avg_steal;
0220 }
0221
0222 static u64 vtime_delta(void)
0223 {
0224     u64 timer = S390_lowcore.last_update_timer;
0225
0226     S390_lowcore.last_update_timer = get_vtimer();
0227
0228     return timer - S390_lowcore.last_update_timer;
0229 }
0230
0231 /*
0232  * Update process times based on virtual cpu times stored by entry.S
0233  * to the lowcore fields user_timer, system_timer & steal_clock.
0234  */
0235 void vtime_account_kernel(struct task_struct *tsk)
0236 {
0237     u64 delta = vtime_delta();
0238
0239     if (tsk->flags & PF_VCPU)
0240         S390_lowcore.guest_timer += delta;
0241     else
0242         S390_lowcore.system_timer += delta;
0243
0244     virt_timer_forward(delta);
0245 }
0246 EXPORT_SYMBOL_GPL(vtime_account_kernel);
0247
0248 void vtime_account_softirq(struct task_struct *tsk)
0249 {
0250     u64 delta = vtime_delta();
0251
0252     S390_lowcore.softirq_timer += delta;
0253
0254     virt_timer_forward(delta);
0255 }
0256
0257 void vtime_account_hardirq(struct task_struct *tsk)
0258 {
0259     u64 delta = vtime_delta();
0260
0261     S390_lowcore.hardirq_timer += delta;
0262
0263     virt_timer_forward(delta);
0264 }
0265
0266 /*
0267  * Sorted add to a list. List is linear searched until first bigger
0268  * element is found.
0269  */
0270 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
0271 {
0272     struct vtimer_list *tmp;
0273
0274     list_for_each_entry(tmp, head, entry) {
0275         if (tmp->expires > timer->expires) {
0276             list_add_tail(&timer->entry, &tmp->entry);
0277             return;
0278         }
0279     }
0280     list_add_tail(&timer->entry, head);
0281 }
0282
0283 /*
0284  * Handler for expired virtual CPU timer.
0285  */
0286 static void virt_timer_expire(void)
0287 {
0288     struct vtimer_list *timer, *tmp;
0289     unsigned long elapsed;
0290     LIST_HEAD(cb_list);
0291
0292     /* walk timer list, fire all expired timers */
0293     spin_lock(&virt_timer_lock);
0294     elapsed = atomic64_read(&virt_timer_elapsed);
0295     list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
0296         if (timer->expires < elapsed)
0297             /* move expired timer to the callback queue */
0298             list_move_tail(&timer->entry, &cb_list);
0299         else
0300             timer->expires -= elapsed;
0301     }
0302     if (!list_empty(&virt_timer_list)) {
0303         timer = list_first_entry(&virt_timer_list,
0304                      struct vtimer_list, entry);
0305         atomic64_set(&virt_timer_current, timer->expires);
0306     }
0307     atomic64_sub(elapsed, &virt_timer_elapsed);
0308     spin_unlock(&virt_timer_lock);
0309
0310     /* Do callbacks and recharge periodic timers */
0311     list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
0312         list_del_init(&timer->entry);
0313         timer->function(timer->data);
0314         if (timer->interval) {
0315             /* Recharge interval timer */
0316             timer->expires = timer->interval +
0317                 atomic64_read(&virt_timer_elapsed);
0318             spin_lock(&virt_timer_lock);
0319             list_add_sorted(timer, &virt_timer_list);
0320             spin_unlock(&virt_timer_lock);
0321         }
0322     }
0323 }
0324
0325 void init_virt_timer(struct vtimer_list *timer)
0326 {
0327     timer->function = NULL;
0328     INIT_LIST_HEAD(&timer->entry);
0329 }
0330 EXPORT_SYMBOL(init_virt_timer);
0331
0332 static inline int vtimer_pending(struct vtimer_list *timer)
0333 {
0334     return !list_empty(&timer->entry);
0335 }
0336
0337 static void internal_add_vtimer(struct vtimer_list *timer)
0338 {
0339     if (list_empty(&virt_timer_list)) {
0340         /* First timer, just program it. */
0341         atomic64_set(&virt_timer_current, timer->expires);
0342         atomic64_set(&virt_timer_elapsed, 0);
0343         list_add(&timer->entry, &virt_timer_list);
0344     } else {
0345         /* Update timer against current base. */
0346         timer->expires += atomic64_read(&virt_timer_elapsed);
0347         if (likely((s64) timer->expires <
0348                (s64) atomic64_read(&virt_timer_current)))
0349             /* The new timer expires before the current timer. */
0350             atomic64_set(&virt_timer_current, timer->expires);
0351         /* Insert new timer into the list. */
0352         list_add_sorted(timer, &virt_timer_list);
0353     }
0354 }
0355
0356 static void __add_vtimer(struct vtimer_list *timer, int periodic)
0357 {
0358     unsigned long flags;
0359
0360     timer->interval = periodic ? timer->expires : 0;
0361     spin_lock_irqsave(&virt_timer_lock, flags);
0362     internal_add_vtimer(timer);
0363     spin_unlock_irqrestore(&virt_timer_lock, flags);
0364 }
0365
0366 /*
0367  * add_virt_timer - add a oneshot virtual CPU timer
0368  */
0369 void add_virt_timer(struct vtimer_list *timer)
0370 {
0371     __add_vtimer(timer, 0);
0372 }
0373 EXPORT_SYMBOL(add_virt_timer);
0374
0375 /*
0376  * add_virt_timer_int - add an interval virtual CPU timer
0377  */
0378 void add_virt_timer_periodic(struct vtimer_list *timer)
0379 {
0380     __add_vtimer(timer, 1);
0381 }
0382 EXPORT_SYMBOL(add_virt_timer_periodic);
0383
0384 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
0385 {
0386     unsigned long flags;
0387     int rc;
0388
0389     BUG_ON(!timer->function);
0390
0391     if (timer->expires == expires && vtimer_pending(timer))
0392         return 1;
0393     spin_lock_irqsave(&virt_timer_lock, flags);
0394     rc = vtimer_pending(timer);
0395     if (rc)
0396         list_del_init(&timer->entry);
0397     timer->interval = periodic ? expires : 0;
0398     timer->expires = expires;
0399     internal_add_vtimer(timer);
0400     spin_unlock_irqrestore(&virt_timer_lock, flags);
0401     return rc;
0402 }
0403
0404 /*
0405  * returns whether it has modified a pending timer (1) or not (0)
0406  */
0407 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
0408 {
0409     return __mod_vtimer(timer, expires, 0);
0410 }
0411 EXPORT_SYMBOL(mod_virt_timer);
0412
0413 /*
0414  * returns whether it has modified a pending timer (1) or not (0)
0415  */
0416 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
0417 {
0418     return __mod_vtimer(timer, expires, 1);
0419 }
0420 EXPORT_SYMBOL(mod_virt_timer_periodic);
0421
0422 /*
0423  * Delete a virtual timer.
0424  *
0425  * returns whether the deleted timer was pending (1) or not (0)
0426  */
0427 int del_virt_timer(struct vtimer_list *timer)
0428 {
0429     unsigned long flags;
0430
0431     if (!vtimer_pending(timer))
0432         return 0;
0433     spin_lock_irqsave(&virt_timer_lock, flags);
0434     list_del_init(&timer->entry);
0435     spin_unlock_irqrestore(&virt_timer_lock, flags);
0436     return 1;
0437 }
0438 EXPORT_SYMBOL(del_virt_timer);
0439
0440 /*
0441  * Start the virtual CPU timer on the current CPU.
0442  */
0443 void vtime_init(void)
0444 {
0445     /* set initial cpu timer */
0446     set_vtimer(VTIMER_MAX_SLICE);
0447     /* Setup initial MT scaling values */
0448     if (smp_cpu_mtid) {
0449         __this_cpu_write(mt_scaling_jiffies, jiffies);
0450         __this_cpu_write(mt_scaling_mult, 1);
0451         __this_cpu_write(mt_scaling_div, 1);
0452         stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
0453     }
0454 }