lib/dim/net_dim.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
0004  */
0005
0006 #include <linux/dim.h>
0007
0008 /*
0009  * Net DIM profiles:
0010  *        There are different set of profiles for each CQ period mode.
0011  *        There are different set of profiles for RX/TX CQs.
0012  *        Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES
0013  */
0014 #define NET_DIM_PARAMS_NUM_PROFILES 5
0015 #define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256
0016 #define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128
0017 #define NET_DIM_DEF_PROFILE_CQE 1
0018 #define NET_DIM_DEF_PROFILE_EQE 1
0019
0020 #define NET_DIM_RX_EQE_PROFILES { \
0021     {.usec = 1,   .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
0022     {.usec = 8,   .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
0023     {.usec = 64,  .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
0024     {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
0025     {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}  \
0026 }
0027
0028 #define NET_DIM_RX_CQE_PROFILES { \
0029     {.usec = 2,  .pkts = 256,},             \
0030     {.usec = 8,  .pkts = 128,},             \
0031     {.usec = 16, .pkts = 64,},              \
0032     {.usec = 32, .pkts = 64,},              \
0033     {.usec = 64, .pkts = 64,}               \
0034 }
0035
0036 #define NET_DIM_TX_EQE_PROFILES { \
0037     {.usec = 1,   .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
0038     {.usec = 8,   .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
0039     {.usec = 32,  .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
0040     {.usec = 64,  .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
0041     {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}   \
0042 }
0043
0044 #define NET_DIM_TX_CQE_PROFILES { \
0045     {.usec = 5,  .pkts = 128,},  \
0046     {.usec = 8,  .pkts = 64,},  \
0047     {.usec = 16, .pkts = 32,},  \
0048     {.usec = 32, .pkts = 32,},  \
0049     {.usec = 64, .pkts = 32,}   \
0050 }
0051
0052 static const struct dim_cq_moder
0053 rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
0054     NET_DIM_RX_EQE_PROFILES,
0055     NET_DIM_RX_CQE_PROFILES,
0056 };
0057
0058 static const struct dim_cq_moder
0059 tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
0060     NET_DIM_TX_EQE_PROFILES,
0061     NET_DIM_TX_CQE_PROFILES,
0062 };
0063
0064 struct dim_cq_moder
0065 net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
0066 {
0067     struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
0068
0069     cq_moder.cq_period_mode = cq_period_mode;
0070     return cq_moder;
0071 }
0072 EXPORT_SYMBOL(net_dim_get_rx_moderation);
0073
0074 struct dim_cq_moder
0075 net_dim_get_def_rx_moderation(u8 cq_period_mode)
0076 {
0077     u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
0078             NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
0079
0080     return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
0081 }
0082 EXPORT_SYMBOL(net_dim_get_def_rx_moderation);
0083
0084 struct dim_cq_moder
0085 net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
0086 {
0087     struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
0088
0089     cq_moder.cq_period_mode = cq_period_mode;
0090     return cq_moder;
0091 }
0092 EXPORT_SYMBOL(net_dim_get_tx_moderation);
0093
0094 struct dim_cq_moder
0095 net_dim_get_def_tx_moderation(u8 cq_period_mode)
0096 {
0097     u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
0098             NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
0099
0100     return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
0101 }
0102 EXPORT_SYMBOL(net_dim_get_def_tx_moderation);
0103
0104 static int net_dim_step(struct dim *dim)
0105 {
0106     if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
0107         return DIM_TOO_TIRED;
0108
0109     switch (dim->tune_state) {
0110     case DIM_PARKING_ON_TOP:
0111     case DIM_PARKING_TIRED:
0112         break;
0113     case DIM_GOING_RIGHT:
0114         if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
0115             return DIM_ON_EDGE;
0116         dim->profile_ix++;
0117         dim->steps_right++;
0118         break;
0119     case DIM_GOING_LEFT:
0120         if (dim->profile_ix == 0)
0121             return DIM_ON_EDGE;
0122         dim->profile_ix--;
0123         dim->steps_left++;
0124         break;
0125     }
0126
0127     dim->tired++;
0128     return DIM_STEPPED;
0129 }
0130
0131 static void net_dim_exit_parking(struct dim *dim)
0132 {
0133     dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
0134     net_dim_step(dim);
0135 }
0136
0137 static int net_dim_stats_compare(struct dim_stats *curr,
0138                  struct dim_stats *prev)
0139 {
0140     if (!prev->bpms)
0141         return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
0142
0143     if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
0144         return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER :
0145                            DIM_STATS_WORSE;
0146
0147     if (!prev->ppms)
0148         return curr->ppms ? DIM_STATS_BETTER :
0149                     DIM_STATS_SAME;
0150
0151     if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
0152         return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER :
0153                            DIM_STATS_WORSE;
0154
0155     if (!prev->epms)
0156         return DIM_STATS_SAME;
0157
0158     if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
0159         return (curr->epms < prev->epms) ? DIM_STATS_BETTER :
0160                            DIM_STATS_WORSE;
0161
0162     return DIM_STATS_SAME;
0163 }
0164
0165 static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)
0166 {
0167     int prev_state = dim->tune_state;
0168     int prev_ix = dim->profile_ix;
0169     int stats_res;
0170     int step_res;
0171
0172     switch (dim->tune_state) {
0173     case DIM_PARKING_ON_TOP:
0174         stats_res = net_dim_stats_compare(curr_stats,
0175                           &dim->prev_stats);
0176         if (stats_res != DIM_STATS_SAME)
0177             net_dim_exit_parking(dim);
0178         break;
0179
0180     case DIM_PARKING_TIRED:
0181         dim->tired--;
0182         if (!dim->tired)
0183             net_dim_exit_parking(dim);
0184         break;
0185
0186     case DIM_GOING_RIGHT:
0187     case DIM_GOING_LEFT:
0188         stats_res = net_dim_stats_compare(curr_stats,
0189                           &dim->prev_stats);
0190         if (stats_res != DIM_STATS_BETTER)
0191             dim_turn(dim);
0192
0193         if (dim_on_top(dim)) {
0194             dim_park_on_top(dim);
0195             break;
0196         }
0197
0198         step_res = net_dim_step(dim);
0199         switch (step_res) {
0200         case DIM_ON_EDGE:
0201             dim_park_on_top(dim);
0202             break;
0203         case DIM_TOO_TIRED:
0204             dim_park_tired(dim);
0205             break;
0206         }
0207
0208         break;
0209     }
0210
0211     if (prev_state != DIM_PARKING_ON_TOP ||
0212         dim->tune_state != DIM_PARKING_ON_TOP)
0213         dim->prev_stats = *curr_stats;
0214
0215     return dim->profile_ix != prev_ix;
0216 }
0217
0218 void net_dim(struct dim *dim, struct dim_sample end_sample)
0219 {
0220     struct dim_stats curr_stats;
0221     u16 nevents;
0222
0223     switch (dim->state) {
0224     case DIM_MEASURE_IN_PROGRESS:
0225         nevents = BIT_GAP(BITS_PER_TYPE(u16),
0226                   end_sample.event_ctr,
0227                   dim->start_sample.event_ctr);
0228         if (nevents < DIM_NEVENTS)
0229             break;
0230         dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats);
0231         if (net_dim_decision(&curr_stats, dim)) {
0232             dim->state = DIM_APPLY_NEW_PROFILE;
0233             schedule_work(&dim->work);
0234             break;
0235         }
0236         fallthrough;
0237     case DIM_START_MEASURE:
0238         dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr,
0239                   end_sample.byte_ctr, &dim->start_sample);
0240         dim->state = DIM_MEASURE_IN_PROGRESS;
0241         break;
0242     case DIM_APPLY_NEW_PROFILE:
0243         break;
0244     }
0245 }
0246 EXPORT_SYMBOL(net_dim);