Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Loopback bridge driver for the Greybus loopback module.
0004  *
0005  * Copyright 2014 Google Inc.
0006  * Copyright 2014 Linaro Ltd.
0007  */
0008 
0009 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0010 
0011 #include <linux/kernel.h>
0012 #include <linux/module.h>
0013 #include <linux/mutex.h>
0014 #include <linux/slab.h>
0015 #include <linux/kthread.h>
0016 #include <linux/delay.h>
0017 #include <linux/random.h>
0018 #include <linux/sizes.h>
0019 #include <linux/cdev.h>
0020 #include <linux/fs.h>
0021 #include <linux/kfifo.h>
0022 #include <linux/debugfs.h>
0023 #include <linux/list_sort.h>
0024 #include <linux/spinlock.h>
0025 #include <linux/workqueue.h>
0026 #include <linux/atomic.h>
0027 #include <linux/pm_runtime.h>
0028 #include <linux/greybus.h>
0029 #include <asm/div64.h>
0030 
0031 #define NSEC_PER_DAY 86400000000000ULL
0032 
0033 struct gb_loopback_stats {
0034     u32 min;
0035     u32 max;
0036     u64 sum;
0037     u32 count;
0038 };
0039 
0040 struct gb_loopback_device {
0041     struct dentry *root;
0042     u32 count;
0043     size_t size_max;
0044 
0045     /* We need to take a lock in atomic context */
0046     spinlock_t lock;
0047     wait_queue_head_t wq;
0048 };
0049 
0050 static struct gb_loopback_device gb_dev;
0051 
0052 struct gb_loopback_async_operation {
0053     struct gb_loopback *gb;
0054     struct gb_operation *operation;
0055     ktime_t ts;
0056     int (*completion)(struct gb_loopback_async_operation *op_async);
0057 };
0058 
0059 struct gb_loopback {
0060     struct gb_connection *connection;
0061 
0062     struct dentry *file;
0063     struct kfifo kfifo_lat;
0064     struct mutex mutex;
0065     struct task_struct *task;
0066     struct device *dev;
0067     wait_queue_head_t wq;
0068     wait_queue_head_t wq_completion;
0069     atomic_t outstanding_operations;
0070 
0071     /* Per connection stats */
0072     ktime_t ts;
0073     struct gb_loopback_stats latency;
0074     struct gb_loopback_stats throughput;
0075     struct gb_loopback_stats requests_per_second;
0076     struct gb_loopback_stats apbridge_unipro_latency;
0077     struct gb_loopback_stats gbphy_firmware_latency;
0078 
0079     int type;
0080     int async;
0081     int id;
0082     u32 size;
0083     u32 iteration_max;
0084     u32 iteration_count;
0085     int us_wait;
0086     u32 error;
0087     u32 requests_completed;
0088     u32 requests_timedout;
0089     u32 timeout;
0090     u32 jiffy_timeout;
0091     u32 timeout_min;
0092     u32 timeout_max;
0093     u32 outstanding_operations_max;
0094     u64 elapsed_nsecs;
0095     u32 apbridge_latency_ts;
0096     u32 gbphy_latency_ts;
0097 
0098     u32 send_count;
0099 };
0100 
0101 static struct class loopback_class = {
0102     .name       = "gb_loopback",
0103     .owner      = THIS_MODULE,
0104 };
0105 static DEFINE_IDA(loopback_ida);
0106 
0107 /* Min/max values in jiffies */
0108 #define GB_LOOPBACK_TIMEOUT_MIN             1
0109 #define GB_LOOPBACK_TIMEOUT_MAX             10000
0110 
0111 #define GB_LOOPBACK_FIFO_DEFAULT            8192
0112 
0113 static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT;
0114 module_param(kfifo_depth, uint, 0444);
0115 
0116 /* Maximum size of any one send data buffer we support */
0117 #define MAX_PACKET_SIZE (PAGE_SIZE * 2)
0118 
0119 #define GB_LOOPBACK_US_WAIT_MAX             1000000
0120 
0121 /* interface sysfs attributes */
0122 #define gb_loopback_ro_attr(field)              \
0123 static ssize_t field##_show(struct device *dev,         \
0124                 struct device_attribute *attr,      \
0125                 char *buf)                  \
0126 {                                   \
0127     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0128     return sprintf(buf, "%u\n", gb->field);         \
0129 }                                   \
0130 static DEVICE_ATTR_RO(field)
0131 
0132 #define gb_loopback_ro_stats_attr(name, field, type)        \
0133 static ssize_t name##_##field##_show(struct device *dev,    \
0134                 struct device_attribute *attr,      \
0135                 char *buf)                  \
0136 {                                   \
0137     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0138     /* Report 0 for min and max if no transfer succeeded */     \
0139     if (!gb->requests_completed)                    \
0140         return sprintf(buf, "0\n");             \
0141     return sprintf(buf, "%" #type "\n", gb->name.field);        \
0142 }                                   \
0143 static DEVICE_ATTR_RO(name##_##field)
0144 
0145 #define gb_loopback_ro_avg_attr(name)           \
0146 static ssize_t name##_avg_show(struct device *dev,      \
0147                 struct device_attribute *attr,      \
0148                 char *buf)                  \
0149 {                                   \
0150     struct gb_loopback_stats *stats;                \
0151     struct gb_loopback *gb;                     \
0152     u64 avg, rem;                           \
0153     u32 count;                          \
0154     gb = dev_get_drvdata(dev);          \
0155     stats = &gb->name;                  \
0156     count = stats->count ? stats->count : 1;            \
0157     avg = stats->sum + count / 2000000; /* round closest */     \
0158     rem = do_div(avg, count);                   \
0159     rem *= 1000000;                         \
0160     do_div(rem, count);                     \
0161     return sprintf(buf, "%llu.%06u\n", avg, (u32)rem);      \
0162 }                                   \
0163 static DEVICE_ATTR_RO(name##_avg)
0164 
0165 #define gb_loopback_stats_attrs(field)              \
0166     gb_loopback_ro_stats_attr(field, min, u);       \
0167     gb_loopback_ro_stats_attr(field, max, u);       \
0168     gb_loopback_ro_avg_attr(field)
0169 
0170 #define gb_loopback_attr(field, type)                   \
0171 static ssize_t field##_show(struct device *dev,             \
0172                 struct device_attribute *attr,      \
0173                 char *buf)                  \
0174 {                                   \
0175     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0176     return sprintf(buf, "%" #type "\n", gb->field);         \
0177 }                                   \
0178 static ssize_t field##_store(struct device *dev,            \
0179                 struct device_attribute *attr,      \
0180                 const char *buf,                \
0181                 size_t len)                 \
0182 {                                   \
0183     int ret;                            \
0184     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0185     mutex_lock(&gb->mutex);                     \
0186     ret = sscanf(buf, "%"#type, &gb->field);            \
0187     if (ret != 1)                           \
0188         len = -EINVAL;                      \
0189     else                                \
0190         gb_loopback_check_attr(gb, bundle);         \
0191     mutex_unlock(&gb->mutex);                   \
0192     return len;                         \
0193 }                                   \
0194 static DEVICE_ATTR_RW(field)
0195 
0196 #define gb_dev_loopback_ro_attr(field, conn)                \
0197 static ssize_t field##_show(struct device *dev,     \
0198                 struct device_attribute *attr,      \
0199                 char *buf)                  \
0200 {                                   \
0201     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0202     return sprintf(buf, "%u\n", gb->field);             \
0203 }                                   \
0204 static DEVICE_ATTR_RO(field)
0205 
0206 #define gb_dev_loopback_rw_attr(field, type)                \
0207 static ssize_t field##_show(struct device *dev,             \
0208                 struct device_attribute *attr,      \
0209                 char *buf)                  \
0210 {                                   \
0211     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0212     return sprintf(buf, "%" #type "\n", gb->field);         \
0213 }                                   \
0214 static ssize_t field##_store(struct device *dev,            \
0215                 struct device_attribute *attr,      \
0216                 const char *buf,                \
0217                 size_t len)                 \
0218 {                                   \
0219     int ret;                            \
0220     struct gb_loopback *gb = dev_get_drvdata(dev);          \
0221     mutex_lock(&gb->mutex);                     \
0222     ret = sscanf(buf, "%"#type, &gb->field);            \
0223     if (ret != 1)                           \
0224         len = -EINVAL;                      \
0225     else                                \
0226         gb_loopback_check_attr(gb);     \
0227     mutex_unlock(&gb->mutex);                   \
0228     return len;                         \
0229 }                                   \
0230 static DEVICE_ATTR_RW(field)
0231 
0232 static void gb_loopback_reset_stats(struct gb_loopback *gb);
0233 static void gb_loopback_check_attr(struct gb_loopback *gb)
0234 {
0235     if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX)
0236         gb->us_wait = GB_LOOPBACK_US_WAIT_MAX;
0237     if (gb->size > gb_dev.size_max)
0238         gb->size = gb_dev.size_max;
0239     gb->requests_timedout = 0;
0240     gb->requests_completed = 0;
0241     gb->iteration_count = 0;
0242     gb->send_count = 0;
0243     gb->error = 0;
0244 
0245     if (kfifo_depth < gb->iteration_max) {
0246         dev_warn(gb->dev,
0247              "cannot log bytes %u kfifo_depth %u\n",
0248              gb->iteration_max, kfifo_depth);
0249     }
0250     kfifo_reset_out(&gb->kfifo_lat);
0251 
0252     switch (gb->type) {
0253     case GB_LOOPBACK_TYPE_PING:
0254     case GB_LOOPBACK_TYPE_TRANSFER:
0255     case GB_LOOPBACK_TYPE_SINK:
0256         gb->jiffy_timeout = usecs_to_jiffies(gb->timeout);
0257         if (!gb->jiffy_timeout)
0258             gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN;
0259         else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX)
0260             gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX;
0261         gb_loopback_reset_stats(gb);
0262         wake_up(&gb->wq);
0263         break;
0264     default:
0265         gb->type = 0;
0266         break;
0267     }
0268 }
0269 
0270 /* Time to send and receive one message */
0271 gb_loopback_stats_attrs(latency);
0272 /* Number of requests sent per second on this cport */
0273 gb_loopback_stats_attrs(requests_per_second);
0274 /* Quantity of data sent and received on this cport */
0275 gb_loopback_stats_attrs(throughput);
0276 /* Latency across the UniPro link from APBridge's perspective */
0277 gb_loopback_stats_attrs(apbridge_unipro_latency);
0278 /* Firmware induced overhead in the GPBridge */
0279 gb_loopback_stats_attrs(gbphy_firmware_latency);
0280 
0281 /* Number of errors encountered during loop */
0282 gb_loopback_ro_attr(error);
0283 /* Number of requests successfully completed async */
0284 gb_loopback_ro_attr(requests_completed);
0285 /* Number of requests timed out async */
0286 gb_loopback_ro_attr(requests_timedout);
0287 /* Timeout minimum in useconds */
0288 gb_loopback_ro_attr(timeout_min);
0289 /* Timeout minimum in useconds */
0290 gb_loopback_ro_attr(timeout_max);
0291 
0292 /*
0293  * Type of loopback message to send based on protocol type definitions
0294  * 0 => Don't send message
0295  * 2 => Send ping message continuously (message without payload)
0296  * 3 => Send transfer message continuously (message with payload,
0297  *                     payload returned in response)
0298  * 4 => Send a sink message (message with payload, no payload in response)
0299  */
0300 gb_dev_loopback_rw_attr(type, d);
0301 /* Size of transfer message payload: 0-4096 bytes */
0302 gb_dev_loopback_rw_attr(size, u);
0303 /* Time to wait between two messages: 0-1000 ms */
0304 gb_dev_loopback_rw_attr(us_wait, d);
0305 /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */
0306 gb_dev_loopback_rw_attr(iteration_max, u);
0307 /* The current index of the for (i = 0; i < iteration_max; i++) loop */
0308 gb_dev_loopback_ro_attr(iteration_count, false);
0309 /* A flag to indicate synchronous or asynchronous operations */
0310 gb_dev_loopback_rw_attr(async, u);
0311 /* Timeout of an individual asynchronous request */
0312 gb_dev_loopback_rw_attr(timeout, u);
0313 /* Maximum number of in-flight operations before back-off */
0314 gb_dev_loopback_rw_attr(outstanding_operations_max, u);
0315 
0316 static struct attribute *loopback_attrs[] = {
0317     &dev_attr_latency_min.attr,
0318     &dev_attr_latency_max.attr,
0319     &dev_attr_latency_avg.attr,
0320     &dev_attr_requests_per_second_min.attr,
0321     &dev_attr_requests_per_second_max.attr,
0322     &dev_attr_requests_per_second_avg.attr,
0323     &dev_attr_throughput_min.attr,
0324     &dev_attr_throughput_max.attr,
0325     &dev_attr_throughput_avg.attr,
0326     &dev_attr_apbridge_unipro_latency_min.attr,
0327     &dev_attr_apbridge_unipro_latency_max.attr,
0328     &dev_attr_apbridge_unipro_latency_avg.attr,
0329     &dev_attr_gbphy_firmware_latency_min.attr,
0330     &dev_attr_gbphy_firmware_latency_max.attr,
0331     &dev_attr_gbphy_firmware_latency_avg.attr,
0332     &dev_attr_type.attr,
0333     &dev_attr_size.attr,
0334     &dev_attr_us_wait.attr,
0335     &dev_attr_iteration_count.attr,
0336     &dev_attr_iteration_max.attr,
0337     &dev_attr_async.attr,
0338     &dev_attr_error.attr,
0339     &dev_attr_requests_completed.attr,
0340     &dev_attr_requests_timedout.attr,
0341     &dev_attr_timeout.attr,
0342     &dev_attr_outstanding_operations_max.attr,
0343     &dev_attr_timeout_min.attr,
0344     &dev_attr_timeout_max.attr,
0345     NULL,
0346 };
0347 ATTRIBUTE_GROUPS(loopback);
0348 
0349 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error);
0350 
0351 static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs)
0352 {
0353     do_div(elapsed_nsecs, NSEC_PER_USEC);
0354     return elapsed_nsecs;
0355 }
0356 
0357 static u64 __gb_loopback_calc_latency(u64 t1, u64 t2)
0358 {
0359     if (t2 > t1)
0360         return t2 - t1;
0361     else
0362         return NSEC_PER_DAY - t2 + t1;
0363 }
0364 
0365 static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te)
0366 {
0367     return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te));
0368 }
0369 
0370 static int gb_loopback_operation_sync(struct gb_loopback *gb, int type,
0371                       void *request, int request_size,
0372                       void *response, int response_size)
0373 {
0374     struct gb_operation *operation;
0375     ktime_t ts, te;
0376     int ret;
0377 
0378     ts = ktime_get();
0379     operation = gb_operation_create(gb->connection, type, request_size,
0380                     response_size, GFP_KERNEL);
0381     if (!operation)
0382         return -ENOMEM;
0383 
0384     if (request_size)
0385         memcpy(operation->request->payload, request, request_size);
0386 
0387     ret = gb_operation_request_send_sync(operation);
0388     if (ret) {
0389         dev_err(&gb->connection->bundle->dev,
0390             "synchronous operation failed: %d\n", ret);
0391         goto out_put_operation;
0392     } else {
0393         if (response_size == operation->response->payload_size) {
0394             memcpy(response, operation->response->payload,
0395                    response_size);
0396         } else {
0397             dev_err(&gb->connection->bundle->dev,
0398                 "response size %zu expected %d\n",
0399                 operation->response->payload_size,
0400                 response_size);
0401             ret = -EINVAL;
0402             goto out_put_operation;
0403         }
0404     }
0405 
0406     te = ktime_get();
0407 
0408     /* Calculate the total time the message took */
0409     gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te);
0410 
0411 out_put_operation:
0412     gb_operation_put(operation);
0413 
0414     return ret;
0415 }
0416 
0417 static void gb_loopback_async_wait_all(struct gb_loopback *gb)
0418 {
0419     wait_event(gb->wq_completion,
0420            !atomic_read(&gb->outstanding_operations));
0421 }
0422 
0423 static void gb_loopback_async_operation_callback(struct gb_operation *operation)
0424 {
0425     struct gb_loopback_async_operation *op_async;
0426     struct gb_loopback *gb;
0427     ktime_t te;
0428     int result;
0429 
0430     te = ktime_get();
0431     result = gb_operation_result(operation);
0432     op_async = gb_operation_get_data(operation);
0433     gb = op_async->gb;
0434 
0435     mutex_lock(&gb->mutex);
0436 
0437     if (!result && op_async->completion)
0438         result = op_async->completion(op_async);
0439 
0440     if (!result) {
0441         gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te);
0442     } else {
0443         gb->error++;
0444         if (result == -ETIMEDOUT)
0445             gb->requests_timedout++;
0446     }
0447 
0448     gb->iteration_count++;
0449     gb_loopback_calculate_stats(gb, result);
0450 
0451     mutex_unlock(&gb->mutex);
0452 
0453     dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n",
0454         operation->id);
0455 
0456     /* Wake up waiters */
0457     atomic_dec(&op_async->gb->outstanding_operations);
0458     wake_up(&gb->wq_completion);
0459 
0460     /* Release resources */
0461     gb_operation_put(operation);
0462     kfree(op_async);
0463 }
0464 
0465 static int gb_loopback_async_operation(struct gb_loopback *gb, int type,
0466                        void *request, int request_size,
0467                        int response_size,
0468                        void *completion)
0469 {
0470     struct gb_loopback_async_operation *op_async;
0471     struct gb_operation *operation;
0472     int ret;
0473 
0474     op_async = kzalloc(sizeof(*op_async), GFP_KERNEL);
0475     if (!op_async)
0476         return -ENOMEM;
0477 
0478     operation = gb_operation_create(gb->connection, type, request_size,
0479                     response_size, GFP_KERNEL);
0480     if (!operation) {
0481         kfree(op_async);
0482         return -ENOMEM;
0483     }
0484 
0485     if (request_size)
0486         memcpy(operation->request->payload, request, request_size);
0487 
0488     gb_operation_set_data(operation, op_async);
0489 
0490     op_async->gb = gb;
0491     op_async->operation = operation;
0492     op_async->completion = completion;
0493 
0494     op_async->ts = ktime_get();
0495 
0496     atomic_inc(&gb->outstanding_operations);
0497     ret = gb_operation_request_send(operation,
0498                     gb_loopback_async_operation_callback,
0499                     jiffies_to_msecs(gb->jiffy_timeout),
0500                     GFP_KERNEL);
0501     if (ret) {
0502         atomic_dec(&gb->outstanding_operations);
0503         gb_operation_put(operation);
0504         kfree(op_async);
0505     }
0506     return ret;
0507 }
0508 
0509 static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len)
0510 {
0511     struct gb_loopback_transfer_request *request;
0512     int retval;
0513 
0514     request = kmalloc(len + sizeof(*request), GFP_KERNEL);
0515     if (!request)
0516         return -ENOMEM;
0517 
0518     request->len = cpu_to_le32(len);
0519     retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK,
0520                         request, len + sizeof(*request),
0521                         NULL, 0);
0522     kfree(request);
0523     return retval;
0524 }
0525 
0526 static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len)
0527 {
0528     struct gb_loopback_transfer_request *request;
0529     struct gb_loopback_transfer_response *response;
0530     int retval;
0531 
0532     gb->apbridge_latency_ts = 0;
0533     gb->gbphy_latency_ts = 0;
0534 
0535     request = kmalloc(len + sizeof(*request), GFP_KERNEL);
0536     if (!request)
0537         return -ENOMEM;
0538     response = kmalloc(len + sizeof(*response), GFP_KERNEL);
0539     if (!response) {
0540         kfree(request);
0541         return -ENOMEM;
0542     }
0543 
0544     memset(request->data, 0x5A, len);
0545 
0546     request->len = cpu_to_le32(len);
0547     retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER,
0548                         request, len + sizeof(*request),
0549                         response, len + sizeof(*response));
0550     if (retval)
0551         goto gb_error;
0552 
0553     if (memcmp(request->data, response->data, len)) {
0554         dev_err(&gb->connection->bundle->dev,
0555             "Loopback Data doesn't match\n");
0556         retval = -EREMOTEIO;
0557     }
0558     gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0);
0559     gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1);
0560 
0561 gb_error:
0562     kfree(request);
0563     kfree(response);
0564 
0565     return retval;
0566 }
0567 
0568 static int gb_loopback_sync_ping(struct gb_loopback *gb)
0569 {
0570     return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING,
0571                       NULL, 0, NULL, 0);
0572 }
0573 
0574 static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len)
0575 {
0576     struct gb_loopback_transfer_request *request;
0577     int retval;
0578 
0579     request = kmalloc(len + sizeof(*request), GFP_KERNEL);
0580     if (!request)
0581         return -ENOMEM;
0582 
0583     request->len = cpu_to_le32(len);
0584     retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK,
0585                          request, len + sizeof(*request),
0586                          0, NULL);
0587     kfree(request);
0588     return retval;
0589 }
0590 
0591 static int gb_loopback_async_transfer_complete(
0592                 struct gb_loopback_async_operation *op_async)
0593 {
0594     struct gb_loopback *gb;
0595     struct gb_operation *operation;
0596     struct gb_loopback_transfer_request *request;
0597     struct gb_loopback_transfer_response *response;
0598     size_t len;
0599     int retval = 0;
0600 
0601     gb = op_async->gb;
0602     operation = op_async->operation;
0603     request = operation->request->payload;
0604     response = operation->response->payload;
0605     len = le32_to_cpu(request->len);
0606 
0607     if (memcmp(request->data, response->data, len)) {
0608         dev_err(&gb->connection->bundle->dev,
0609             "Loopback Data doesn't match operation id %d\n",
0610             operation->id);
0611         retval = -EREMOTEIO;
0612     } else {
0613         gb->apbridge_latency_ts =
0614             (u32)__le32_to_cpu(response->reserved0);
0615         gb->gbphy_latency_ts =
0616             (u32)__le32_to_cpu(response->reserved1);
0617     }
0618 
0619     return retval;
0620 }
0621 
0622 static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len)
0623 {
0624     struct gb_loopback_transfer_request *request;
0625     int retval, response_len;
0626 
0627     request = kmalloc(len + sizeof(*request), GFP_KERNEL);
0628     if (!request)
0629         return -ENOMEM;
0630 
0631     memset(request->data, 0x5A, len);
0632 
0633     request->len = cpu_to_le32(len);
0634     response_len = sizeof(struct gb_loopback_transfer_response);
0635     retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER,
0636                          request, len + sizeof(*request),
0637                          len + response_len,
0638                          gb_loopback_async_transfer_complete);
0639     if (retval)
0640         goto gb_error;
0641 
0642 gb_error:
0643     kfree(request);
0644     return retval;
0645 }
0646 
0647 static int gb_loopback_async_ping(struct gb_loopback *gb)
0648 {
0649     return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING,
0650                        NULL, 0, 0, NULL);
0651 }
0652 
0653 static int gb_loopback_request_handler(struct gb_operation *operation)
0654 {
0655     struct gb_connection *connection = operation->connection;
0656     struct gb_loopback_transfer_request *request;
0657     struct gb_loopback_transfer_response *response;
0658     struct device *dev = &connection->bundle->dev;
0659     size_t len;
0660 
0661     /* By convention, the AP initiates the version operation */
0662     switch (operation->type) {
0663     case GB_LOOPBACK_TYPE_PING:
0664     case GB_LOOPBACK_TYPE_SINK:
0665         return 0;
0666     case GB_LOOPBACK_TYPE_TRANSFER:
0667         if (operation->request->payload_size < sizeof(*request)) {
0668             dev_err(dev, "transfer request too small (%zu < %zu)\n",
0669                 operation->request->payload_size,
0670                 sizeof(*request));
0671             return -EINVAL; /* -EMSGSIZE */
0672         }
0673         request = operation->request->payload;
0674         len = le32_to_cpu(request->len);
0675         if (len > gb_dev.size_max) {
0676             dev_err(dev, "transfer request too large (%zu > %zu)\n",
0677                 len, gb_dev.size_max);
0678             return -EINVAL;
0679         }
0680 
0681         if (!gb_operation_response_alloc(operation,
0682                 len + sizeof(*response), GFP_KERNEL)) {
0683             dev_err(dev, "error allocating response\n");
0684             return -ENOMEM;
0685         }
0686         response = operation->response->payload;
0687         response->len = cpu_to_le32(len);
0688         if (len)
0689             memcpy(response->data, request->data, len);
0690 
0691         return 0;
0692     default:
0693         dev_err(dev, "unsupported request: %u\n", operation->type);
0694         return -EINVAL;
0695     }
0696 }
0697 
0698 static void gb_loopback_reset_stats(struct gb_loopback *gb)
0699 {
0700     struct gb_loopback_stats reset = {
0701         .min = U32_MAX,
0702     };
0703 
0704     /* Reset per-connection stats */
0705     memcpy(&gb->latency, &reset,
0706            sizeof(struct gb_loopback_stats));
0707     memcpy(&gb->throughput, &reset,
0708            sizeof(struct gb_loopback_stats));
0709     memcpy(&gb->requests_per_second, &reset,
0710            sizeof(struct gb_loopback_stats));
0711     memcpy(&gb->apbridge_unipro_latency, &reset,
0712            sizeof(struct gb_loopback_stats));
0713     memcpy(&gb->gbphy_firmware_latency, &reset,
0714            sizeof(struct gb_loopback_stats));
0715 
0716     /* Should be initialized at least once per transaction set */
0717     gb->apbridge_latency_ts = 0;
0718     gb->gbphy_latency_ts = 0;
0719     gb->ts = ktime_set(0, 0);
0720 }
0721 
0722 static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val)
0723 {
0724     if (stats->min > val)
0725         stats->min = val;
0726     if (stats->max < val)
0727         stats->max = val;
0728     stats->sum += val;
0729     stats->count++;
0730 }
0731 
0732 static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats,
0733                         u64 val, u32 count)
0734 {
0735     stats->sum += val;
0736     stats->count += count;
0737 
0738     do_div(val, count);
0739     if (stats->min > val)
0740         stats->min = val;
0741     if (stats->max < val)
0742         stats->max = val;
0743 }
0744 
0745 static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency)
0746 {
0747     u64 req = gb->requests_completed * USEC_PER_SEC;
0748 
0749     gb_loopback_update_stats_window(&gb->requests_per_second, req, latency);
0750 }
0751 
0752 static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency)
0753 {
0754     u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2;
0755 
0756     switch (gb->type) {
0757     case GB_LOOPBACK_TYPE_PING:
0758         break;
0759     case GB_LOOPBACK_TYPE_SINK:
0760         aggregate_size += sizeof(struct gb_loopback_transfer_request) +
0761                   gb->size;
0762         break;
0763     case GB_LOOPBACK_TYPE_TRANSFER:
0764         aggregate_size += sizeof(struct gb_loopback_transfer_request) +
0765                   sizeof(struct gb_loopback_transfer_response) +
0766                   gb->size * 2;
0767         break;
0768     default:
0769         return;
0770     }
0771 
0772     aggregate_size *= gb->requests_completed;
0773     aggregate_size *= USEC_PER_SEC;
0774     gb_loopback_update_stats_window(&gb->throughput, aggregate_size,
0775                     latency);
0776 }
0777 
0778 static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb)
0779 {
0780     u32 lat;
0781 
0782     /* Express latency in terms of microseconds */
0783     lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs);
0784 
0785     /* Log latency stastic */
0786     gb_loopback_update_stats(&gb->latency, lat);
0787 
0788     /* Raw latency log on a per thread basis */
0789     kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat));
0790 
0791     /* Log the firmware supplied latency values */
0792     gb_loopback_update_stats(&gb->apbridge_unipro_latency,
0793                  gb->apbridge_latency_ts);
0794     gb_loopback_update_stats(&gb->gbphy_firmware_latency,
0795                  gb->gbphy_latency_ts);
0796 }
0797 
0798 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error)
0799 {
0800     u64 nlat;
0801     u32 lat;
0802     ktime_t te;
0803 
0804     if (!error) {
0805         gb->requests_completed++;
0806         gb_loopback_calculate_latency_stats(gb);
0807     }
0808 
0809     te = ktime_get();
0810     nlat = gb_loopback_calc_latency(gb->ts, te);
0811     if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) {
0812         lat = gb_loopback_nsec_to_usec_latency(nlat);
0813 
0814         gb_loopback_throughput_update(gb, lat);
0815         gb_loopback_requests_update(gb, lat);
0816 
0817         if (gb->iteration_count != gb->iteration_max) {
0818             gb->ts = te;
0819             gb->requests_completed = 0;
0820         }
0821     }
0822 }
0823 
0824 static void gb_loopback_async_wait_to_send(struct gb_loopback *gb)
0825 {
0826     if (!(gb->async && gb->outstanding_operations_max))
0827         return;
0828     wait_event_interruptible(gb->wq_completion,
0829                  (atomic_read(&gb->outstanding_operations) <
0830                   gb->outstanding_operations_max) ||
0831                   kthread_should_stop());
0832 }
0833 
0834 static int gb_loopback_fn(void *data)
0835 {
0836     int error = 0;
0837     int us_wait = 0;
0838     int type;
0839     int ret;
0840     u32 size;
0841 
0842     struct gb_loopback *gb = data;
0843     struct gb_bundle *bundle = gb->connection->bundle;
0844 
0845     ret = gb_pm_runtime_get_sync(bundle);
0846     if (ret)
0847         return ret;
0848 
0849     while (1) {
0850         if (!gb->type) {
0851             gb_pm_runtime_put_autosuspend(bundle);
0852             wait_event_interruptible(gb->wq, gb->type ||
0853                          kthread_should_stop());
0854             ret = gb_pm_runtime_get_sync(bundle);
0855             if (ret)
0856                 return ret;
0857         }
0858 
0859         if (kthread_should_stop())
0860             break;
0861 
0862         /* Limit the maximum number of in-flight async operations */
0863         gb_loopback_async_wait_to_send(gb);
0864         if (kthread_should_stop())
0865             break;
0866 
0867         mutex_lock(&gb->mutex);
0868 
0869         /* Optionally terminate */
0870         if (gb->send_count == gb->iteration_max) {
0871             mutex_unlock(&gb->mutex);
0872 
0873             /* Wait for synchronous and asynchronous completion */
0874             gb_loopback_async_wait_all(gb);
0875 
0876             /* Mark complete unless user-space has poked us */
0877             mutex_lock(&gb->mutex);
0878             if (gb->iteration_count == gb->iteration_max) {
0879                 gb->type = 0;
0880                 gb->send_count = 0;
0881                 sysfs_notify(&gb->dev->kobj,  NULL,
0882                          "iteration_count");
0883                 dev_dbg(&bundle->dev, "load test complete\n");
0884             } else {
0885                 dev_dbg(&bundle->dev,
0886                     "continuing on with new test set\n");
0887             }
0888             mutex_unlock(&gb->mutex);
0889             continue;
0890         }
0891         size = gb->size;
0892         us_wait = gb->us_wait;
0893         type = gb->type;
0894         if (ktime_to_ns(gb->ts) == 0)
0895             gb->ts = ktime_get();
0896 
0897         /* Else operations to perform */
0898         if (gb->async) {
0899             if (type == GB_LOOPBACK_TYPE_PING)
0900                 error = gb_loopback_async_ping(gb);
0901             else if (type == GB_LOOPBACK_TYPE_TRANSFER)
0902                 error = gb_loopback_async_transfer(gb, size);
0903             else if (type == GB_LOOPBACK_TYPE_SINK)
0904                 error = gb_loopback_async_sink(gb, size);
0905 
0906             if (error) {
0907                 gb->error++;
0908                 gb->iteration_count++;
0909             }
0910         } else {
0911             /* We are effectively single threaded here */
0912             if (type == GB_LOOPBACK_TYPE_PING)
0913                 error = gb_loopback_sync_ping(gb);
0914             else if (type == GB_LOOPBACK_TYPE_TRANSFER)
0915                 error = gb_loopback_sync_transfer(gb, size);
0916             else if (type == GB_LOOPBACK_TYPE_SINK)
0917                 error = gb_loopback_sync_sink(gb, size);
0918 
0919             if (error)
0920                 gb->error++;
0921             gb->iteration_count++;
0922             gb_loopback_calculate_stats(gb, !!error);
0923         }
0924         gb->send_count++;
0925         mutex_unlock(&gb->mutex);
0926 
0927         if (us_wait) {
0928             if (us_wait < 20000)
0929                 usleep_range(us_wait, us_wait + 100);
0930             else
0931                 msleep(us_wait / 1000);
0932         }
0933     }
0934 
0935     gb_pm_runtime_put_autosuspend(bundle);
0936 
0937     return 0;
0938 }
0939 
0940 static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s,
0941                          struct kfifo *kfifo,
0942                          struct mutex *mutex)
0943 {
0944     u32 latency;
0945     int retval;
0946 
0947     if (kfifo_len(kfifo) == 0) {
0948         retval = -EAGAIN;
0949         goto done;
0950     }
0951 
0952     mutex_lock(mutex);
0953     retval = kfifo_out(kfifo, &latency, sizeof(latency));
0954     if (retval > 0) {
0955         seq_printf(s, "%u", latency);
0956         retval = 0;
0957     }
0958     mutex_unlock(mutex);
0959 done:
0960     return retval;
0961 }
0962 
0963 static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused)
0964 {
0965     struct gb_loopback *gb = s->private;
0966 
0967     return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat,
0968                              &gb->mutex);
0969 }
0970 DEFINE_SHOW_ATTRIBUTE(gb_loopback_dbgfs_latency);
0971 
0972 #define DEBUGFS_NAMELEN 32
0973 
0974 static int gb_loopback_probe(struct gb_bundle *bundle,
0975                  const struct greybus_bundle_id *id)
0976 {
0977     struct greybus_descriptor_cport *cport_desc;
0978     struct gb_connection *connection;
0979     struct gb_loopback *gb;
0980     struct device *dev;
0981     int retval;
0982     char name[DEBUGFS_NAMELEN];
0983     unsigned long flags;
0984 
0985     if (bundle->num_cports != 1)
0986         return -ENODEV;
0987 
0988     cport_desc = &bundle->cport_desc[0];
0989     if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK)
0990         return -ENODEV;
0991 
0992     gb = kzalloc(sizeof(*gb), GFP_KERNEL);
0993     if (!gb)
0994         return -ENOMEM;
0995 
0996     connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
0997                       gb_loopback_request_handler);
0998     if (IS_ERR(connection)) {
0999         retval = PTR_ERR(connection);
1000         goto out_kzalloc;
1001     }
1002 
1003     gb->connection = connection;
1004     greybus_set_drvdata(bundle, gb);
1005 
1006     init_waitqueue_head(&gb->wq);
1007     init_waitqueue_head(&gb->wq_completion);
1008     atomic_set(&gb->outstanding_operations, 0);
1009     gb_loopback_reset_stats(gb);
1010 
1011     /* Reported values to user-space for min/max timeouts */
1012     gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN);
1013     gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX);
1014 
1015     if (!gb_dev.count) {
1016         /* Calculate maximum payload */
1017         gb_dev.size_max = gb_operation_get_payload_size_max(connection);
1018         if (gb_dev.size_max <=
1019             sizeof(struct gb_loopback_transfer_request)) {
1020             retval = -EINVAL;
1021             goto out_connection_destroy;
1022         }
1023         gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request);
1024     }
1025 
1026     /* Create per-connection sysfs and debugfs data-points */
1027     snprintf(name, sizeof(name), "raw_latency_%s",
1028          dev_name(&connection->bundle->dev));
1029     gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb,
1030                        &gb_loopback_dbgfs_latency_fops);
1031 
1032     gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL);
1033     if (gb->id < 0) {
1034         retval = gb->id;
1035         goto out_debugfs_remove;
1036     }
1037 
1038     retval = gb_connection_enable(connection);
1039     if (retval)
1040         goto out_ida_remove;
1041 
1042     dev = device_create_with_groups(&loopback_class,
1043                     &connection->bundle->dev,
1044                     MKDEV(0, 0), gb, loopback_groups,
1045                     "gb_loopback%d", gb->id);
1046     if (IS_ERR(dev)) {
1047         retval = PTR_ERR(dev);
1048         goto out_connection_disable;
1049     }
1050     gb->dev = dev;
1051 
1052     /* Allocate kfifo */
1053     if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32),
1054             GFP_KERNEL)) {
1055         retval = -ENOMEM;
1056         goto out_conn;
1057     }
1058     /* Fork worker thread */
1059     mutex_init(&gb->mutex);
1060     gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback");
1061     if (IS_ERR(gb->task)) {
1062         retval = PTR_ERR(gb->task);
1063         goto out_kfifo;
1064     }
1065 
1066     spin_lock_irqsave(&gb_dev.lock, flags);
1067     gb_dev.count++;
1068     spin_unlock_irqrestore(&gb_dev.lock, flags);
1069 
1070     gb_connection_latency_tag_enable(connection);
1071 
1072     gb_pm_runtime_put_autosuspend(bundle);
1073 
1074     return 0;
1075 
1076 out_kfifo:
1077     kfifo_free(&gb->kfifo_lat);
1078 out_conn:
1079     device_unregister(dev);
1080 out_connection_disable:
1081     gb_connection_disable(connection);
1082 out_ida_remove:
1083     ida_simple_remove(&loopback_ida, gb->id);
1084 out_debugfs_remove:
1085     debugfs_remove(gb->file);
1086 out_connection_destroy:
1087     gb_connection_destroy(connection);
1088 out_kzalloc:
1089     kfree(gb);
1090 
1091     return retval;
1092 }
1093 
1094 static void gb_loopback_disconnect(struct gb_bundle *bundle)
1095 {
1096     struct gb_loopback *gb = greybus_get_drvdata(bundle);
1097     unsigned long flags;
1098     int ret;
1099 
1100     ret = gb_pm_runtime_get_sync(bundle);
1101     if (ret)
1102         gb_pm_runtime_get_noresume(bundle);
1103 
1104     gb_connection_disable(gb->connection);
1105 
1106     if (!IS_ERR_OR_NULL(gb->task))
1107         kthread_stop(gb->task);
1108 
1109     kfifo_free(&gb->kfifo_lat);
1110     gb_connection_latency_tag_disable(gb->connection);
1111     debugfs_remove(gb->file);
1112 
1113     /*
1114      * FIXME: gb_loopback_async_wait_all() is redundant now, as connection
1115      * is disabled at the beginning and so we can't have any more
1116      * incoming/outgoing requests.
1117      */
1118     gb_loopback_async_wait_all(gb);
1119 
1120     spin_lock_irqsave(&gb_dev.lock, flags);
1121     gb_dev.count--;
1122     spin_unlock_irqrestore(&gb_dev.lock, flags);
1123 
1124     device_unregister(gb->dev);
1125     ida_simple_remove(&loopback_ida, gb->id);
1126 
1127     gb_connection_destroy(gb->connection);
1128     kfree(gb);
1129 }
1130 
1131 static const struct greybus_bundle_id gb_loopback_id_table[] = {
1132     { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) },
1133     { }
1134 };
1135 MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table);
1136 
1137 static struct greybus_driver gb_loopback_driver = {
1138     .name       = "loopback",
1139     .probe      = gb_loopback_probe,
1140     .disconnect = gb_loopback_disconnect,
1141     .id_table   = gb_loopback_id_table,
1142 };
1143 
1144 static int loopback_init(void)
1145 {
1146     int retval;
1147 
1148     spin_lock_init(&gb_dev.lock);
1149     gb_dev.root = debugfs_create_dir("gb_loopback", NULL);
1150 
1151     retval = class_register(&loopback_class);
1152     if (retval)
1153         goto err;
1154 
1155     retval = greybus_register(&gb_loopback_driver);
1156     if (retval)
1157         goto err_unregister;
1158 
1159     return 0;
1160 
1161 err_unregister:
1162     class_unregister(&loopback_class);
1163 err:
1164     debugfs_remove_recursive(gb_dev.root);
1165     return retval;
1166 }
1167 module_init(loopback_init);
1168 
1169 static void __exit loopback_exit(void)
1170 {
1171     debugfs_remove_recursive(gb_dev.root);
1172     greybus_deregister(&gb_loopback_driver);
1173     class_unregister(&loopback_class);
1174     ida_destroy(&loopback_ida);
1175 }
1176 module_exit(loopback_exit);
1177 
1178 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!