0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012 #include <linux/sunrpc/svc_xprt.h>
0013 #include <linux/slab.h>
0014 #include <linux/vmalloc.h>
0015 #include <linux/sunrpc/addr.h>
0016 #include <linux/highmem.h>
0017 #include <linux/log2.h>
0018 #include <linux/hash.h>
0019 #include <net/checksum.h>
0020
0021 #include "nfsd.h"
0022 #include "cache.h"
0023 #include "trace.h"
0024
0025
0026
0027
0028
0029
0030 #define TARGET_BUCKET_SIZE 64
0031
0032 struct nfsd_drc_bucket {
0033 struct rb_root rb_head;
0034 struct list_head lru_head;
0035 spinlock_t cache_lock;
0036 };
0037
0038 static struct kmem_cache *drc_slab;
0039
0040 static int nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *vec);
0041 static unsigned long nfsd_reply_cache_count(struct shrinker *shrink,
0042 struct shrink_control *sc);
0043 static unsigned long nfsd_reply_cache_scan(struct shrinker *shrink,
0044 struct shrink_control *sc);
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067 static unsigned int
0068 nfsd_cache_size_limit(void)
0069 {
0070 unsigned int limit;
0071 unsigned long low_pages = totalram_pages() - totalhigh_pages();
0072
0073 limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10);
0074 return min_t(unsigned int, limit, 256*1024);
0075 }
0076
0077
0078
0079
0080
0081 static unsigned int
0082 nfsd_hashsize(unsigned int limit)
0083 {
0084 return roundup_pow_of_two(limit / TARGET_BUCKET_SIZE);
0085 }
0086
0087 static struct svc_cacherep *
0088 nfsd_reply_cache_alloc(struct svc_rqst *rqstp, __wsum csum,
0089 struct nfsd_net *nn)
0090 {
0091 struct svc_cacherep *rp;
0092
0093 rp = kmem_cache_alloc(drc_slab, GFP_KERNEL);
0094 if (rp) {
0095 rp->c_state = RC_UNUSED;
0096 rp->c_type = RC_NOCACHE;
0097 RB_CLEAR_NODE(&rp->c_node);
0098 INIT_LIST_HEAD(&rp->c_lru);
0099
0100 memset(&rp->c_key, 0, sizeof(rp->c_key));
0101 rp->c_key.k_xid = rqstp->rq_xid;
0102 rp->c_key.k_proc = rqstp->rq_proc;
0103 rpc_copy_addr((struct sockaddr *)&rp->c_key.k_addr, svc_addr(rqstp));
0104 rpc_set_port((struct sockaddr *)&rp->c_key.k_addr, rpc_get_port(svc_addr(rqstp)));
0105 rp->c_key.k_prot = rqstp->rq_prot;
0106 rp->c_key.k_vers = rqstp->rq_vers;
0107 rp->c_key.k_len = rqstp->rq_arg.len;
0108 rp->c_key.k_csum = csum;
0109 }
0110 return rp;
0111 }
0112
0113 static void
0114 nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct svc_cacherep *rp,
0115 struct nfsd_net *nn)
0116 {
0117 if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) {
0118 nfsd_stats_drc_mem_usage_sub(nn, rp->c_replvec.iov_len);
0119 kfree(rp->c_replvec.iov_base);
0120 }
0121 if (rp->c_state != RC_UNUSED) {
0122 rb_erase(&rp->c_node, &b->rb_head);
0123 list_del(&rp->c_lru);
0124 atomic_dec(&nn->num_drc_entries);
0125 nfsd_stats_drc_mem_usage_sub(nn, sizeof(*rp));
0126 }
0127 kmem_cache_free(drc_slab, rp);
0128 }
0129
0130 static void
0131 nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct svc_cacherep *rp,
0132 struct nfsd_net *nn)
0133 {
0134 spin_lock(&b->cache_lock);
0135 nfsd_reply_cache_free_locked(b, rp, nn);
0136 spin_unlock(&b->cache_lock);
0137 }
0138
0139 int nfsd_drc_slab_create(void)
0140 {
0141 drc_slab = kmem_cache_create("nfsd_drc",
0142 sizeof(struct svc_cacherep), 0, 0, NULL);
0143 return drc_slab ? 0: -ENOMEM;
0144 }
0145
0146 void nfsd_drc_slab_free(void)
0147 {
0148 kmem_cache_destroy(drc_slab);
0149 }
0150
0151 static int nfsd_reply_cache_stats_init(struct nfsd_net *nn)
0152 {
0153 return nfsd_percpu_counters_init(nn->counter, NFSD_NET_COUNTERS_NUM);
0154 }
0155
0156 static void nfsd_reply_cache_stats_destroy(struct nfsd_net *nn)
0157 {
0158 nfsd_percpu_counters_destroy(nn->counter, NFSD_NET_COUNTERS_NUM);
0159 }
0160
0161 int nfsd_reply_cache_init(struct nfsd_net *nn)
0162 {
0163 unsigned int hashsize;
0164 unsigned int i;
0165 int status = 0;
0166
0167 nn->max_drc_entries = nfsd_cache_size_limit();
0168 atomic_set(&nn->num_drc_entries, 0);
0169 hashsize = nfsd_hashsize(nn->max_drc_entries);
0170 nn->maskbits = ilog2(hashsize);
0171
0172 status = nfsd_reply_cache_stats_init(nn);
0173 if (status)
0174 goto out_nomem;
0175
0176 nn->nfsd_reply_cache_shrinker.scan_objects = nfsd_reply_cache_scan;
0177 nn->nfsd_reply_cache_shrinker.count_objects = nfsd_reply_cache_count;
0178 nn->nfsd_reply_cache_shrinker.seeks = 1;
0179 status = register_shrinker(&nn->nfsd_reply_cache_shrinker,
0180 "nfsd-reply:%s", nn->nfsd_name);
0181 if (status)
0182 goto out_stats_destroy;
0183
0184 nn->drc_hashtbl = kvzalloc(array_size(hashsize,
0185 sizeof(*nn->drc_hashtbl)), GFP_KERNEL);
0186 if (!nn->drc_hashtbl)
0187 goto out_shrinker;
0188
0189 for (i = 0; i < hashsize; i++) {
0190 INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head);
0191 spin_lock_init(&nn->drc_hashtbl[i].cache_lock);
0192 }
0193 nn->drc_hashsize = hashsize;
0194
0195 return 0;
0196 out_shrinker:
0197 unregister_shrinker(&nn->nfsd_reply_cache_shrinker);
0198 out_stats_destroy:
0199 nfsd_reply_cache_stats_destroy(nn);
0200 out_nomem:
0201 printk(KERN_ERR "nfsd: failed to allocate reply cache\n");
0202 return -ENOMEM;
0203 }
0204
0205 void nfsd_reply_cache_shutdown(struct nfsd_net *nn)
0206 {
0207 struct svc_cacherep *rp;
0208 unsigned int i;
0209
0210 unregister_shrinker(&nn->nfsd_reply_cache_shrinker);
0211
0212 for (i = 0; i < nn->drc_hashsize; i++) {
0213 struct list_head *head = &nn->drc_hashtbl[i].lru_head;
0214 while (!list_empty(head)) {
0215 rp = list_first_entry(head, struct svc_cacherep, c_lru);
0216 nfsd_reply_cache_free_locked(&nn->drc_hashtbl[i],
0217 rp, nn);
0218 }
0219 }
0220 nfsd_reply_cache_stats_destroy(nn);
0221
0222 kvfree(nn->drc_hashtbl);
0223 nn->drc_hashtbl = NULL;
0224 nn->drc_hashsize = 0;
0225
0226 }
0227
0228
0229
0230
0231
0232 static void
0233 lru_put_end(struct nfsd_drc_bucket *b, struct svc_cacherep *rp)
0234 {
0235 rp->c_timestamp = jiffies;
0236 list_move_tail(&rp->c_lru, &b->lru_head);
0237 }
0238
0239 static noinline struct nfsd_drc_bucket *
0240 nfsd_cache_bucket_find(__be32 xid, struct nfsd_net *nn)
0241 {
0242 unsigned int hash = hash_32((__force u32)xid, nn->maskbits);
0243
0244 return &nn->drc_hashtbl[hash];
0245 }
0246
0247 static long prune_bucket(struct nfsd_drc_bucket *b, struct nfsd_net *nn,
0248 unsigned int max)
0249 {
0250 struct svc_cacherep *rp, *tmp;
0251 long freed = 0;
0252
0253 list_for_each_entry_safe(rp, tmp, &b->lru_head, c_lru) {
0254
0255
0256
0257
0258 if (rp->c_state == RC_INPROG)
0259 continue;
0260 if (atomic_read(&nn->num_drc_entries) <= nn->max_drc_entries &&
0261 time_before(jiffies, rp->c_timestamp + RC_EXPIRE))
0262 break;
0263 nfsd_reply_cache_free_locked(b, rp, nn);
0264 if (max && freed++ > max)
0265 break;
0266 }
0267 return freed;
0268 }
0269
0270 static long nfsd_prune_bucket(struct nfsd_drc_bucket *b, struct nfsd_net *nn)
0271 {
0272 return prune_bucket(b, nn, 3);
0273 }
0274
0275
0276
0277
0278
0279 static long
0280 prune_cache_entries(struct nfsd_net *nn)
0281 {
0282 unsigned int i;
0283 long freed = 0;
0284
0285 for (i = 0; i < nn->drc_hashsize; i++) {
0286 struct nfsd_drc_bucket *b = &nn->drc_hashtbl[i];
0287
0288 if (list_empty(&b->lru_head))
0289 continue;
0290 spin_lock(&b->cache_lock);
0291 freed += prune_bucket(b, nn, 0);
0292 spin_unlock(&b->cache_lock);
0293 }
0294 return freed;
0295 }
0296
0297 static unsigned long
0298 nfsd_reply_cache_count(struct shrinker *shrink, struct shrink_control *sc)
0299 {
0300 struct nfsd_net *nn = container_of(shrink,
0301 struct nfsd_net, nfsd_reply_cache_shrinker);
0302
0303 return atomic_read(&nn->num_drc_entries);
0304 }
0305
0306 static unsigned long
0307 nfsd_reply_cache_scan(struct shrinker *shrink, struct shrink_control *sc)
0308 {
0309 struct nfsd_net *nn = container_of(shrink,
0310 struct nfsd_net, nfsd_reply_cache_shrinker);
0311
0312 return prune_cache_entries(nn);
0313 }
0314
0315
0316
0317 static __wsum
0318 nfsd_cache_csum(struct svc_rqst *rqstp)
0319 {
0320 int idx;
0321 unsigned int base;
0322 __wsum csum;
0323 struct xdr_buf *buf = &rqstp->rq_arg;
0324 const unsigned char *p = buf->head[0].iov_base;
0325 size_t csum_len = min_t(size_t, buf->head[0].iov_len + buf->page_len,
0326 RC_CSUMLEN);
0327 size_t len = min(buf->head[0].iov_len, csum_len);
0328
0329
0330 csum = csum_partial(p, len, 0);
0331 csum_len -= len;
0332
0333
0334 idx = buf->page_base / PAGE_SIZE;
0335 base = buf->page_base & ~PAGE_MASK;
0336 while (csum_len) {
0337 p = page_address(buf->pages[idx]) + base;
0338 len = min_t(size_t, PAGE_SIZE - base, csum_len);
0339 csum = csum_partial(p, len, csum);
0340 csum_len -= len;
0341 base = 0;
0342 ++idx;
0343 }
0344 return csum;
0345 }
0346
0347 static int
0348 nfsd_cache_key_cmp(const struct svc_cacherep *key,
0349 const struct svc_cacherep *rp, struct nfsd_net *nn)
0350 {
0351 if (key->c_key.k_xid == rp->c_key.k_xid &&
0352 key->c_key.k_csum != rp->c_key.k_csum) {
0353 nfsd_stats_payload_misses_inc(nn);
0354 trace_nfsd_drc_mismatch(nn, key, rp);
0355 }
0356
0357 return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key));
0358 }
0359
0360
0361
0362
0363
0364
0365 static struct svc_cacherep *
0366 nfsd_cache_insert(struct nfsd_drc_bucket *b, struct svc_cacherep *key,
0367 struct nfsd_net *nn)
0368 {
0369 struct svc_cacherep *rp, *ret = key;
0370 struct rb_node **p = &b->rb_head.rb_node,
0371 *parent = NULL;
0372 unsigned int entries = 0;
0373 int cmp;
0374
0375 while (*p != NULL) {
0376 ++entries;
0377 parent = *p;
0378 rp = rb_entry(parent, struct svc_cacherep, c_node);
0379
0380 cmp = nfsd_cache_key_cmp(key, rp, nn);
0381 if (cmp < 0)
0382 p = &parent->rb_left;
0383 else if (cmp > 0)
0384 p = &parent->rb_right;
0385 else {
0386 ret = rp;
0387 goto out;
0388 }
0389 }
0390 rb_link_node(&key->c_node, parent, p);
0391 rb_insert_color(&key->c_node, &b->rb_head);
0392 out:
0393
0394 if (entries > nn->longest_chain) {
0395 nn->longest_chain = entries;
0396 nn->longest_chain_cachesize = atomic_read(&nn->num_drc_entries);
0397 } else if (entries == nn->longest_chain) {
0398
0399 nn->longest_chain_cachesize = min_t(unsigned int,
0400 nn->longest_chain_cachesize,
0401 atomic_read(&nn->num_drc_entries));
0402 }
0403
0404 lru_put_end(b, ret);
0405 return ret;
0406 }
0407
0408
0409
0410
0411
0412
0413
0414
0415
0416
0417
0418
0419
0420
0421
0422
0423 int nfsd_cache_lookup(struct svc_rqst *rqstp)
0424 {
0425 struct nfsd_net *nn;
0426 struct svc_cacherep *rp, *found;
0427 __wsum csum;
0428 struct nfsd_drc_bucket *b;
0429 int type = rqstp->rq_cachetype;
0430 int rtn = RC_DOIT;
0431
0432 rqstp->rq_cacherep = NULL;
0433 if (type == RC_NOCACHE) {
0434 nfsd_stats_rc_nocache_inc();
0435 goto out;
0436 }
0437
0438 csum = nfsd_cache_csum(rqstp);
0439
0440
0441
0442
0443
0444 nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
0445 rp = nfsd_reply_cache_alloc(rqstp, csum, nn);
0446 if (!rp)
0447 goto out;
0448
0449 b = nfsd_cache_bucket_find(rqstp->rq_xid, nn);
0450 spin_lock(&b->cache_lock);
0451 found = nfsd_cache_insert(b, rp, nn);
0452 if (found != rp)
0453 goto found_entry;
0454
0455 nfsd_stats_rc_misses_inc();
0456 rqstp->rq_cacherep = rp;
0457 rp->c_state = RC_INPROG;
0458
0459 atomic_inc(&nn->num_drc_entries);
0460 nfsd_stats_drc_mem_usage_add(nn, sizeof(*rp));
0461
0462 nfsd_prune_bucket(b, nn);
0463
0464 out_unlock:
0465 spin_unlock(&b->cache_lock);
0466 out:
0467 return rtn;
0468
0469 found_entry:
0470
0471 nfsd_reply_cache_free_locked(NULL, rp, nn);
0472 nfsd_stats_rc_hits_inc();
0473 rtn = RC_DROPIT;
0474 rp = found;
0475
0476
0477 if (rp->c_state == RC_INPROG)
0478 goto out_trace;
0479
0480
0481
0482 rtn = RC_DOIT;
0483 if (!test_bit(RQ_SECURE, &rqstp->rq_flags) && rp->c_secure)
0484 goto out_trace;
0485
0486
0487 switch (rp->c_type) {
0488 case RC_NOCACHE:
0489 break;
0490 case RC_REPLSTAT:
0491 svc_putu32(&rqstp->rq_res.head[0], rp->c_replstat);
0492 rtn = RC_REPLY;
0493 break;
0494 case RC_REPLBUFF:
0495 if (!nfsd_cache_append(rqstp, &rp->c_replvec))
0496 goto out_unlock;
0497 rtn = RC_REPLY;
0498 break;
0499 default:
0500 WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp->c_type);
0501 }
0502
0503 out_trace:
0504 trace_nfsd_drc_found(nn, rqstp, rtn);
0505 goto out_unlock;
0506 }
0507
0508
0509
0510
0511
0512
0513
0514
0515
0516
0517
0518
0519
0520
0521
0522
0523
0524
0525
0526
0527
0528 void nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp)
0529 {
0530 struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
0531 struct svc_cacherep *rp = rqstp->rq_cacherep;
0532 struct kvec *resv = &rqstp->rq_res.head[0], *cachv;
0533 struct nfsd_drc_bucket *b;
0534 int len;
0535 size_t bufsize = 0;
0536
0537 if (!rp)
0538 return;
0539
0540 b = nfsd_cache_bucket_find(rp->c_key.k_xid, nn);
0541
0542 len = resv->iov_len - ((char*)statp - (char*)resv->iov_base);
0543 len >>= 2;
0544
0545
0546 if (!statp || len > (256 >> 2)) {
0547 nfsd_reply_cache_free(b, rp, nn);
0548 return;
0549 }
0550
0551 switch (cachetype) {
0552 case RC_REPLSTAT:
0553 if (len != 1)
0554 printk("nfsd: RC_REPLSTAT/reply len %d!\n",len);
0555 rp->c_replstat = *statp;
0556 break;
0557 case RC_REPLBUFF:
0558 cachv = &rp->c_replvec;
0559 bufsize = len << 2;
0560 cachv->iov_base = kmalloc(bufsize, GFP_KERNEL);
0561 if (!cachv->iov_base) {
0562 nfsd_reply_cache_free(b, rp, nn);
0563 return;
0564 }
0565 cachv->iov_len = bufsize;
0566 memcpy(cachv->iov_base, statp, bufsize);
0567 break;
0568 case RC_NOCACHE:
0569 nfsd_reply_cache_free(b, rp, nn);
0570 return;
0571 }
0572 spin_lock(&b->cache_lock);
0573 nfsd_stats_drc_mem_usage_add(nn, bufsize);
0574 lru_put_end(b, rp);
0575 rp->c_secure = test_bit(RQ_SECURE, &rqstp->rq_flags);
0576 rp->c_type = cachetype;
0577 rp->c_state = RC_DONE;
0578 spin_unlock(&b->cache_lock);
0579 return;
0580 }
0581
0582
0583
0584
0585
0586
0587 static int
0588 nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *data)
0589 {
0590 struct kvec *vec = &rqstp->rq_res.head[0];
0591
0592 if (vec->iov_len + data->iov_len > PAGE_SIZE) {
0593 printk(KERN_WARNING "nfsd: cached reply too large (%zd).\n",
0594 data->iov_len);
0595 return 0;
0596 }
0597 memcpy((char*)vec->iov_base + vec->iov_len, data->iov_base, data->iov_len);
0598 vec->iov_len += data->iov_len;
0599 return 1;
0600 }
0601
0602
0603
0604
0605
0606
0607 static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v)
0608 {
0609 struct nfsd_net *nn = m->private;
0610
0611 seq_printf(m, "max entries: %u\n", nn->max_drc_entries);
0612 seq_printf(m, "num entries: %u\n",
0613 atomic_read(&nn->num_drc_entries));
0614 seq_printf(m, "hash buckets: %u\n", 1 << nn->maskbits);
0615 seq_printf(m, "mem usage: %lld\n",
0616 percpu_counter_sum_positive(&nn->counter[NFSD_NET_DRC_MEM_USAGE]));
0617 seq_printf(m, "cache hits: %lld\n",
0618 percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_HITS]));
0619 seq_printf(m, "cache misses: %lld\n",
0620 percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_MISSES]));
0621 seq_printf(m, "not cached: %lld\n",
0622 percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_NOCACHE]));
0623 seq_printf(m, "payload misses: %lld\n",
0624 percpu_counter_sum_positive(&nn->counter[NFSD_NET_PAYLOAD_MISSES]));
0625 seq_printf(m, "longest chain len: %u\n", nn->longest_chain);
0626 seq_printf(m, "cachesize at longest: %u\n", nn->longest_chain_cachesize);
0627 return 0;
0628 }
0629
0630 int nfsd_reply_cache_stats_open(struct inode *inode, struct file *file)
0631 {
0632 struct nfsd_net *nn = net_generic(file_inode(file)->i_sb->s_fs_info,
0633 nfsd_net_id);
0634
0635 return single_open(file, nfsd_reply_cache_stats_show, nn);
0636 }