fs/overlayfs/readdir.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *
0004  * Copyright (C) 2011 Novell Inc.
0005  */
0006
0007 #include <linux/fs.h>
0008 #include <linux/slab.h>
0009 #include <linux/namei.h>
0010 #include <linux/file.h>
0011 #include <linux/xattr.h>
0012 #include <linux/rbtree.h>
0013 #include <linux/security.h>
0014 #include <linux/cred.h>
0015 #include <linux/ratelimit.h>
0016 #include "overlayfs.h"
0017
0018 struct ovl_cache_entry {
0019     unsigned int len;
0020     unsigned int type;
0021     u64 real_ino;
0022     u64 ino;
0023     struct list_head l_node;
0024     struct rb_node node;
0025     struct ovl_cache_entry *next_maybe_whiteout;
0026     bool is_upper;
0027     bool is_whiteout;
0028     char name[];
0029 };
0030
0031 struct ovl_dir_cache {
0032     long refcount;
0033     u64 version;
0034     struct list_head entries;
0035     struct rb_root root;
0036 };
0037
0038 struct ovl_readdir_data {
0039     struct dir_context ctx;
0040     struct dentry *dentry;
0041     bool is_lowest;
0042     struct rb_root *root;
0043     struct list_head *list;
0044     struct list_head middle;
0045     struct ovl_cache_entry *first_maybe_whiteout;
0046     int count;
0047     int err;
0048     bool is_upper;
0049     bool d_type_supported;
0050 };
0051
0052 struct ovl_dir_file {
0053     bool is_real;
0054     bool is_upper;
0055     struct ovl_dir_cache *cache;
0056     struct list_head *cursor;
0057     struct file *realfile;
0058     struct file *upperfile;
0059 };
0060
0061 static struct ovl_cache_entry *ovl_cache_entry_from_node(struct rb_node *n)
0062 {
0063     return rb_entry(n, struct ovl_cache_entry, node);
0064 }
0065
0066 static bool ovl_cache_entry_find_link(const char *name, int len,
0067                       struct rb_node ***link,
0068                       struct rb_node **parent)
0069 {
0070     bool found = false;
0071     struct rb_node **newp = *link;
0072
0073     while (!found && *newp) {
0074         int cmp;
0075         struct ovl_cache_entry *tmp;
0076
0077         *parent = *newp;
0078         tmp = ovl_cache_entry_from_node(*newp);
0079         cmp = strncmp(name, tmp->name, len);
0080         if (cmp > 0)
0081             newp = &tmp->node.rb_right;
0082         else if (cmp < 0 || len < tmp->len)
0083             newp = &tmp->node.rb_left;
0084         else
0085             found = true;
0086     }
0087     *link = newp;
0088
0089     return found;
0090 }
0091
0092 static struct ovl_cache_entry *ovl_cache_entry_find(struct rb_root *root,
0093                             const char *name, int len)
0094 {
0095     struct rb_node *node = root->rb_node;
0096     int cmp;
0097
0098     while (node) {
0099         struct ovl_cache_entry *p = ovl_cache_entry_from_node(node);
0100
0101         cmp = strncmp(name, p->name, len);
0102         if (cmp > 0)
0103             node = p->node.rb_right;
0104         else if (cmp < 0 || len < p->len)
0105             node = p->node.rb_left;
0106         else
0107             return p;
0108     }
0109
0110     return NULL;
0111 }
0112
0113 static bool ovl_calc_d_ino(struct ovl_readdir_data *rdd,
0114                struct ovl_cache_entry *p)
0115 {
0116     /* Don't care if not doing ovl_iter() */
0117     if (!rdd->dentry)
0118         return false;
0119
0120     /* Always recalc d_ino when remapping lower inode numbers */
0121     if (ovl_xino_bits(rdd->dentry->d_sb))
0122         return true;
0123
0124     /* Always recalc d_ino for parent */
0125     if (strcmp(p->name, "..") == 0)
0126         return true;
0127
0128     /* If this is lower, then native d_ino will do */
0129     if (!rdd->is_upper)
0130         return false;
0131
0132     /*
0133      * Recalc d_ino for '.' and for all entries if dir is impure (contains
0134      * copied up entries)
0135      */
0136     if ((p->name[0] == '.' && p->len == 1) ||
0137         ovl_test_flag(OVL_IMPURE, d_inode(rdd->dentry)))
0138         return true;
0139
0140     return false;
0141 }
0142
0143 static struct ovl_cache_entry *ovl_cache_entry_new(struct ovl_readdir_data *rdd,
0144                            const char *name, int len,
0145                            u64 ino, unsigned int d_type)
0146 {
0147     struct ovl_cache_entry *p;
0148     size_t size = offsetof(struct ovl_cache_entry, name[len + 1]);
0149
0150     p = kmalloc(size, GFP_KERNEL);
0151     if (!p)
0152         return NULL;
0153
0154     memcpy(p->name, name, len);
0155     p->name[len] = '\0';
0156     p->len = len;
0157     p->type = d_type;
0158     p->real_ino = ino;
0159     p->ino = ino;
0160     /* Defer setting d_ino for upper entry to ovl_iterate() */
0161     if (ovl_calc_d_ino(rdd, p))
0162         p->ino = 0;
0163     p->is_upper = rdd->is_upper;
0164     p->is_whiteout = false;
0165
0166     if (d_type == DT_CHR) {
0167         p->next_maybe_whiteout = rdd->first_maybe_whiteout;
0168         rdd->first_maybe_whiteout = p;
0169     }
0170     return p;
0171 }
0172
0173 static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd,
0174                   const char *name, int len, u64 ino,
0175                   unsigned int d_type)
0176 {
0177     struct rb_node **newp = &rdd->root->rb_node;
0178     struct rb_node *parent = NULL;
0179     struct ovl_cache_entry *p;
0180
0181     if (ovl_cache_entry_find_link(name, len, &newp, &parent))
0182         return 0;
0183
0184     p = ovl_cache_entry_new(rdd, name, len, ino, d_type);
0185     if (p == NULL) {
0186         rdd->err = -ENOMEM;
0187         return -ENOMEM;
0188     }
0189
0190     list_add_tail(&p->l_node, rdd->list);
0191     rb_link_node(&p->node, parent, newp);
0192     rb_insert_color(&p->node, rdd->root);
0193
0194     return 0;
0195 }
0196
0197 static int ovl_fill_lowest(struct ovl_readdir_data *rdd,
0198                const char *name, int namelen,
0199                loff_t offset, u64 ino, unsigned int d_type)
0200 {
0201     struct ovl_cache_entry *p;
0202
0203     p = ovl_cache_entry_find(rdd->root, name, namelen);
0204     if (p) {
0205         list_move_tail(&p->l_node, &rdd->middle);
0206     } else {
0207         p = ovl_cache_entry_new(rdd, name, namelen, ino, d_type);
0208         if (p == NULL)
0209             rdd->err = -ENOMEM;
0210         else
0211             list_add_tail(&p->l_node, &rdd->middle);
0212     }
0213
0214     return rdd->err;
0215 }
0216
0217 void ovl_cache_free(struct list_head *list)
0218 {
0219     struct ovl_cache_entry *p;
0220     struct ovl_cache_entry *n;
0221
0222     list_for_each_entry_safe(p, n, list, l_node)
0223         kfree(p);
0224
0225     INIT_LIST_HEAD(list);
0226 }
0227
0228 void ovl_dir_cache_free(struct inode *inode)
0229 {
0230     struct ovl_dir_cache *cache = ovl_dir_cache(inode);
0231
0232     if (cache) {
0233         ovl_cache_free(&cache->entries);
0234         kfree(cache);
0235     }
0236 }
0237
0238 static void ovl_cache_put(struct ovl_dir_file *od, struct dentry *dentry)
0239 {
0240     struct ovl_dir_cache *cache = od->cache;
0241
0242     WARN_ON(cache->refcount <= 0);
0243     cache->refcount--;
0244     if (!cache->refcount) {
0245         if (ovl_dir_cache(d_inode(dentry)) == cache)
0246             ovl_set_dir_cache(d_inode(dentry), NULL);
0247
0248         ovl_cache_free(&cache->entries);
0249         kfree(cache);
0250     }
0251 }
0252
0253 static int ovl_fill_merge(struct dir_context *ctx, const char *name,
0254               int namelen, loff_t offset, u64 ino,
0255               unsigned int d_type)
0256 {
0257     struct ovl_readdir_data *rdd =
0258         container_of(ctx, struct ovl_readdir_data, ctx);
0259
0260     rdd->count++;
0261     if (!rdd->is_lowest)
0262         return ovl_cache_entry_add_rb(rdd, name, namelen, ino, d_type);
0263     else
0264         return ovl_fill_lowest(rdd, name, namelen, offset, ino, d_type);
0265 }
0266
0267 static int ovl_check_whiteouts(struct path *path, struct ovl_readdir_data *rdd)
0268 {
0269     int err;
0270     struct ovl_cache_entry *p;
0271     struct dentry *dentry, *dir = path->dentry;
0272     const struct cred *old_cred;
0273
0274     old_cred = ovl_override_creds(rdd->dentry->d_sb);
0275
0276     err = down_write_killable(&dir->d_inode->i_rwsem);
0277     if (!err) {
0278         while (rdd->first_maybe_whiteout) {
0279             p = rdd->first_maybe_whiteout;
0280             rdd->first_maybe_whiteout = p->next_maybe_whiteout;
0281             dentry = lookup_one(mnt_user_ns(path->mnt), p->name, dir, p->len);
0282             if (!IS_ERR(dentry)) {
0283                 p->is_whiteout = ovl_is_whiteout(dentry);
0284                 dput(dentry);
0285             }
0286         }
0287         inode_unlock(dir->d_inode);
0288     }
0289     revert_creds(old_cred);
0290
0291     return err;
0292 }
0293
0294 static inline int ovl_dir_read(struct path *realpath,
0295                    struct ovl_readdir_data *rdd)
0296 {
0297     struct file *realfile;
0298     int err;
0299
0300     realfile = ovl_path_open(realpath, O_RDONLY | O_LARGEFILE);
0301     if (IS_ERR(realfile))
0302         return PTR_ERR(realfile);
0303
0304     rdd->first_maybe_whiteout = NULL;
0305     rdd->ctx.pos = 0;
0306     do {
0307         rdd->count = 0;
0308         rdd->err = 0;
0309         err = iterate_dir(realfile, &rdd->ctx);
0310         if (err >= 0)
0311             err = rdd->err;
0312     } while (!err && rdd->count);
0313
0314     if (!err && rdd->first_maybe_whiteout && rdd->dentry)
0315         err = ovl_check_whiteouts(realpath, rdd);
0316
0317     fput(realfile);
0318
0319     return err;
0320 }
0321
0322 static void ovl_dir_reset(struct file *file)
0323 {
0324     struct ovl_dir_file *od = file->private_data;
0325     struct ovl_dir_cache *cache = od->cache;
0326     struct dentry *dentry = file->f_path.dentry;
0327     bool is_real;
0328
0329     if (cache && ovl_dentry_version_get(dentry) != cache->version) {
0330         ovl_cache_put(od, dentry);
0331         od->cache = NULL;
0332         od->cursor = NULL;
0333     }
0334     is_real = ovl_dir_is_real(dentry);
0335     if (od->is_real != is_real) {
0336         /* is_real can only become false when dir is copied up */
0337         if (WARN_ON(is_real))
0338             return;
0339         od->is_real = false;
0340     }
0341 }
0342
0343 static int ovl_dir_read_merged(struct dentry *dentry, struct list_head *list,
0344     struct rb_root *root)
0345 {
0346     int err;
0347     struct path realpath;
0348     struct ovl_readdir_data rdd = {
0349         .ctx.actor = ovl_fill_merge,
0350         .dentry = dentry,
0351         .list = list,
0352         .root = root,
0353         .is_lowest = false,
0354     };
0355     int idx, next;
0356
0357     for (idx = 0; idx != -1; idx = next) {
0358         next = ovl_path_next(idx, dentry, &realpath);
0359         rdd.is_upper = ovl_dentry_upper(dentry) == realpath.dentry;
0360
0361         if (next != -1) {
0362             err = ovl_dir_read(&realpath, &rdd);
0363             if (err)
0364                 break;
0365         } else {
0366             /*
0367              * Insert lowest layer entries before upper ones, this
0368              * allows offsets to be reasonably constant
0369              */
0370             list_add(&rdd.middle, rdd.list);
0371             rdd.is_lowest = true;
0372             err = ovl_dir_read(&realpath, &rdd);
0373             list_del(&rdd.middle);
0374         }
0375     }
0376     return err;
0377 }
0378
0379 static void ovl_seek_cursor(struct ovl_dir_file *od, loff_t pos)
0380 {
0381     struct list_head *p;
0382     loff_t off = 0;
0383
0384     list_for_each(p, &od->cache->entries) {
0385         if (off >= pos)
0386             break;
0387         off++;
0388     }
0389     /* Cursor is safe since the cache is stable */
0390     od->cursor = p;
0391 }
0392
0393 static struct ovl_dir_cache *ovl_cache_get(struct dentry *dentry)
0394 {
0395     int res;
0396     struct ovl_dir_cache *cache;
0397
0398     cache = ovl_dir_cache(d_inode(dentry));
0399     if (cache && ovl_dentry_version_get(dentry) == cache->version) {
0400         WARN_ON(!cache->refcount);
0401         cache->refcount++;
0402         return cache;
0403     }
0404     ovl_set_dir_cache(d_inode(dentry), NULL);
0405
0406     cache = kzalloc(sizeof(struct ovl_dir_cache), GFP_KERNEL);
0407     if (!cache)
0408         return ERR_PTR(-ENOMEM);
0409
0410     cache->refcount = 1;
0411     INIT_LIST_HEAD(&cache->entries);
0412     cache->root = RB_ROOT;
0413
0414     res = ovl_dir_read_merged(dentry, &cache->entries, &cache->root);
0415     if (res) {
0416         ovl_cache_free(&cache->entries);
0417         kfree(cache);
0418         return ERR_PTR(res);
0419     }
0420
0421     cache->version = ovl_dentry_version_get(dentry);
0422     ovl_set_dir_cache(d_inode(dentry), cache);
0423
0424     return cache;
0425 }
0426
0427 /* Map inode number to lower fs unique range */
0428 static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid,
0429                    const char *name, int namelen, bool warn)
0430 {
0431     unsigned int xinoshift = 64 - xinobits;
0432
0433     if (unlikely(ino >> xinoshift)) {
0434         if (warn) {
0435             pr_warn_ratelimited("d_ino too big (%.*s, ino=%llu, xinobits=%d)\n",
0436                         namelen, name, ino, xinobits);
0437         }
0438         return ino;
0439     }
0440
0441     /*
0442      * The lowest xinobit is reserved for mapping the non-peresistent inode
0443      * numbers range, but this range is only exposed via st_ino, not here.
0444      */
0445     return ino | ((u64)fsid) << (xinoshift + 1);
0446 }
0447
0448 /*
0449  * Set d_ino for upper entries. Non-upper entries should always report
0450  * the uppermost real inode ino and should not call this function.
0451  *
0452  * When not all layer are on same fs, report real ino also for upper.
0453  *
0454  * When all layers are on the same fs, and upper has a reference to
0455  * copy up origin, call vfs_getattr() on the overlay entry to make
0456  * sure that d_ino will be consistent with st_ino from stat(2).
0457  */
0458 static int ovl_cache_update_ino(struct path *path, struct ovl_cache_entry *p)
0459
0460 {
0461     struct dentry *dir = path->dentry;
0462     struct dentry *this = NULL;
0463     enum ovl_path_type type;
0464     u64 ino = p->real_ino;
0465     int xinobits = ovl_xino_bits(dir->d_sb);
0466     int err = 0;
0467
0468     if (!ovl_same_dev(dir->d_sb))
0469         goto out;
0470
0471     if (p->name[0] == '.') {
0472         if (p->len == 1) {
0473             this = dget(dir);
0474             goto get;
0475         }
0476         if (p->len == 2 && p->name[1] == '.') {
0477             /* we shall not be moved */
0478             this = dget(dir->d_parent);
0479             goto get;
0480         }
0481     }
0482     this = lookup_one(mnt_user_ns(path->mnt), p->name, dir, p->len);
0483     if (IS_ERR_OR_NULL(this) || !this->d_inode) {
0484         /* Mark a stale entry */
0485         p->is_whiteout = true;
0486         if (IS_ERR(this)) {
0487             err = PTR_ERR(this);
0488             this = NULL;
0489             goto fail;
0490         }
0491         goto out;
0492     }
0493
0494 get:
0495     type = ovl_path_type(this);
0496     if (OVL_TYPE_ORIGIN(type)) {
0497         struct kstat stat;
0498         struct path statpath = *path;
0499
0500         statpath.dentry = this;
0501         err = vfs_getattr(&statpath, &stat, STATX_INO, 0);
0502         if (err)
0503             goto fail;
0504
0505         /*
0506          * Directory inode is always on overlay st_dev.
0507          * Non-dir with ovl_same_dev() could be on pseudo st_dev in case
0508          * of xino bits overflow.
0509          */
0510         WARN_ON_ONCE(S_ISDIR(stat.mode) &&
0511                  dir->d_sb->s_dev != stat.dev);
0512         ino = stat.ino;
0513     } else if (xinobits && !OVL_TYPE_UPPER(type)) {
0514         ino = ovl_remap_lower_ino(ino, xinobits,
0515                       ovl_layer_lower(this)->fsid,
0516                       p->name, p->len,
0517                       ovl_xino_warn(dir->d_sb));
0518     }
0519
0520 out:
0521     p->ino = ino;
0522     dput(this);
0523     return err;
0524
0525 fail:
0526     pr_warn_ratelimited("failed to look up (%s) for ino (%i)\n",
0527                 p->name, err);
0528     goto out;
0529 }
0530
0531 static int ovl_fill_plain(struct dir_context *ctx, const char *name,
0532               int namelen, loff_t offset, u64 ino,
0533               unsigned int d_type)
0534 {
0535     struct ovl_cache_entry *p;
0536     struct ovl_readdir_data *rdd =
0537         container_of(ctx, struct ovl_readdir_data, ctx);
0538
0539     rdd->count++;
0540     p = ovl_cache_entry_new(rdd, name, namelen, ino, d_type);
0541     if (p == NULL) {
0542         rdd->err = -ENOMEM;
0543         return -ENOMEM;
0544     }
0545     list_add_tail(&p->l_node, rdd->list);
0546
0547     return 0;
0548 }
0549
0550 static int ovl_dir_read_impure(struct path *path,  struct list_head *list,
0551                    struct rb_root *root)
0552 {
0553     int err;
0554     struct path realpath;
0555     struct ovl_cache_entry *p, *n;
0556     struct ovl_readdir_data rdd = {
0557         .ctx.actor = ovl_fill_plain,
0558         .list = list,
0559         .root = root,
0560     };
0561
0562     INIT_LIST_HEAD(list);
0563     *root = RB_ROOT;
0564     ovl_path_upper(path->dentry, &realpath);
0565
0566     err = ovl_dir_read(&realpath, &rdd);
0567     if (err)
0568         return err;
0569
0570     list_for_each_entry_safe(p, n, list, l_node) {
0571         if (strcmp(p->name, ".") != 0 &&
0572             strcmp(p->name, "..") != 0) {
0573             err = ovl_cache_update_ino(path, p);
0574             if (err)
0575                 return err;
0576         }
0577         if (p->ino == p->real_ino) {
0578             list_del(&p->l_node);
0579             kfree(p);
0580         } else {
0581             struct rb_node **newp = &root->rb_node;
0582             struct rb_node *parent = NULL;
0583
0584             if (WARN_ON(ovl_cache_entry_find_link(p->name, p->len,
0585                                   &newp, &parent)))
0586                 return -EIO;
0587
0588             rb_link_node(&p->node, parent, newp);
0589             rb_insert_color(&p->node, root);
0590         }
0591     }
0592     return 0;
0593 }
0594
0595 static struct ovl_dir_cache *ovl_cache_get_impure(struct path *path)
0596 {
0597     int res;
0598     struct dentry *dentry = path->dentry;
0599     struct ovl_fs *ofs = OVL_FS(dentry->d_sb);
0600     struct ovl_dir_cache *cache;
0601
0602     cache = ovl_dir_cache(d_inode(dentry));
0603     if (cache && ovl_dentry_version_get(dentry) == cache->version)
0604         return cache;
0605
0606     /* Impure cache is not refcounted, free it here */
0607     ovl_dir_cache_free(d_inode(dentry));
0608     ovl_set_dir_cache(d_inode(dentry), NULL);
0609
0610     cache = kzalloc(sizeof(struct ovl_dir_cache), GFP_KERNEL);
0611     if (!cache)
0612         return ERR_PTR(-ENOMEM);
0613
0614     res = ovl_dir_read_impure(path, &cache->entries, &cache->root);
0615     if (res) {
0616         ovl_cache_free(&cache->entries);
0617         kfree(cache);
0618         return ERR_PTR(res);
0619     }
0620     if (list_empty(&cache->entries)) {
0621         /*
0622          * A good opportunity to get rid of an unneeded "impure" flag.
0623          * Removing the "impure" xattr is best effort.
0624          */
0625         if (!ovl_want_write(dentry)) {
0626             ovl_removexattr(ofs, ovl_dentry_upper(dentry),
0627                     OVL_XATTR_IMPURE);
0628             ovl_drop_write(dentry);
0629         }
0630         ovl_clear_flag(OVL_IMPURE, d_inode(dentry));
0631         kfree(cache);
0632         return NULL;
0633     }
0634
0635     cache->version = ovl_dentry_version_get(dentry);
0636     ovl_set_dir_cache(d_inode(dentry), cache);
0637
0638     return cache;
0639 }
0640
0641 struct ovl_readdir_translate {
0642     struct dir_context *orig_ctx;
0643     struct ovl_dir_cache *cache;
0644     struct dir_context ctx;
0645     u64 parent_ino;
0646     int fsid;
0647     int xinobits;
0648     bool xinowarn;
0649 };
0650
0651 static int ovl_fill_real(struct dir_context *ctx, const char *name,
0652                int namelen, loff_t offset, u64 ino,
0653                unsigned int d_type)
0654 {
0655     struct ovl_readdir_translate *rdt =
0656         container_of(ctx, struct ovl_readdir_translate, ctx);
0657     struct dir_context *orig_ctx = rdt->orig_ctx;
0658
0659     if (rdt->parent_ino && strcmp(name, "..") == 0) {
0660         ino = rdt->parent_ino;
0661     } else if (rdt->cache) {
0662         struct ovl_cache_entry *p;
0663
0664         p = ovl_cache_entry_find(&rdt->cache->root, name, namelen);
0665         if (p)
0666             ino = p->ino;
0667     } else if (rdt->xinobits) {
0668         ino = ovl_remap_lower_ino(ino, rdt->xinobits, rdt->fsid,
0669                       name, namelen, rdt->xinowarn);
0670     }
0671
0672     return orig_ctx->actor(orig_ctx, name, namelen, offset, ino, d_type);
0673 }
0674
0675 static bool ovl_is_impure_dir(struct file *file)
0676 {
0677     struct ovl_dir_file *od = file->private_data;
0678     struct inode *dir = d_inode(file->f_path.dentry);
0679
0680     /*
0681      * Only upper dir can be impure, but if we are in the middle of
0682      * iterating a lower real dir, dir could be copied up and marked
0683      * impure. We only want the impure cache if we started iterating
0684      * a real upper dir to begin with.
0685      */
0686     return od->is_upper && ovl_test_flag(OVL_IMPURE, dir);
0687
0688 }
0689
0690 static int ovl_iterate_real(struct file *file, struct dir_context *ctx)
0691 {
0692     int err;
0693     struct ovl_dir_file *od = file->private_data;
0694     struct dentry *dir = file->f_path.dentry;
0695     const struct ovl_layer *lower_layer = ovl_layer_lower(dir);
0696     struct ovl_readdir_translate rdt = {
0697         .ctx.actor = ovl_fill_real,
0698         .orig_ctx = ctx,
0699         .xinobits = ovl_xino_bits(dir->d_sb),
0700         .xinowarn = ovl_xino_warn(dir->d_sb),
0701     };
0702
0703     if (rdt.xinobits && lower_layer)
0704         rdt.fsid = lower_layer->fsid;
0705
0706     if (OVL_TYPE_MERGE(ovl_path_type(dir->d_parent))) {
0707         struct kstat stat;
0708         struct path statpath = file->f_path;
0709
0710         statpath.dentry = dir->d_parent;
0711         err = vfs_getattr(&statpath, &stat, STATX_INO, 0);
0712         if (err)
0713             return err;
0714
0715         WARN_ON_ONCE(dir->d_sb->s_dev != stat.dev);
0716         rdt.parent_ino = stat.ino;
0717     }
0718
0719     if (ovl_is_impure_dir(file)) {
0720         rdt.cache = ovl_cache_get_impure(&file->f_path);
0721         if (IS_ERR(rdt.cache))
0722             return PTR_ERR(rdt.cache);
0723     }
0724
0725     err = iterate_dir(od->realfile, &rdt.ctx);
0726     ctx->pos = rdt.ctx.pos;
0727
0728     return err;
0729 }
0730
0731
0732 static int ovl_iterate(struct file *file, struct dir_context *ctx)
0733 {
0734     struct ovl_dir_file *od = file->private_data;
0735     struct dentry *dentry = file->f_path.dentry;
0736     struct ovl_cache_entry *p;
0737     const struct cred *old_cred;
0738     int err;
0739
0740     old_cred = ovl_override_creds(dentry->d_sb);
0741     if (!ctx->pos)
0742         ovl_dir_reset(file);
0743
0744     if (od->is_real) {
0745         /*
0746          * If parent is merge, then need to adjust d_ino for '..', if
0747          * dir is impure then need to adjust d_ino for copied up
0748          * entries.
0749          */
0750         if (ovl_xino_bits(dentry->d_sb) ||
0751             (ovl_same_fs(dentry->d_sb) &&
0752              (ovl_is_impure_dir(file) ||
0753               OVL_TYPE_MERGE(ovl_path_type(dentry->d_parent))))) {
0754             err = ovl_iterate_real(file, ctx);
0755         } else {
0756             err = iterate_dir(od->realfile, ctx);
0757         }
0758         goto out;
0759     }
0760
0761     if (!od->cache) {
0762         struct ovl_dir_cache *cache;
0763
0764         cache = ovl_cache_get(dentry);
0765         err = PTR_ERR(cache);
0766         if (IS_ERR(cache))
0767             goto out;
0768
0769         od->cache = cache;
0770         ovl_seek_cursor(od, ctx->pos);
0771     }
0772
0773     while (od->cursor != &od->cache->entries) {
0774         p = list_entry(od->cursor, struct ovl_cache_entry, l_node);
0775         if (!p->is_whiteout) {
0776             if (!p->ino) {
0777                 err = ovl_cache_update_ino(&file->f_path, p);
0778                 if (err)
0779                     goto out;
0780             }
0781         }
0782         /* ovl_cache_update_ino() sets is_whiteout on stale entry */
0783         if (!p->is_whiteout) {
0784             if (!dir_emit(ctx, p->name, p->len, p->ino, p->type))
0785                 break;
0786         }
0787         od->cursor = p->l_node.next;
0788         ctx->pos++;
0789     }
0790     err = 0;
0791 out:
0792     revert_creds(old_cred);
0793     return err;
0794 }
0795
0796 static loff_t ovl_dir_llseek(struct file *file, loff_t offset, int origin)
0797 {
0798     loff_t res;
0799     struct ovl_dir_file *od = file->private_data;
0800
0801     inode_lock(file_inode(file));
0802     if (!file->f_pos)
0803         ovl_dir_reset(file);
0804
0805     if (od->is_real) {
0806         res = vfs_llseek(od->realfile, offset, origin);
0807         file->f_pos = od->realfile->f_pos;
0808     } else {
0809         res = -EINVAL;
0810
0811         switch (origin) {
0812         case SEEK_CUR:
0813             offset += file->f_pos;
0814             break;
0815         case SEEK_SET:
0816             break;
0817         default:
0818             goto out_unlock;
0819         }
0820         if (offset < 0)
0821             goto out_unlock;
0822
0823         if (offset != file->f_pos) {
0824             file->f_pos = offset;
0825             if (od->cache)
0826                 ovl_seek_cursor(od, offset);
0827         }
0828         res = offset;
0829     }
0830 out_unlock:
0831     inode_unlock(file_inode(file));
0832
0833     return res;
0834 }
0835
0836 static struct file *ovl_dir_open_realfile(const struct file *file,
0837                       struct path *realpath)
0838 {
0839     struct file *res;
0840     const struct cred *old_cred;
0841
0842     old_cred = ovl_override_creds(file_inode(file)->i_sb);
0843     res = ovl_path_open(realpath, O_RDONLY | (file->f_flags & O_LARGEFILE));
0844     revert_creds(old_cred);
0845
0846     return res;
0847 }
0848
0849 /*
0850  * Like ovl_real_fdget(), returns upperfile if dir was copied up since open.
0851  * Unlike ovl_real_fdget(), this caches upperfile in file->private_data.
0852  *
0853  * TODO: use same abstract type for file->private_data of dir and file so
0854  * upperfile could also be cached for files as well.
0855  */
0856 struct file *ovl_dir_real_file(const struct file *file, bool want_upper)
0857 {
0858
0859     struct ovl_dir_file *od = file->private_data;
0860     struct dentry *dentry = file->f_path.dentry;
0861     struct file *old, *realfile = od->realfile;
0862
0863     if (!OVL_TYPE_UPPER(ovl_path_type(dentry)))
0864         return want_upper ? NULL : realfile;
0865
0866     /*
0867      * Need to check if we started out being a lower dir, but got copied up
0868      */
0869     if (!od->is_upper) {
0870         realfile = READ_ONCE(od->upperfile);
0871         if (!realfile) {
0872             struct path upperpath;
0873
0874             ovl_path_upper(dentry, &upperpath);
0875             realfile = ovl_dir_open_realfile(file, &upperpath);
0876             if (IS_ERR(realfile))
0877                 return realfile;
0878
0879             old = cmpxchg_release(&od->upperfile, NULL, realfile);
0880             if (old) {
0881                 fput(realfile);
0882                 realfile = old;
0883             }
0884         }
0885     }
0886
0887     return realfile;
0888 }
0889
0890 static int ovl_dir_fsync(struct file *file, loff_t start, loff_t end,
0891              int datasync)
0892 {
0893     struct file *realfile;
0894     int err;
0895
0896     err = ovl_sync_status(OVL_FS(file->f_path.dentry->d_sb));
0897     if (err <= 0)
0898         return err;
0899
0900     realfile = ovl_dir_real_file(file, true);
0901     err = PTR_ERR_OR_ZERO(realfile);
0902
0903     /* Nothing to sync for lower */
0904     if (!realfile || err)
0905         return err;
0906
0907     return vfs_fsync_range(realfile, start, end, datasync);
0908 }
0909
0910 static int ovl_dir_release(struct inode *inode, struct file *file)
0911 {
0912     struct ovl_dir_file *od = file->private_data;
0913
0914     if (od->cache) {
0915         inode_lock(inode);
0916         ovl_cache_put(od, file->f_path.dentry);
0917         inode_unlock(inode);
0918     }
0919     fput(od->realfile);
0920     if (od->upperfile)
0921         fput(od->upperfile);
0922     kfree(od);
0923
0924     return 0;
0925 }
0926
0927 static int ovl_dir_open(struct inode *inode, struct file *file)
0928 {
0929     struct path realpath;
0930     struct file *realfile;
0931     struct ovl_dir_file *od;
0932     enum ovl_path_type type;
0933
0934     od = kzalloc(sizeof(struct ovl_dir_file), GFP_KERNEL);
0935     if (!od)
0936         return -ENOMEM;
0937
0938     type = ovl_path_real(file->f_path.dentry, &realpath);
0939     realfile = ovl_dir_open_realfile(file, &realpath);
0940     if (IS_ERR(realfile)) {
0941         kfree(od);
0942         return PTR_ERR(realfile);
0943     }
0944     od->realfile = realfile;
0945     od->is_real = ovl_dir_is_real(file->f_path.dentry);
0946     od->is_upper = OVL_TYPE_UPPER(type);
0947     file->private_data = od;
0948
0949     return 0;
0950 }
0951
0952 const struct file_operations ovl_dir_operations = {
0953     .read       = generic_read_dir,
0954     .open       = ovl_dir_open,
0955     .iterate    = ovl_iterate,
0956     .llseek     = ovl_dir_llseek,
0957     .fsync      = ovl_dir_fsync,
0958     .release    = ovl_dir_release,
0959 };
0960
0961 int ovl_check_empty_dir(struct dentry *dentry, struct list_head *list)
0962 {
0963     int err;
0964     struct ovl_cache_entry *p, *n;
0965     struct rb_root root = RB_ROOT;
0966     const struct cred *old_cred;
0967
0968     old_cred = ovl_override_creds(dentry->d_sb);
0969     err = ovl_dir_read_merged(dentry, list, &root);
0970     revert_creds(old_cred);
0971     if (err)
0972         return err;
0973
0974     err = 0;
0975
0976     list_for_each_entry_safe(p, n, list, l_node) {
0977         /*
0978          * Select whiteouts in upperdir, they should
0979          * be cleared when deleting this directory.
0980          */
0981         if (p->is_whiteout) {
0982             if (p->is_upper)
0983                 continue;
0984             goto del_entry;
0985         }
0986
0987         if (p->name[0] == '.') {
0988             if (p->len == 1)
0989                 goto del_entry;
0990             if (p->len == 2 && p->name[1] == '.')
0991                 goto del_entry;
0992         }
0993         err = -ENOTEMPTY;
0994         break;
0995
0996 del_entry:
0997         list_del(&p->l_node);
0998         kfree(p);
0999     }
1000
1001     return err;
1002 }
1003
1004 void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper,
1005                struct list_head *list)
1006 {
1007     struct ovl_cache_entry *p;
1008
1009     inode_lock_nested(upper->d_inode, I_MUTEX_CHILD);
1010     list_for_each_entry(p, list, l_node) {
1011         struct dentry *dentry;
1012
1013         if (WARN_ON(!p->is_whiteout || !p->is_upper))
1014             continue;
1015
1016         dentry = ovl_lookup_upper(ofs, p->name, upper, p->len);
1017         if (IS_ERR(dentry)) {
1018             pr_err("lookup '%s/%.*s' failed (%i)\n",
1019                    upper->d_name.name, p->len, p->name,
1020                    (int) PTR_ERR(dentry));
1021             continue;
1022         }
1023         if (dentry->d_inode)
1024             ovl_cleanup(ofs, upper->d_inode, dentry);
1025         dput(dentry);
1026     }
1027     inode_unlock(upper->d_inode);
1028 }
1029
1030 static int ovl_check_d_type(struct dir_context *ctx, const char *name,
1031               int namelen, loff_t offset, u64 ino,
1032               unsigned int d_type)
1033 {
1034     struct ovl_readdir_data *rdd =
1035         container_of(ctx, struct ovl_readdir_data, ctx);
1036
1037     /* Even if d_type is not supported, DT_DIR is returned for . and .. */
1038     if (!strncmp(name, ".", namelen) || !strncmp(name, "..", namelen))
1039         return 0;
1040
1041     if (d_type != DT_UNKNOWN)
1042         rdd->d_type_supported = true;
1043
1044     return 0;
1045 }
1046
1047 /*
1048  * Returns 1 if d_type is supported, 0 not supported/unknown. Negative values
1049  * if error is encountered.
1050  */
1051 int ovl_check_d_type_supported(struct path *realpath)
1052 {
1053     int err;
1054     struct ovl_readdir_data rdd = {
1055         .ctx.actor = ovl_check_d_type,
1056         .d_type_supported = false,
1057     };
1058
1059     err = ovl_dir_read(realpath, &rdd);
1060     if (err)
1061         return err;
1062
1063     return rdd.d_type_supported;
1064 }
1065
1066 #define OVL_INCOMPATDIR_NAME "incompat"
1067
1068 static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, struct path *path,
1069                        int level)
1070 {
1071     int err;
1072     struct inode *dir = path->dentry->d_inode;
1073     LIST_HEAD(list);
1074     struct rb_root root = RB_ROOT;
1075     struct ovl_cache_entry *p;
1076     struct ovl_readdir_data rdd = {
1077         .ctx.actor = ovl_fill_merge,
1078         .dentry = NULL,
1079         .list = &list,
1080         .root = &root,
1081         .is_lowest = false,
1082     };
1083     bool incompat = false;
1084
1085     /*
1086      * The "work/incompat" directory is treated specially - if it is not
1087      * empty, instead of printing a generic error and mounting read-only,
1088      * we will error about incompat features and fail the mount.
1089      *
1090      * When called from ovl_indexdir_cleanup(), path->dentry->d_name.name
1091      * starts with '#'.
1092      */
1093     if (level == 2 &&
1094         !strcmp(path->dentry->d_name.name, OVL_INCOMPATDIR_NAME))
1095         incompat = true;
1096
1097     err = ovl_dir_read(path, &rdd);
1098     if (err)
1099         goto out;
1100
1101     inode_lock_nested(dir, I_MUTEX_PARENT);
1102     list_for_each_entry(p, &list, l_node) {
1103         struct dentry *dentry;
1104
1105         if (p->name[0] == '.') {
1106             if (p->len == 1)
1107                 continue;
1108             if (p->len == 2 && p->name[1] == '.')
1109                 continue;
1110         } else if (incompat) {
1111             pr_err("overlay with incompat feature '%s' cannot be mounted\n",
1112                 p->name);
1113             err = -EINVAL;
1114             break;
1115         }
1116         dentry = ovl_lookup_upper(ofs, p->name, path->dentry, p->len);
1117         if (IS_ERR(dentry))
1118             continue;
1119         if (dentry->d_inode)
1120             err = ovl_workdir_cleanup(ofs, dir, path->mnt, dentry, level);
1121         dput(dentry);
1122         if (err)
1123             break;
1124     }
1125     inode_unlock(dir);
1126 out:
1127     ovl_cache_free(&list);
1128     return err;
1129 }
1130
1131 int ovl_workdir_cleanup(struct ovl_fs *ofs, struct inode *dir,
1132             struct vfsmount *mnt, struct dentry *dentry, int level)
1133 {
1134     int err;
1135
1136     if (!d_is_dir(dentry) || level > 1) {
1137         return ovl_cleanup(ofs, dir, dentry);
1138     }
1139
1140     err = ovl_do_rmdir(ofs, dir, dentry);
1141     if (err) {
1142         struct path path = { .mnt = mnt, .dentry = dentry };
1143
1144         inode_unlock(dir);
1145         err = ovl_workdir_cleanup_recurse(ofs, &path, level + 1);
1146         inode_lock_nested(dir, I_MUTEX_PARENT);
1147         if (!err)
1148             err = ovl_cleanup(ofs, dir, dentry);
1149     }
1150
1151     return err;
1152 }
1153
1154 int ovl_indexdir_cleanup(struct ovl_fs *ofs)
1155 {
1156     int err;
1157     struct dentry *indexdir = ofs->indexdir;
1158     struct dentry *index = NULL;
1159     struct inode *dir = indexdir->d_inode;
1160     struct path path = { .mnt = ovl_upper_mnt(ofs), .dentry = indexdir };
1161     LIST_HEAD(list);
1162     struct rb_root root = RB_ROOT;
1163     struct ovl_cache_entry *p;
1164     struct ovl_readdir_data rdd = {
1165         .ctx.actor = ovl_fill_merge,
1166         .dentry = NULL,
1167         .list = &list,
1168         .root = &root,
1169         .is_lowest = false,
1170     };
1171
1172     err = ovl_dir_read(&path, &rdd);
1173     if (err)
1174         goto out;
1175
1176     inode_lock_nested(dir, I_MUTEX_PARENT);
1177     list_for_each_entry(p, &list, l_node) {
1178         if (p->name[0] == '.') {
1179             if (p->len == 1)
1180                 continue;
1181             if (p->len == 2 && p->name[1] == '.')
1182                 continue;
1183         }
1184         index = ovl_lookup_upper(ofs, p->name, indexdir, p->len);
1185         if (IS_ERR(index)) {
1186             err = PTR_ERR(index);
1187             index = NULL;
1188             break;
1189         }
1190         /* Cleanup leftover from index create/cleanup attempt */
1191         if (index->d_name.name[0] == '#') {
1192             err = ovl_workdir_cleanup(ofs, dir, path.mnt, index, 1);
1193             if (err)
1194                 break;
1195             goto next;
1196         }
1197         err = ovl_verify_index(ofs, index);
1198         if (!err) {
1199             goto next;
1200         } else if (err == -ESTALE) {
1201             /* Cleanup stale index entries */
1202             err = ovl_cleanup(ofs, dir, index);
1203         } else if (err != -ENOENT) {
1204             /*
1205              * Abort mount to avoid corrupting the index if
1206              * an incompatible index entry was found or on out
1207              * of memory.
1208              */
1209             break;
1210         } else if (ofs->config.nfs_export) {
1211             /*
1212              * Whiteout orphan index to block future open by
1213              * handle after overlay nlink dropped to zero.
1214              */
1215             err = ovl_cleanup_and_whiteout(ofs, dir, index);
1216         } else {
1217             /* Cleanup orphan index entries */
1218             err = ovl_cleanup(ofs, dir, index);
1219         }
1220
1221         if (err)
1222             break;
1223
1224 next:
1225         dput(index);
1226         index = NULL;
1227     }
1228     dput(index);
1229     inode_unlock(dir);
1230 out:
1231     ovl_cache_free(&list);
1232     if (err)
1233         pr_err("failed index dir cleanup (%i)\n", err);
1234     return err;
1235 }