0001
0002
0003
0004
0005
0006
0007 #ifndef __EROFS_INTERNAL_H
0008 #define __EROFS_INTERNAL_H
0009
0010 #include <linux/fs.h>
0011 #include <linux/dcache.h>
0012 #include <linux/mm.h>
0013 #include <linux/pagemap.h>
0014 #include <linux/bio.h>
0015 #include <linux/buffer_head.h>
0016 #include <linux/magic.h>
0017 #include <linux/slab.h>
0018 #include <linux/vmalloc.h>
0019 #include <linux/iomap.h>
0020 #include "erofs_fs.h"
0021
0022
0023 #undef pr_fmt
0024 #define pr_fmt(fmt) "erofs: " fmt
0025
0026 __printf(3, 4) void _erofs_err(struct super_block *sb,
0027 const char *function, const char *fmt, ...);
0028 #define erofs_err(sb, fmt, ...) \
0029 _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__)
0030 __printf(3, 4) void _erofs_info(struct super_block *sb,
0031 const char *function, const char *fmt, ...);
0032 #define erofs_info(sb, fmt, ...) \
0033 _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__)
0034 #ifdef CONFIG_EROFS_FS_DEBUG
0035 #define erofs_dbg(x, ...) pr_debug(x "\n", ##__VA_ARGS__)
0036 #define DBG_BUGON BUG_ON
0037 #else
0038 #define erofs_dbg(x, ...) ((void)0)
0039 #define DBG_BUGON(x) ((void)(x))
0040 #endif
0041
0042
0043 #define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1
0044
0045 typedef u64 erofs_nid_t;
0046 typedef u64 erofs_off_t;
0047
0048 typedef u32 erofs_blk_t;
0049
0050 struct erofs_device_info {
0051 char *path;
0052 struct erofs_fscache *fscache;
0053 struct block_device *bdev;
0054 struct dax_device *dax_dev;
0055 u64 dax_part_off;
0056
0057 u32 blocks;
0058 u32 mapped_blkaddr;
0059 };
0060
0061 enum {
0062 EROFS_SYNC_DECOMPRESS_AUTO,
0063 EROFS_SYNC_DECOMPRESS_FORCE_ON,
0064 EROFS_SYNC_DECOMPRESS_FORCE_OFF
0065 };
0066
0067 struct erofs_mount_opts {
0068 #ifdef CONFIG_EROFS_FS_ZIP
0069
0070 unsigned char cache_strategy;
0071
0072 unsigned int sync_decompress;
0073
0074
0075 unsigned int max_sync_decompress_pages;
0076 #endif
0077 unsigned int mount_opt;
0078 char *fsid;
0079 };
0080
0081 struct erofs_dev_context {
0082 struct idr tree;
0083 struct rw_semaphore rwsem;
0084
0085 unsigned int extra_devices;
0086 };
0087
0088 struct erofs_fs_context {
0089 struct erofs_mount_opts opt;
0090 struct erofs_dev_context *devs;
0091 };
0092
0093
0094 struct erofs_sb_lz4_info {
0095
0096 u16 max_distance_pages;
0097
0098 u16 max_pclusterblks;
0099 };
0100
0101 struct erofs_fscache {
0102 struct fscache_cookie *cookie;
0103 struct inode *inode;
0104 };
0105
0106 struct erofs_sb_info {
0107 struct erofs_mount_opts opt;
0108 #ifdef CONFIG_EROFS_FS_ZIP
0109
0110 struct list_head list;
0111 struct mutex umount_mutex;
0112
0113
0114 struct xarray managed_pslots;
0115
0116 unsigned int shrinker_run_no;
0117 u16 available_compr_algs;
0118
0119
0120 struct inode *managed_cache;
0121
0122 struct erofs_sb_lz4_info lz4;
0123 #endif
0124 struct erofs_dev_context *devs;
0125 struct dax_device *dax_dev;
0126 u64 dax_part_off;
0127 u64 total_blocks;
0128 u32 primarydevice_blocks;
0129
0130 u32 meta_blkaddr;
0131 #ifdef CONFIG_EROFS_FS_XATTR
0132 u32 xattr_blkaddr;
0133 #endif
0134 u16 device_id_mask;
0135
0136
0137 unsigned char islotbits;
0138
0139 u32 sb_size;
0140 u32 build_time_nsec;
0141 u64 build_time;
0142
0143
0144 erofs_nid_t root_nid;
0145
0146 u64 inos;
0147
0148 u8 uuid[16];
0149 u8 volume_name[16];
0150 u32 feature_compat;
0151 u32 feature_incompat;
0152
0153
0154 struct kobject s_kobj;
0155 struct completion s_kobj_unregister;
0156
0157
0158 struct fscache_volume *volume;
0159 struct erofs_fscache *s_fscache;
0160 };
0161
0162 #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
0163 #define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)
0164
0165
0166 #define EROFS_MOUNT_XATTR_USER 0x00000010
0167 #define EROFS_MOUNT_POSIX_ACL 0x00000020
0168 #define EROFS_MOUNT_DAX_ALWAYS 0x00000040
0169 #define EROFS_MOUNT_DAX_NEVER 0x00000080
0170
0171 #define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
0172 #define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option)
0173 #define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option)
0174
0175 static inline bool erofs_is_fscache_mode(struct super_block *sb)
0176 {
0177 return IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && !sb->s_bdev;
0178 }
0179
0180 enum {
0181 EROFS_ZIP_CACHE_DISABLED,
0182 EROFS_ZIP_CACHE_READAHEAD,
0183 EROFS_ZIP_CACHE_READAROUND
0184 };
0185
0186 #ifdef CONFIG_EROFS_FS_ZIP
0187 #define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
0188
0189
0190 struct erofs_workgroup {
0191
0192 pgoff_t index;
0193
0194
0195 atomic_t refcount;
0196 };
0197
0198 static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
0199 int val)
0200 {
0201 preempt_disable();
0202 if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) {
0203 preempt_enable();
0204 return false;
0205 }
0206 return true;
0207 }
0208
0209 static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
0210 int orig_val)
0211 {
0212
0213
0214
0215
0216 smp_mb();
0217 atomic_set(&grp->refcount, orig_val);
0218 preempt_enable();
0219 }
0220
0221 static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
0222 {
0223 return atomic_cond_read_relaxed(&grp->refcount,
0224 VAL != EROFS_LOCKED_MAGIC);
0225 }
0226 #endif
0227
0228
0229 #define LOG_BLOCK_SIZE PAGE_SHIFT
0230
0231 #undef LOG_SECTORS_PER_BLOCK
0232 #define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9)
0233
0234 #undef SECTORS_PER_BLOCK
0235 #define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK)
0236
0237 #define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE)
0238
0239 #if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ)
0240 #error erofs cannot be used in this platform
0241 #endif
0242
0243 enum erofs_kmap_type {
0244 EROFS_NO_KMAP,
0245 EROFS_KMAP,
0246 EROFS_KMAP_ATOMIC,
0247 };
0248
0249 struct erofs_buf {
0250 struct page *page;
0251 void *base;
0252 enum erofs_kmap_type kmap_type;
0253 };
0254 #define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL })
0255
0256 #define ROOT_NID(sb) ((sb)->root_nid)
0257
0258 #define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ)
0259 #define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ)
0260 #define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ)
0261
0262 static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
0263 {
0264 return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
0265 }
0266
0267 #define EROFS_FEATURE_FUNCS(name, compat, feature) \
0268 static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
0269 { \
0270 return sbi->feature_##compat & EROFS_FEATURE_##feature; \
0271 }
0272
0273 EROFS_FEATURE_FUNCS(zero_padding, incompat, INCOMPAT_ZERO_PADDING)
0274 EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
0275 EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
0276 EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
0277 EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
0278 EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
0279 EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
0280 EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
0281
0282
0283 #define EROFS_I_EA_INITED_BIT 0
0284 #define EROFS_I_Z_INITED_BIT 1
0285
0286
0287 #define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)
0288 #define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)
0289
0290 struct erofs_inode {
0291 erofs_nid_t nid;
0292
0293
0294 unsigned long flags;
0295
0296 unsigned char datalayout;
0297 unsigned char inode_isize;
0298 unsigned short xattr_isize;
0299
0300 unsigned int xattr_shared_count;
0301 unsigned int *xattr_shared_xattrs;
0302
0303 union {
0304 erofs_blk_t raw_blkaddr;
0305 struct {
0306 unsigned short chunkformat;
0307 unsigned char chunkbits;
0308 };
0309 #ifdef CONFIG_EROFS_FS_ZIP
0310 struct {
0311 unsigned short z_advise;
0312 unsigned char z_algorithmtype[2];
0313 unsigned char z_logical_clusterbits;
0314 unsigned long z_tailextent_headlcn;
0315 erofs_off_t z_idataoff;
0316 unsigned short z_idata_size;
0317 };
0318 #endif
0319 };
0320
0321 struct inode vfs_inode;
0322 };
0323
0324 #define EROFS_I(ptr) \
0325 container_of(ptr, struct erofs_inode, vfs_inode)
0326
0327 static inline unsigned long erofs_inode_datablocks(struct inode *inode)
0328 {
0329
0330 return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
0331 }
0332
0333 static inline unsigned int erofs_bitrange(unsigned int value, unsigned int bit,
0334 unsigned int bits)
0335 {
0336
0337 return (value >> bit) & ((1 << bits) - 1);
0338 }
0339
0340
0341 static inline unsigned int erofs_inode_version(unsigned int value)
0342 {
0343 return erofs_bitrange(value, EROFS_I_VERSION_BIT,
0344 EROFS_I_VERSION_BITS);
0345 }
0346
0347 static inline unsigned int erofs_inode_datalayout(unsigned int value)
0348 {
0349 return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT,
0350 EROFS_I_DATALAYOUT_BITS);
0351 }
0352
0353
0354
0355
0356
0357 static inline
0358 struct page *erofs_grab_cache_page_nowait(struct address_space *mapping,
0359 pgoff_t index)
0360 {
0361 return pagecache_get_page(mapping, index,
0362 FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
0363 readahead_gfp_mask(mapping) & ~__GFP_RECLAIM);
0364 }
0365
0366 extern const struct super_operations erofs_sops;
0367
0368 extern const struct address_space_operations erofs_raw_access_aops;
0369 extern const struct address_space_operations z_erofs_aops;
0370
0371 enum {
0372 BH_Encoded = BH_PrivateStart,
0373 BH_FullMapped,
0374 };
0375
0376
0377 #define EROFS_MAP_MAPPED (1 << BH_Mapped)
0378
0379 #define EROFS_MAP_META (1 << BH_Meta)
0380
0381 #define EROFS_MAP_ENCODED (1 << BH_Encoded)
0382
0383 #define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
0384
0385 struct erofs_map_blocks {
0386 struct erofs_buf buf;
0387
0388 erofs_off_t m_pa, m_la;
0389 u64 m_plen, m_llen;
0390
0391 unsigned short m_deviceid;
0392 char m_algorithmformat;
0393 unsigned int m_flags;
0394 };
0395
0396
0397 #define EROFS_GET_BLOCKS_RAW 0x0001
0398
0399
0400
0401
0402 #define EROFS_GET_BLOCKS_FIEMAP 0x0002
0403
0404 #define EROFS_GET_BLOCKS_READMORE 0x0004
0405
0406 #define EROFS_GET_BLOCKS_FINDTAIL 0x0008
0407
0408 enum {
0409 Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
0410 Z_EROFS_COMPRESSION_RUNTIME_MAX
0411 };
0412
0413
0414 extern const struct iomap_ops z_erofs_iomap_report_ops;
0415
0416 #ifdef CONFIG_EROFS_FS_ZIP
0417 int z_erofs_fill_inode(struct inode *inode);
0418 int z_erofs_map_blocks_iter(struct inode *inode,
0419 struct erofs_map_blocks *map,
0420 int flags);
0421 #else
0422 static inline int z_erofs_fill_inode(struct inode *inode) { return -EOPNOTSUPP; }
0423 static inline int z_erofs_map_blocks_iter(struct inode *inode,
0424 struct erofs_map_blocks *map,
0425 int flags)
0426 {
0427 return -EOPNOTSUPP;
0428 }
0429 #endif
0430
0431 struct erofs_map_dev {
0432 struct erofs_fscache *m_fscache;
0433 struct block_device *m_bdev;
0434 struct dax_device *m_daxdev;
0435 u64 m_dax_part_off;
0436
0437 erofs_off_t m_pa;
0438 unsigned int m_deviceid;
0439 };
0440
0441
0442 extern const struct file_operations erofs_file_fops;
0443 void erofs_unmap_metabuf(struct erofs_buf *buf);
0444 void erofs_put_metabuf(struct erofs_buf *buf);
0445 void *erofs_bread(struct erofs_buf *buf, struct inode *inode,
0446 erofs_blk_t blkaddr, enum erofs_kmap_type type);
0447 void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,
0448 erofs_blk_t blkaddr, enum erofs_kmap_type type);
0449 int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev);
0450 int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
0451 u64 start, u64 len);
0452 int erofs_map_blocks(struct inode *inode,
0453 struct erofs_map_blocks *map, int flags);
0454
0455
0456 static inline unsigned long erofs_inode_hash(erofs_nid_t nid)
0457 {
0458 #if BITS_PER_LONG == 32
0459 return (nid >> 32) ^ (nid & 0xffffffff);
0460 #else
0461 return nid;
0462 #endif
0463 }
0464
0465 extern const struct inode_operations erofs_generic_iops;
0466 extern const struct inode_operations erofs_symlink_iops;
0467 extern const struct inode_operations erofs_fast_symlink_iops;
0468
0469 struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir);
0470 int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path,
0471 struct kstat *stat, u32 request_mask,
0472 unsigned int query_flags);
0473
0474
0475 extern const struct inode_operations erofs_dir_iops;
0476
0477 int erofs_namei(struct inode *dir, const struct qstr *name,
0478 erofs_nid_t *nid, unsigned int *d_type);
0479
0480
0481 extern const struct file_operations erofs_dir_fops;
0482
0483 static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
0484 {
0485 int retried = 0;
0486
0487 while (1) {
0488 void *p = vm_map_ram(pages, count, -1);
0489
0490
0491 if (p || ++retried >= 3)
0492 return p;
0493 vm_unmap_aliases();
0494 }
0495 return NULL;
0496 }
0497
0498
0499 void *erofs_get_pcpubuf(unsigned int requiredpages);
0500 void erofs_put_pcpubuf(void *ptr);
0501 int erofs_pcpubuf_growsize(unsigned int nrpages);
0502 void erofs_pcpubuf_init(void);
0503 void erofs_pcpubuf_exit(void);
0504
0505
0506 int erofs_register_sysfs(struct super_block *sb);
0507 void erofs_unregister_sysfs(struct super_block *sb);
0508 int __init erofs_init_sysfs(void);
0509 void erofs_exit_sysfs(void);
0510
0511
0512 struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp);
0513 static inline void erofs_pagepool_add(struct page **pagepool,
0514 struct page *page)
0515 {
0516 set_page_private(page, (unsigned long)*pagepool);
0517 *pagepool = page;
0518 }
0519 void erofs_release_pages(struct page **pagepool);
0520
0521 #ifdef CONFIG_EROFS_FS_ZIP
0522 int erofs_workgroup_put(struct erofs_workgroup *grp);
0523 struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
0524 pgoff_t index);
0525 struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
0526 struct erofs_workgroup *grp);
0527 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);
0528 void erofs_shrinker_register(struct super_block *sb);
0529 void erofs_shrinker_unregister(struct super_block *sb);
0530 int __init erofs_init_shrinker(void);
0531 void erofs_exit_shrinker(void);
0532 int __init z_erofs_init_zip_subsystem(void);
0533 void z_erofs_exit_zip_subsystem(void);
0534 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
0535 struct erofs_workgroup *egrp);
0536 int erofs_try_to_free_cached_page(struct page *page);
0537 int z_erofs_load_lz4_config(struct super_block *sb,
0538 struct erofs_super_block *dsb,
0539 struct z_erofs_lz4_cfgs *lz4, int len);
0540 #else
0541 static inline void erofs_shrinker_register(struct super_block *sb) {}
0542 static inline void erofs_shrinker_unregister(struct super_block *sb) {}
0543 static inline int erofs_init_shrinker(void) { return 0; }
0544 static inline void erofs_exit_shrinker(void) {}
0545 static inline int z_erofs_init_zip_subsystem(void) { return 0; }
0546 static inline void z_erofs_exit_zip_subsystem(void) {}
0547 static inline int z_erofs_load_lz4_config(struct super_block *sb,
0548 struct erofs_super_block *dsb,
0549 struct z_erofs_lz4_cfgs *lz4, int len)
0550 {
0551 if (lz4 || dsb->u1.lz4_max_distance) {
0552 erofs_err(sb, "lz4 algorithm isn't enabled");
0553 return -EINVAL;
0554 }
0555 return 0;
0556 }
0557 #endif
0558
0559 #ifdef CONFIG_EROFS_FS_ZIP_LZMA
0560 int z_erofs_lzma_init(void);
0561 void z_erofs_lzma_exit(void);
0562 int z_erofs_load_lzma_config(struct super_block *sb,
0563 struct erofs_super_block *dsb,
0564 struct z_erofs_lzma_cfgs *lzma, int size);
0565 #else
0566 static inline int z_erofs_lzma_init(void) { return 0; }
0567 static inline int z_erofs_lzma_exit(void) { return 0; }
0568 static inline int z_erofs_load_lzma_config(struct super_block *sb,
0569 struct erofs_super_block *dsb,
0570 struct z_erofs_lzma_cfgs *lzma, int size) {
0571 if (lzma) {
0572 erofs_err(sb, "lzma algorithm isn't enabled");
0573 return -EINVAL;
0574 }
0575 return 0;
0576 }
0577 #endif
0578
0579
0580 #ifdef CONFIG_EROFS_FS_ONDEMAND
0581 int erofs_fscache_register_fs(struct super_block *sb);
0582 void erofs_fscache_unregister_fs(struct super_block *sb);
0583
0584 int erofs_fscache_register_cookie(struct super_block *sb,
0585 struct erofs_fscache **fscache,
0586 char *name, bool need_inode);
0587 void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache);
0588
0589 extern const struct address_space_operations erofs_fscache_access_aops;
0590 #else
0591 static inline int erofs_fscache_register_fs(struct super_block *sb)
0592 {
0593 return 0;
0594 }
0595 static inline void erofs_fscache_unregister_fs(struct super_block *sb) {}
0596
0597 static inline int erofs_fscache_register_cookie(struct super_block *sb,
0598 struct erofs_fscache **fscache,
0599 char *name, bool need_inode)
0600 {
0601 return -EOPNOTSUPP;
0602 }
0603
0604 static inline void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
0605 {
0606 }
0607 #endif
0608
0609 #define EFSCORRUPTED EUCLEAN
0610
0611 #endif