include/linux/buffer_head.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * include/linux/buffer_head.h
0004  *
0005  * Everything to do with buffer_heads.
0006  */
0007
0008 #ifndef _LINUX_BUFFER_HEAD_H
0009 #define _LINUX_BUFFER_HEAD_H
0010
0011 #include <linux/types.h>
0012 #include <linux/blk_types.h>
0013 #include <linux/fs.h>
0014 #include <linux/linkage.h>
0015 #include <linux/pagemap.h>
0016 #include <linux/wait.h>
0017 #include <linux/atomic.h>
0018
0019 #ifdef CONFIG_BLOCK
0020
0021 enum bh_state_bits {
0022     BH_Uptodate,    /* Contains valid data */
0023     BH_Dirty,   /* Is dirty */
0024     BH_Lock,    /* Is locked */
0025     BH_Req,     /* Has been submitted for I/O */
0026
0027     BH_Mapped,  /* Has a disk mapping */
0028     BH_New,     /* Disk mapping was newly created by get_block */
0029     BH_Async_Read,  /* Is under end_buffer_async_read I/O */
0030     BH_Async_Write, /* Is under end_buffer_async_write I/O */
0031     BH_Delay,   /* Buffer is not yet allocated on disk */
0032     BH_Boundary,    /* Block is followed by a discontiguity */
0033     BH_Write_EIO,   /* I/O error on write */
0034     BH_Unwritten,   /* Buffer is allocated on disk but not written */
0035     BH_Quiet,   /* Buffer Error Prinks to be quiet */
0036     BH_Meta,    /* Buffer contains metadata */
0037     BH_Prio,    /* Buffer should be submitted with REQ_PRIO */
0038     BH_Defer_Completion, /* Defer AIO completion to workqueue */
0039
0040     BH_PrivateStart,/* not a state bit, but the first bit available
0041              * for private allocation by other entities
0042              */
0043 };
0044
0045 #define MAX_BUF_PER_PAGE (PAGE_SIZE / 512)
0046
0047 struct page;
0048 struct buffer_head;
0049 struct address_space;
0050 typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate);
0051
0052 /*
0053  * Historically, a buffer_head was used to map a single block
0054  * within a page, and of course as the unit of I/O through the
0055  * filesystem and block layers.  Nowadays the basic I/O unit
0056  * is the bio, and buffer_heads are used for extracting block
0057  * mappings (via a get_block_t call), for tracking state within
0058  * a page (via a page_mapping) and for wrapping bio submission
0059  * for backward compatibility reasons (e.g. submit_bh).
0060  */
0061 struct buffer_head {
0062     unsigned long b_state;      /* buffer state bitmap (see above) */
0063     struct buffer_head *b_this_page;/* circular list of page's buffers */
0064     struct page *b_page;        /* the page this bh is mapped to */
0065
0066     sector_t b_blocknr;     /* start block number */
0067     size_t b_size;          /* size of mapping */
0068     char *b_data;           /* pointer to data within the page */
0069
0070     struct block_device *b_bdev;
0071     bh_end_io_t *b_end_io;      /* I/O completion */
0072     void *b_private;        /* reserved for b_end_io */
0073     struct list_head b_assoc_buffers; /* associated with another mapping */
0074     struct address_space *b_assoc_map;  /* mapping this buffer is
0075                            associated with */
0076     atomic_t b_count;       /* users using this buffer_head */
0077     spinlock_t b_uptodate_lock; /* Used by the first bh in a page, to
0078                      * serialise IO completion of other
0079                      * buffers in the page */
0080 };
0081
0082 /*
0083  * macro tricks to expand the set_buffer_foo(), clear_buffer_foo()
0084  * and buffer_foo() functions.
0085  * To avoid reset buffer flags that are already set, because that causes
0086  * a costly cache line transition, check the flag first.
0087  */
0088 #define BUFFER_FNS(bit, name)                       \
0089 static __always_inline void set_buffer_##name(struct buffer_head *bh)   \
0090 {                                   \
0091     if (!test_bit(BH_##bit, &(bh)->b_state))            \
0092         set_bit(BH_##bit, &(bh)->b_state);          \
0093 }                                   \
0094 static __always_inline void clear_buffer_##name(struct buffer_head *bh) \
0095 {                                   \
0096     clear_bit(BH_##bit, &(bh)->b_state);                \
0097 }                                   \
0098 static __always_inline int buffer_##name(const struct buffer_head *bh)  \
0099 {                                   \
0100     return test_bit(BH_##bit, &(bh)->b_state);          \
0101 }
0102
0103 /*
0104  * test_set_buffer_foo() and test_clear_buffer_foo()
0105  */
0106 #define TAS_BUFFER_FNS(bit, name)                   \
0107 static __always_inline int test_set_buffer_##name(struct buffer_head *bh) \
0108 {                                   \
0109     return test_and_set_bit(BH_##bit, &(bh)->b_state);      \
0110 }                                   \
0111 static __always_inline int test_clear_buffer_##name(struct buffer_head *bh) \
0112 {                                   \
0113     return test_and_clear_bit(BH_##bit, &(bh)->b_state);        \
0114 }                                   \
0115
0116 /*
0117  * Emit the buffer bitops functions.   Note that there are also functions
0118  * of the form "mark_buffer_foo()".  These are higher-level functions which
0119  * do something in addition to setting a b_state bit.
0120  */
0121 BUFFER_FNS(Dirty, dirty)
0122 TAS_BUFFER_FNS(Dirty, dirty)
0123 BUFFER_FNS(Lock, locked)
0124 BUFFER_FNS(Req, req)
0125 TAS_BUFFER_FNS(Req, req)
0126 BUFFER_FNS(Mapped, mapped)
0127 BUFFER_FNS(New, new)
0128 BUFFER_FNS(Async_Read, async_read)
0129 BUFFER_FNS(Async_Write, async_write)
0130 BUFFER_FNS(Delay, delay)
0131 BUFFER_FNS(Boundary, boundary)
0132 BUFFER_FNS(Write_EIO, write_io_error)
0133 BUFFER_FNS(Unwritten, unwritten)
0134 BUFFER_FNS(Meta, meta)
0135 BUFFER_FNS(Prio, prio)
0136 BUFFER_FNS(Defer_Completion, defer_completion)
0137
0138 static __always_inline void set_buffer_uptodate(struct buffer_head *bh)
0139 {
0140     /*
0141      * If somebody else already set this uptodate, they will
0142      * have done the memory barrier, and a reader will thus
0143      * see *some* valid buffer state.
0144      *
0145      * Any other serialization (with IO errors or whatever that
0146      * might clear the bit) has to come from other state (eg BH_Lock).
0147      */
0148     if (test_bit(BH_Uptodate, &bh->b_state))
0149         return;
0150
0151     /*
0152      * make it consistent with folio_mark_uptodate
0153      * pairs with smp_load_acquire in buffer_uptodate
0154      */
0155     smp_mb__before_atomic();
0156     set_bit(BH_Uptodate, &bh->b_state);
0157 }
0158
0159 static __always_inline void clear_buffer_uptodate(struct buffer_head *bh)
0160 {
0161     clear_bit(BH_Uptodate, &bh->b_state);
0162 }
0163
0164 static __always_inline int buffer_uptodate(const struct buffer_head *bh)
0165 {
0166     /*
0167      * make it consistent with folio_test_uptodate
0168      * pairs with smp_mb__before_atomic in set_buffer_uptodate
0169      */
0170     return test_bit_acquire(BH_Uptodate, &bh->b_state);
0171 }
0172
0173 #define bh_offset(bh)       ((unsigned long)(bh)->b_data & ~PAGE_MASK)
0174
0175 /* If we *know* page->private refers to buffer_heads */
0176 #define page_buffers(page)                  \
0177     ({                          \
0178         BUG_ON(!PagePrivate(page));         \
0179         ((struct buffer_head *)page_private(page)); \
0180     })
0181 #define page_has_buffers(page)  PagePrivate(page)
0182 #define folio_buffers(folio)        folio_get_private(folio)
0183
0184 void buffer_check_dirty_writeback(struct folio *folio,
0185                      bool *dirty, bool *writeback);
0186
0187 /*
0188  * Declarations
0189  */
0190
0191 void mark_buffer_dirty(struct buffer_head *bh);
0192 void mark_buffer_write_io_error(struct buffer_head *bh);
0193 void touch_buffer(struct buffer_head *bh);
0194 void set_bh_page(struct buffer_head *bh,
0195         struct page *page, unsigned long offset);
0196 bool try_to_free_buffers(struct folio *);
0197 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
0198         bool retry);
0199 void create_empty_buffers(struct page *, unsigned long,
0200             unsigned long b_state);
0201 void end_buffer_read_sync(struct buffer_head *bh, int uptodate);
0202 void end_buffer_write_sync(struct buffer_head *bh, int uptodate);
0203 void end_buffer_async_write(struct buffer_head *bh, int uptodate);
0204
0205 /* Things to do with buffers at mapping->private_list */
0206 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode);
0207 int inode_has_buffers(struct inode *);
0208 void invalidate_inode_buffers(struct inode *);
0209 int remove_inode_buffers(struct inode *inode);
0210 int sync_mapping_buffers(struct address_space *mapping);
0211 void clean_bdev_aliases(struct block_device *bdev, sector_t block,
0212             sector_t len);
0213 static inline void clean_bdev_bh_alias(struct buffer_head *bh)
0214 {
0215     clean_bdev_aliases(bh->b_bdev, bh->b_blocknr, 1);
0216 }
0217
0218 void mark_buffer_async_write(struct buffer_head *bh);
0219 void __wait_on_buffer(struct buffer_head *);
0220 wait_queue_head_t *bh_waitq_head(struct buffer_head *bh);
0221 struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block,
0222             unsigned size);
0223 struct buffer_head *__getblk_gfp(struct block_device *bdev, sector_t block,
0224                   unsigned size, gfp_t gfp);
0225 void __brelse(struct buffer_head *);
0226 void __bforget(struct buffer_head *);
0227 void __breadahead(struct block_device *, sector_t block, unsigned int size);
0228 void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
0229           gfp_t gfp);
0230 struct buffer_head *__bread_gfp(struct block_device *,
0231                 sector_t block, unsigned size, gfp_t gfp);
0232 void invalidate_bh_lrus(void);
0233 void invalidate_bh_lrus_cpu(void);
0234 bool has_bh_in_lru(int cpu, void *dummy);
0235 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
0236 void free_buffer_head(struct buffer_head * bh);
0237 void unlock_buffer(struct buffer_head *bh);
0238 void __lock_buffer(struct buffer_head *bh);
0239 void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]);
0240 int sync_dirty_buffer(struct buffer_head *bh);
0241 int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
0242 void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
0243 int submit_bh(blk_opf_t, struct buffer_head *);
0244 void write_boundary_block(struct block_device *bdev,
0245             sector_t bblock, unsigned blocksize);
0246 int bh_uptodate_or_lock(struct buffer_head *bh);
0247 int bh_submit_read(struct buffer_head *bh);
0248
0249 extern int buffer_heads_over_limit;
0250
0251 /*
0252  * Generic address_space_operations implementations for buffer_head-backed
0253  * address_spaces.
0254  */
0255 void block_invalidate_folio(struct folio *folio, size_t offset, size_t length);
0256 int block_write_full_page(struct page *page, get_block_t *get_block,
0257                 struct writeback_control *wbc);
0258 int __block_write_full_page(struct inode *inode, struct page *page,
0259             get_block_t *get_block, struct writeback_control *wbc,
0260             bh_end_io_t *handler);
0261 int block_read_full_folio(struct folio *, get_block_t *);
0262 bool block_is_partially_uptodate(struct folio *, size_t from, size_t count);
0263 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
0264         struct page **pagep, get_block_t *get_block);
0265 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
0266         get_block_t *get_block);
0267 int block_write_end(struct file *, struct address_space *,
0268                 loff_t, unsigned, unsigned,
0269                 struct page *, void *);
0270 int generic_write_end(struct file *, struct address_space *,
0271                 loff_t, unsigned, unsigned,
0272                 struct page *, void *);
0273 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to);
0274 void clean_page_buffers(struct page *page);
0275 int cont_write_begin(struct file *, struct address_space *, loff_t,
0276             unsigned, struct page **, void **,
0277             get_block_t *, loff_t *);
0278 int generic_cont_expand_simple(struct inode *inode, loff_t size);
0279 int block_commit_write(struct page *page, unsigned from, unsigned to);
0280 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
0281                 get_block_t get_block);
0282 /* Convert errno to return value from ->page_mkwrite() call */
0283 static inline vm_fault_t block_page_mkwrite_return(int err)
0284 {
0285     if (err == 0)
0286         return VM_FAULT_LOCKED;
0287     if (err == -EFAULT || err == -EAGAIN)
0288         return VM_FAULT_NOPAGE;
0289     if (err == -ENOMEM)
0290         return VM_FAULT_OOM;
0291     /* -ENOSPC, -EDQUOT, -EIO ... */
0292     return VM_FAULT_SIGBUS;
0293 }
0294 sector_t generic_block_bmap(struct address_space *, sector_t, get_block_t *);
0295 int block_truncate_page(struct address_space *, loff_t, get_block_t *);
0296
0297 #ifdef CONFIG_MIGRATION
0298 extern int buffer_migrate_folio(struct address_space *,
0299         struct folio *dst, struct folio *src, enum migrate_mode);
0300 extern int buffer_migrate_folio_norefs(struct address_space *,
0301         struct folio *dst, struct folio *src, enum migrate_mode);
0302 #else
0303 #define buffer_migrate_folio NULL
0304 #define buffer_migrate_folio_norefs NULL
0305 #endif
0306
0307 void buffer_init(void);
0308
0309 /*
0310  * inline definitions
0311  */
0312
0313 static inline void get_bh(struct buffer_head *bh)
0314 {
0315         atomic_inc(&bh->b_count);
0316 }
0317
0318 static inline void put_bh(struct buffer_head *bh)
0319 {
0320         smp_mb__before_atomic();
0321         atomic_dec(&bh->b_count);
0322 }
0323
0324 static inline void brelse(struct buffer_head *bh)
0325 {
0326     if (bh)
0327         __brelse(bh);
0328 }
0329
0330 static inline void bforget(struct buffer_head *bh)
0331 {
0332     if (bh)
0333         __bforget(bh);
0334 }
0335
0336 static inline struct buffer_head *
0337 sb_bread(struct super_block *sb, sector_t block)
0338 {
0339     return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE);
0340 }
0341
0342 static inline struct buffer_head *
0343 sb_bread_unmovable(struct super_block *sb, sector_t block)
0344 {
0345     return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
0346 }
0347
0348 static inline void
0349 sb_breadahead(struct super_block *sb, sector_t block)
0350 {
0351     __breadahead(sb->s_bdev, block, sb->s_blocksize);
0352 }
0353
0354 static inline void
0355 sb_breadahead_unmovable(struct super_block *sb, sector_t block)
0356 {
0357     __breadahead_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
0358 }
0359
0360 static inline struct buffer_head *
0361 sb_getblk(struct super_block *sb, sector_t block)
0362 {
0363     return __getblk_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE);
0364 }
0365
0366
0367 static inline struct buffer_head *
0368 sb_getblk_gfp(struct super_block *sb, sector_t block, gfp_t gfp)
0369 {
0370     return __getblk_gfp(sb->s_bdev, block, sb->s_blocksize, gfp);
0371 }
0372
0373 static inline struct buffer_head *
0374 sb_find_get_block(struct super_block *sb, sector_t block)
0375 {
0376     return __find_get_block(sb->s_bdev, block, sb->s_blocksize);
0377 }
0378
0379 static inline void
0380 map_bh(struct buffer_head *bh, struct super_block *sb, sector_t block)
0381 {
0382     set_buffer_mapped(bh);
0383     bh->b_bdev = sb->s_bdev;
0384     bh->b_blocknr = block;
0385     bh->b_size = sb->s_blocksize;
0386 }
0387
0388 static inline void wait_on_buffer(struct buffer_head *bh)
0389 {
0390     might_sleep();
0391     if (buffer_locked(bh))
0392         __wait_on_buffer(bh);
0393 }
0394
0395 static inline int trylock_buffer(struct buffer_head *bh)
0396 {
0397     return likely(!test_and_set_bit_lock(BH_Lock, &bh->b_state));
0398 }
0399
0400 static inline void lock_buffer(struct buffer_head *bh)
0401 {
0402     might_sleep();
0403     if (!trylock_buffer(bh))
0404         __lock_buffer(bh);
0405 }
0406
0407 static inline struct buffer_head *getblk_unmovable(struct block_device *bdev,
0408                            sector_t block,
0409                            unsigned size)
0410 {
0411     return __getblk_gfp(bdev, block, size, 0);
0412 }
0413
0414 static inline struct buffer_head *__getblk(struct block_device *bdev,
0415                        sector_t block,
0416                        unsigned size)
0417 {
0418     return __getblk_gfp(bdev, block, size, __GFP_MOVABLE);
0419 }
0420
0421 /**
0422  *  __bread() - reads a specified block and returns the bh
0423  *  @bdev: the block_device to read from
0424  *  @block: number of block
0425  *  @size: size (in bytes) to read
0426  *
0427  *  Reads a specified block, and returns buffer head that contains it.
0428  *  The page cache is allocated from movable area so that it can be migrated.
0429  *  It returns NULL if the block was unreadable.
0430  */
0431 static inline struct buffer_head *
0432 __bread(struct block_device *bdev, sector_t block, unsigned size)
0433 {
0434     return __bread_gfp(bdev, block, size, __GFP_MOVABLE);
0435 }
0436
0437 bool block_dirty_folio(struct address_space *mapping, struct folio *folio);
0438
0439 #else /* CONFIG_BLOCK */
0440
0441 static inline void buffer_init(void) {}
0442 static inline bool try_to_free_buffers(struct folio *folio) { return true; }
0443 static inline int inode_has_buffers(struct inode *inode) { return 0; }
0444 static inline void invalidate_inode_buffers(struct inode *inode) {}
0445 static inline int remove_inode_buffers(struct inode *inode) { return 1; }
0446 static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
0447 static inline void invalidate_bh_lrus_cpu(void) {}
0448 static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; }
0449 #define buffer_heads_over_limit 0
0450
0451 #endif /* CONFIG_BLOCK */
0452 #endif /* _LINUX_BUFFER_HEAD_H */