fs/f2fs/segment.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * fs/f2fs/segment.c
0004  *
0005  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
0006  *             http://www.samsung.com/
0007  */
0008 #include <linux/fs.h>
0009 #include <linux/f2fs_fs.h>
0010 #include <linux/bio.h>
0011 #include <linux/blkdev.h>
0012 #include <linux/sched/mm.h>
0013 #include <linux/prefetch.h>
0014 #include <linux/kthread.h>
0015 #include <linux/swap.h>
0016 #include <linux/timer.h>
0017 #include <linux/freezer.h>
0018 #include <linux/sched/signal.h>
0019 #include <linux/random.h>
0020
0021 #include "f2fs.h"
0022 #include "segment.h"
0023 #include "node.h"
0024 #include "gc.h"
0025 #include "iostat.h"
0026 #include <trace/events/f2fs.h>
0027
0028 #define __reverse_ffz(x) __reverse_ffs(~(x))
0029
0030 static struct kmem_cache *discard_entry_slab;
0031 static struct kmem_cache *discard_cmd_slab;
0032 static struct kmem_cache *sit_entry_set_slab;
0033 static struct kmem_cache *revoke_entry_slab;
0034
0035 static unsigned long __reverse_ulong(unsigned char *str)
0036 {
0037     unsigned long tmp = 0;
0038     int shift = 24, idx = 0;
0039
0040 #if BITS_PER_LONG == 64
0041     shift = 56;
0042 #endif
0043     while (shift >= 0) {
0044         tmp |= (unsigned long)str[idx++] << shift;
0045         shift -= BITS_PER_BYTE;
0046     }
0047     return tmp;
0048 }
0049
0050 /*
0051  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
0052  * MSB and LSB are reversed in a byte by f2fs_set_bit.
0053  */
0054 static inline unsigned long __reverse_ffs(unsigned long word)
0055 {
0056     int num = 0;
0057
0058 #if BITS_PER_LONG == 64
0059     if ((word & 0xffffffff00000000UL) == 0)
0060         num += 32;
0061     else
0062         word >>= 32;
0063 #endif
0064     if ((word & 0xffff0000) == 0)
0065         num += 16;
0066     else
0067         word >>= 16;
0068
0069     if ((word & 0xff00) == 0)
0070         num += 8;
0071     else
0072         word >>= 8;
0073
0074     if ((word & 0xf0) == 0)
0075         num += 4;
0076     else
0077         word >>= 4;
0078
0079     if ((word & 0xc) == 0)
0080         num += 2;
0081     else
0082         word >>= 2;
0083
0084     if ((word & 0x2) == 0)
0085         num += 1;
0086     return num;
0087 }
0088
0089 /*
0090  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
0091  * f2fs_set_bit makes MSB and LSB reversed in a byte.
0092  * @size must be integral times of unsigned long.
0093  * Example:
0094  *                             MSB <--> LSB
0095  *   f2fs_set_bit(0, bitmap) => 1000 0000
0096  *   f2fs_set_bit(7, bitmap) => 0000 0001
0097  */
0098 static unsigned long __find_rev_next_bit(const unsigned long *addr,
0099             unsigned long size, unsigned long offset)
0100 {
0101     const unsigned long *p = addr + BIT_WORD(offset);
0102     unsigned long result = size;
0103     unsigned long tmp;
0104
0105     if (offset >= size)
0106         return size;
0107
0108     size -= (offset & ~(BITS_PER_LONG - 1));
0109     offset %= BITS_PER_LONG;
0110
0111     while (1) {
0112         if (*p == 0)
0113             goto pass;
0114
0115         tmp = __reverse_ulong((unsigned char *)p);
0116
0117         tmp &= ~0UL >> offset;
0118         if (size < BITS_PER_LONG)
0119             tmp &= (~0UL << (BITS_PER_LONG - size));
0120         if (tmp)
0121             goto found;
0122 pass:
0123         if (size <= BITS_PER_LONG)
0124             break;
0125         size -= BITS_PER_LONG;
0126         offset = 0;
0127         p++;
0128     }
0129     return result;
0130 found:
0131     return result - size + __reverse_ffs(tmp);
0132 }
0133
0134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
0135             unsigned long size, unsigned long offset)
0136 {
0137     const unsigned long *p = addr + BIT_WORD(offset);
0138     unsigned long result = size;
0139     unsigned long tmp;
0140
0141     if (offset >= size)
0142         return size;
0143
0144     size -= (offset & ~(BITS_PER_LONG - 1));
0145     offset %= BITS_PER_LONG;
0146
0147     while (1) {
0148         if (*p == ~0UL)
0149             goto pass;
0150
0151         tmp = __reverse_ulong((unsigned char *)p);
0152
0153         if (offset)
0154             tmp |= ~0UL << (BITS_PER_LONG - offset);
0155         if (size < BITS_PER_LONG)
0156             tmp |= ~0UL >> size;
0157         if (tmp != ~0UL)
0158             goto found;
0159 pass:
0160         if (size <= BITS_PER_LONG)
0161             break;
0162         size -= BITS_PER_LONG;
0163         offset = 0;
0164         p++;
0165     }
0166     return result;
0167 found:
0168     return result - size + __reverse_ffz(tmp);
0169 }
0170
0171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
0172 {
0173     int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
0174     int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
0175     int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
0176
0177     if (f2fs_lfs_mode(sbi))
0178         return false;
0179     if (sbi->gc_mode == GC_URGENT_HIGH)
0180         return true;
0181     if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
0182         return true;
0183
0184     return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
0185             SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
0186 }
0187
0188 void f2fs_abort_atomic_write(struct inode *inode, bool clean)
0189 {
0190     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
0191     struct f2fs_inode_info *fi = F2FS_I(inode);
0192
0193     if (!f2fs_is_atomic_file(inode))
0194         return;
0195
0196     if (clean)
0197         truncate_inode_pages_final(inode->i_mapping);
0198     clear_inode_flag(fi->cow_inode, FI_COW_FILE);
0199     iput(fi->cow_inode);
0200     fi->cow_inode = NULL;
0201     release_atomic_write_cnt(inode);
0202     clear_inode_flag(inode, FI_ATOMIC_FILE);
0203
0204     spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
0205     sbi->atomic_files--;
0206     spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
0207 }
0208
0209 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
0210             block_t new_addr, block_t *old_addr, bool recover)
0211 {
0212     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
0213     struct dnode_of_data dn;
0214     struct node_info ni;
0215     int err;
0216
0217 retry:
0218     set_new_dnode(&dn, inode, NULL, NULL, 0);
0219     err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE_RA);
0220     if (err) {
0221         if (err == -ENOMEM) {
0222             f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
0223             goto retry;
0224         }
0225         return err;
0226     }
0227
0228     err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
0229     if (err) {
0230         f2fs_put_dnode(&dn);
0231         return err;
0232     }
0233
0234     if (recover) {
0235         /* dn.data_blkaddr is always valid */
0236         if (!__is_valid_data_blkaddr(new_addr)) {
0237             if (new_addr == NULL_ADDR)
0238                 dec_valid_block_count(sbi, inode, 1);
0239             f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
0240             f2fs_update_data_blkaddr(&dn, new_addr);
0241         } else {
0242             f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
0243                 new_addr, ni.version, true, true);
0244         }
0245     } else {
0246         blkcnt_t count = 1;
0247
0248         *old_addr = dn.data_blkaddr;
0249         f2fs_truncate_data_blocks_range(&dn, 1);
0250         dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
0251         inc_valid_block_count(sbi, inode, &count);
0252         f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
0253                     ni.version, true, false);
0254     }
0255
0256     f2fs_put_dnode(&dn);
0257     return 0;
0258 }
0259
0260 static void __complete_revoke_list(struct inode *inode, struct list_head *head,
0261                     bool revoke)
0262 {
0263     struct revoke_entry *cur, *tmp;
0264
0265     list_for_each_entry_safe(cur, tmp, head, list) {
0266         if (revoke)
0267             __replace_atomic_write_block(inode, cur->index,
0268                         cur->old_addr, NULL, true);
0269         list_del(&cur->list);
0270         kmem_cache_free(revoke_entry_slab, cur);
0271     }
0272 }
0273
0274 static int __f2fs_commit_atomic_write(struct inode *inode)
0275 {
0276     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
0277     struct f2fs_inode_info *fi = F2FS_I(inode);
0278     struct inode *cow_inode = fi->cow_inode;
0279     struct revoke_entry *new;
0280     struct list_head revoke_list;
0281     block_t blkaddr;
0282     struct dnode_of_data dn;
0283     pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
0284     pgoff_t off = 0, blen, index;
0285     int ret = 0, i;
0286
0287     INIT_LIST_HEAD(&revoke_list);
0288
0289     while (len) {
0290         blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
0291
0292         set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
0293         ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
0294         if (ret && ret != -ENOENT) {
0295             goto out;
0296         } else if (ret == -ENOENT) {
0297             ret = 0;
0298             if (dn.max_level == 0)
0299                 goto out;
0300             goto next;
0301         }
0302
0303         blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode),
0304                 len);
0305         index = off;
0306         for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
0307             blkaddr = f2fs_data_blkaddr(&dn);
0308
0309             if (!__is_valid_data_blkaddr(blkaddr)) {
0310                 continue;
0311             } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
0312                     DATA_GENERIC_ENHANCE)) {
0313                 f2fs_put_dnode(&dn);
0314                 ret = -EFSCORRUPTED;
0315                 goto out;
0316             }
0317
0318             new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
0319                             true, NULL);
0320
0321             ret = __replace_atomic_write_block(inode, index, blkaddr,
0322                             &new->old_addr, false);
0323             if (ret) {
0324                 f2fs_put_dnode(&dn);
0325                 kmem_cache_free(revoke_entry_slab, new);
0326                 goto out;
0327             }
0328
0329             f2fs_update_data_blkaddr(&dn, NULL_ADDR);
0330             new->index = index;
0331             list_add_tail(&new->list, &revoke_list);
0332         }
0333         f2fs_put_dnode(&dn);
0334 next:
0335         off += blen;
0336         len -= blen;
0337     }
0338
0339 out:
0340     if (ret)
0341         sbi->revoked_atomic_block += fi->atomic_write_cnt;
0342     else
0343         sbi->committed_atomic_block += fi->atomic_write_cnt;
0344
0345     __complete_revoke_list(inode, &revoke_list, ret ? true : false);
0346
0347     return ret;
0348 }
0349
0350 int f2fs_commit_atomic_write(struct inode *inode)
0351 {
0352     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
0353     struct f2fs_inode_info *fi = F2FS_I(inode);
0354     int err;
0355
0356     err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
0357     if (err)
0358         return err;
0359
0360     f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
0361     f2fs_lock_op(sbi);
0362
0363     err = __f2fs_commit_atomic_write(inode);
0364
0365     f2fs_unlock_op(sbi);
0366     f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
0367
0368     return err;
0369 }
0370
0371 /*
0372  * This function balances dirty node and dentry pages.
0373  * In addition, it controls garbage collection.
0374  */
0375 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
0376 {
0377     if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
0378         f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
0379         f2fs_stop_checkpoint(sbi, false);
0380     }
0381
0382     /* balance_fs_bg is able to be pending */
0383     if (need && excess_cached_nats(sbi))
0384         f2fs_balance_fs_bg(sbi, false);
0385
0386     if (!f2fs_is_checkpoint_ready(sbi))
0387         return;
0388
0389     /*
0390      * We should do GC or end up with checkpoint, if there are so many dirty
0391      * dir/node pages without enough free segments.
0392      */
0393     if (has_not_enough_free_secs(sbi, 0, 0)) {
0394         if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
0395                     sbi->gc_thread->f2fs_gc_task) {
0396             DEFINE_WAIT(wait);
0397
0398             prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
0399                         TASK_UNINTERRUPTIBLE);
0400             wake_up(&sbi->gc_thread->gc_wait_queue_head);
0401             io_schedule();
0402             finish_wait(&sbi->gc_thread->fggc_wq, &wait);
0403         } else {
0404             struct f2fs_gc_control gc_control = {
0405                 .victim_segno = NULL_SEGNO,
0406                 .init_gc_type = BG_GC,
0407                 .no_bg_gc = true,
0408                 .should_migrate_blocks = false,
0409                 .err_gc_skipped = false,
0410                 .nr_free_secs = 1 };
0411             f2fs_down_write(&sbi->gc_lock);
0412             f2fs_gc(sbi, &gc_control);
0413         }
0414     }
0415 }
0416
0417 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
0418 {
0419     int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
0420     unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
0421     unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
0422     unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
0423     unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
0424     unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
0425     unsigned int threshold = sbi->blocks_per_seg * factor *
0426                     DEFAULT_DIRTY_THRESHOLD;
0427     unsigned int global_threshold = threshold * 3 / 2;
0428
0429     if (dents >= threshold || qdata >= threshold ||
0430         nodes >= threshold || meta >= threshold ||
0431         imeta >= threshold)
0432         return true;
0433     return dents + qdata + nodes + meta + imeta >  global_threshold;
0434 }
0435
0436 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
0437 {
0438     if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
0439         return;
0440
0441     /* try to shrink extent cache when there is no enough memory */
0442     if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
0443         f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
0444
0445     /* check the # of cached NAT entries */
0446     if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
0447         f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
0448
0449     if (!f2fs_available_free_memory(sbi, FREE_NIDS))
0450         f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
0451     else
0452         f2fs_build_free_nids(sbi, false, false);
0453
0454     if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
0455         excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
0456         goto do_sync;
0457
0458     /* there is background inflight IO or foreground operation recently */
0459     if (is_inflight_io(sbi, REQ_TIME) ||
0460         (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
0461         return;
0462
0463     /* exceed periodical checkpoint timeout threshold */
0464     if (f2fs_time_over(sbi, CP_TIME))
0465         goto do_sync;
0466
0467     /* checkpoint is the only way to shrink partial cached entries */
0468     if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
0469         f2fs_available_free_memory(sbi, INO_ENTRIES))
0470         return;
0471
0472 do_sync:
0473     if (test_opt(sbi, DATA_FLUSH) && from_bg) {
0474         struct blk_plug plug;
0475
0476         mutex_lock(&sbi->flush_lock);
0477
0478         blk_start_plug(&plug);
0479         f2fs_sync_dirty_inodes(sbi, FILE_INODE);
0480         blk_finish_plug(&plug);
0481
0482         mutex_unlock(&sbi->flush_lock);
0483     }
0484     f2fs_sync_fs(sbi->sb, true);
0485     stat_inc_bg_cp_count(sbi->stat_info);
0486 }
0487
0488 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
0489                 struct block_device *bdev)
0490 {
0491     int ret = blkdev_issue_flush(bdev);
0492
0493     trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
0494                 test_opt(sbi, FLUSH_MERGE), ret);
0495     return ret;
0496 }
0497
0498 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
0499 {
0500     int ret = 0;
0501     int i;
0502
0503     if (!f2fs_is_multi_device(sbi))
0504         return __submit_flush_wait(sbi, sbi->sb->s_bdev);
0505
0506     for (i = 0; i < sbi->s_ndevs; i++) {
0507         if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
0508             continue;
0509         ret = __submit_flush_wait(sbi, FDEV(i).bdev);
0510         if (ret)
0511             break;
0512     }
0513     return ret;
0514 }
0515
0516 static int issue_flush_thread(void *data)
0517 {
0518     struct f2fs_sb_info *sbi = data;
0519     struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
0520     wait_queue_head_t *q = &fcc->flush_wait_queue;
0521 repeat:
0522     if (kthread_should_stop())
0523         return 0;
0524
0525     if (!llist_empty(&fcc->issue_list)) {
0526         struct flush_cmd *cmd, *next;
0527         int ret;
0528
0529         fcc->dispatch_list = llist_del_all(&fcc->issue_list);
0530         fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
0531
0532         cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
0533
0534         ret = submit_flush_wait(sbi, cmd->ino);
0535         atomic_inc(&fcc->issued_flush);
0536
0537         llist_for_each_entry_safe(cmd, next,
0538                       fcc->dispatch_list, llnode) {
0539             cmd->ret = ret;
0540             complete(&cmd->wait);
0541         }
0542         fcc->dispatch_list = NULL;
0543     }
0544
0545     wait_event_interruptible(*q,
0546         kthread_should_stop() || !llist_empty(&fcc->issue_list));
0547     goto repeat;
0548 }
0549
0550 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
0551 {
0552     struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
0553     struct flush_cmd cmd;
0554     int ret;
0555
0556     if (test_opt(sbi, NOBARRIER))
0557         return 0;
0558
0559     if (!test_opt(sbi, FLUSH_MERGE)) {
0560         atomic_inc(&fcc->queued_flush);
0561         ret = submit_flush_wait(sbi, ino);
0562         atomic_dec(&fcc->queued_flush);
0563         atomic_inc(&fcc->issued_flush);
0564         return ret;
0565     }
0566
0567     if (atomic_inc_return(&fcc->queued_flush) == 1 ||
0568         f2fs_is_multi_device(sbi)) {
0569         ret = submit_flush_wait(sbi, ino);
0570         atomic_dec(&fcc->queued_flush);
0571
0572         atomic_inc(&fcc->issued_flush);
0573         return ret;
0574     }
0575
0576     cmd.ino = ino;
0577     init_completion(&cmd.wait);
0578
0579     llist_add(&cmd.llnode, &fcc->issue_list);
0580
0581     /*
0582      * update issue_list before we wake up issue_flush thread, this
0583      * smp_mb() pairs with another barrier in ___wait_event(), see
0584      * more details in comments of waitqueue_active().
0585      */
0586     smp_mb();
0587
0588     if (waitqueue_active(&fcc->flush_wait_queue))
0589         wake_up(&fcc->flush_wait_queue);
0590
0591     if (fcc->f2fs_issue_flush) {
0592         wait_for_completion(&cmd.wait);
0593         atomic_dec(&fcc->queued_flush);
0594     } else {
0595         struct llist_node *list;
0596
0597         list = llist_del_all(&fcc->issue_list);
0598         if (!list) {
0599             wait_for_completion(&cmd.wait);
0600             atomic_dec(&fcc->queued_flush);
0601         } else {
0602             struct flush_cmd *tmp, *next;
0603
0604             ret = submit_flush_wait(sbi, ino);
0605
0606             llist_for_each_entry_safe(tmp, next, list, llnode) {
0607                 if (tmp == &cmd) {
0608                     cmd.ret = ret;
0609                     atomic_dec(&fcc->queued_flush);
0610                     continue;
0611                 }
0612                 tmp->ret = ret;
0613                 complete(&tmp->wait);
0614             }
0615         }
0616     }
0617
0618     return cmd.ret;
0619 }
0620
0621 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
0622 {
0623     dev_t dev = sbi->sb->s_bdev->bd_dev;
0624     struct flush_cmd_control *fcc;
0625     int err = 0;
0626
0627     if (SM_I(sbi)->fcc_info) {
0628         fcc = SM_I(sbi)->fcc_info;
0629         if (fcc->f2fs_issue_flush)
0630             return err;
0631         goto init_thread;
0632     }
0633
0634     fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
0635     if (!fcc)
0636         return -ENOMEM;
0637     atomic_set(&fcc->issued_flush, 0);
0638     atomic_set(&fcc->queued_flush, 0);
0639     init_waitqueue_head(&fcc->flush_wait_queue);
0640     init_llist_head(&fcc->issue_list);
0641     SM_I(sbi)->fcc_info = fcc;
0642     if (!test_opt(sbi, FLUSH_MERGE))
0643         return err;
0644
0645 init_thread:
0646     fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
0647                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
0648     if (IS_ERR(fcc->f2fs_issue_flush)) {
0649         err = PTR_ERR(fcc->f2fs_issue_flush);
0650         kfree(fcc);
0651         SM_I(sbi)->fcc_info = NULL;
0652         return err;
0653     }
0654
0655     return err;
0656 }
0657
0658 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
0659 {
0660     struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
0661
0662     if (fcc && fcc->f2fs_issue_flush) {
0663         struct task_struct *flush_thread = fcc->f2fs_issue_flush;
0664
0665         fcc->f2fs_issue_flush = NULL;
0666         kthread_stop(flush_thread);
0667     }
0668     if (free) {
0669         kfree(fcc);
0670         SM_I(sbi)->fcc_info = NULL;
0671     }
0672 }
0673
0674 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
0675 {
0676     int ret = 0, i;
0677
0678     if (!f2fs_is_multi_device(sbi))
0679         return 0;
0680
0681     if (test_opt(sbi, NOBARRIER))
0682         return 0;
0683
0684     for (i = 1; i < sbi->s_ndevs; i++) {
0685         int count = DEFAULT_RETRY_IO_COUNT;
0686
0687         if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
0688             continue;
0689
0690         do {
0691             ret = __submit_flush_wait(sbi, FDEV(i).bdev);
0692             if (ret)
0693                 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
0694         } while (ret && --count);
0695
0696         if (ret) {
0697             f2fs_stop_checkpoint(sbi, false);
0698             break;
0699         }
0700
0701         spin_lock(&sbi->dev_lock);
0702         f2fs_clear_bit(i, (char *)&sbi->dirty_device);
0703         spin_unlock(&sbi->dev_lock);
0704     }
0705
0706     return ret;
0707 }
0708
0709 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
0710         enum dirty_type dirty_type)
0711 {
0712     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0713
0714     /* need not be added */
0715     if (IS_CURSEG(sbi, segno))
0716         return;
0717
0718     if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
0719         dirty_i->nr_dirty[dirty_type]++;
0720
0721     if (dirty_type == DIRTY) {
0722         struct seg_entry *sentry = get_seg_entry(sbi, segno);
0723         enum dirty_type t = sentry->type;
0724
0725         if (unlikely(t >= DIRTY)) {
0726             f2fs_bug_on(sbi, 1);
0727             return;
0728         }
0729         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
0730             dirty_i->nr_dirty[t]++;
0731
0732         if (__is_large_section(sbi)) {
0733             unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
0734             block_t valid_blocks =
0735                 get_valid_blocks(sbi, segno, true);
0736
0737             f2fs_bug_on(sbi, unlikely(!valid_blocks ||
0738                     valid_blocks == CAP_BLKS_PER_SEC(sbi)));
0739
0740             if (!IS_CURSEC(sbi, secno))
0741                 set_bit(secno, dirty_i->dirty_secmap);
0742         }
0743     }
0744 }
0745
0746 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
0747         enum dirty_type dirty_type)
0748 {
0749     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0750     block_t valid_blocks;
0751
0752     if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
0753         dirty_i->nr_dirty[dirty_type]--;
0754
0755     if (dirty_type == DIRTY) {
0756         struct seg_entry *sentry = get_seg_entry(sbi, segno);
0757         enum dirty_type t = sentry->type;
0758
0759         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
0760             dirty_i->nr_dirty[t]--;
0761
0762         valid_blocks = get_valid_blocks(sbi, segno, true);
0763         if (valid_blocks == 0) {
0764             clear_bit(GET_SEC_FROM_SEG(sbi, segno),
0765                         dirty_i->victim_secmap);
0766 #ifdef CONFIG_F2FS_CHECK_FS
0767             clear_bit(segno, SIT_I(sbi)->invalid_segmap);
0768 #endif
0769         }
0770         if (__is_large_section(sbi)) {
0771             unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
0772
0773             if (!valid_blocks ||
0774                     valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
0775                 clear_bit(secno, dirty_i->dirty_secmap);
0776                 return;
0777             }
0778
0779             if (!IS_CURSEC(sbi, secno))
0780                 set_bit(secno, dirty_i->dirty_secmap);
0781         }
0782     }
0783 }
0784
0785 /*
0786  * Should not occur error such as -ENOMEM.
0787  * Adding dirty entry into seglist is not critical operation.
0788  * If a given segment is one of current working segments, it won't be added.
0789  */
0790 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
0791 {
0792     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0793     unsigned short valid_blocks, ckpt_valid_blocks;
0794     unsigned int usable_blocks;
0795
0796     if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
0797         return;
0798
0799     usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
0800     mutex_lock(&dirty_i->seglist_lock);
0801
0802     valid_blocks = get_valid_blocks(sbi, segno, false);
0803     ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
0804
0805     if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
0806         ckpt_valid_blocks == usable_blocks)) {
0807         __locate_dirty_segment(sbi, segno, PRE);
0808         __remove_dirty_segment(sbi, segno, DIRTY);
0809     } else if (valid_blocks < usable_blocks) {
0810         __locate_dirty_segment(sbi, segno, DIRTY);
0811     } else {
0812         /* Recovery routine with SSR needs this */
0813         __remove_dirty_segment(sbi, segno, DIRTY);
0814     }
0815
0816     mutex_unlock(&dirty_i->seglist_lock);
0817 }
0818
0819 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
0820 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
0821 {
0822     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0823     unsigned int segno;
0824
0825     mutex_lock(&dirty_i->seglist_lock);
0826     for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
0827         if (get_valid_blocks(sbi, segno, false))
0828             continue;
0829         if (IS_CURSEG(sbi, segno))
0830             continue;
0831         __locate_dirty_segment(sbi, segno, PRE);
0832         __remove_dirty_segment(sbi, segno, DIRTY);
0833     }
0834     mutex_unlock(&dirty_i->seglist_lock);
0835 }
0836
0837 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
0838 {
0839     int ovp_hole_segs =
0840         (overprovision_segments(sbi) - reserved_segments(sbi));
0841     block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
0842     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0843     block_t holes[2] = {0, 0};  /* DATA and NODE */
0844     block_t unusable;
0845     struct seg_entry *se;
0846     unsigned int segno;
0847
0848     mutex_lock(&dirty_i->seglist_lock);
0849     for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
0850         se = get_seg_entry(sbi, segno);
0851         if (IS_NODESEG(se->type))
0852             holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
0853                             se->valid_blocks;
0854         else
0855             holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
0856                             se->valid_blocks;
0857     }
0858     mutex_unlock(&dirty_i->seglist_lock);
0859
0860     unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
0861     if (unusable > ovp_holes)
0862         return unusable - ovp_holes;
0863     return 0;
0864 }
0865
0866 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
0867 {
0868     int ovp_hole_segs =
0869         (overprovision_segments(sbi) - reserved_segments(sbi));
0870     if (unusable > F2FS_OPTION(sbi).unusable_cap)
0871         return -EAGAIN;
0872     if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
0873         dirty_segments(sbi) > ovp_hole_segs)
0874         return -EAGAIN;
0875     return 0;
0876 }
0877
0878 /* This is only used by SBI_CP_DISABLED */
0879 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
0880 {
0881     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
0882     unsigned int segno = 0;
0883
0884     mutex_lock(&dirty_i->seglist_lock);
0885     for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
0886         if (get_valid_blocks(sbi, segno, false))
0887             continue;
0888         if (get_ckpt_valid_blocks(sbi, segno, false))
0889             continue;
0890         mutex_unlock(&dirty_i->seglist_lock);
0891         return segno;
0892     }
0893     mutex_unlock(&dirty_i->seglist_lock);
0894     return NULL_SEGNO;
0895 }
0896
0897 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
0898         struct block_device *bdev, block_t lstart,
0899         block_t start, block_t len)
0900 {
0901     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
0902     struct list_head *pend_list;
0903     struct discard_cmd *dc;
0904
0905     f2fs_bug_on(sbi, !len);
0906
0907     pend_list = &dcc->pend_list[plist_idx(len)];
0908
0909     dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
0910     INIT_LIST_HEAD(&dc->list);
0911     dc->bdev = bdev;
0912     dc->lstart = lstart;
0913     dc->start = start;
0914     dc->len = len;
0915     dc->ref = 0;
0916     dc->state = D_PREP;
0917     dc->queued = 0;
0918     dc->error = 0;
0919     init_completion(&dc->wait);
0920     list_add_tail(&dc->list, pend_list);
0921     spin_lock_init(&dc->lock);
0922     dc->bio_ref = 0;
0923     atomic_inc(&dcc->discard_cmd_cnt);
0924     dcc->undiscard_blks += len;
0925
0926     return dc;
0927 }
0928
0929 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
0930                 struct block_device *bdev, block_t lstart,
0931                 block_t start, block_t len,
0932                 struct rb_node *parent, struct rb_node **p,
0933                 bool leftmost)
0934 {
0935     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
0936     struct discard_cmd *dc;
0937
0938     dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
0939
0940     rb_link_node(&dc->rb_node, parent, p);
0941     rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
0942
0943     return dc;
0944 }
0945
0946 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
0947                             struct discard_cmd *dc)
0948 {
0949     if (dc->state == D_DONE)
0950         atomic_sub(dc->queued, &dcc->queued_discard);
0951
0952     list_del(&dc->list);
0953     rb_erase_cached(&dc->rb_node, &dcc->root);
0954     dcc->undiscard_blks -= dc->len;
0955
0956     kmem_cache_free(discard_cmd_slab, dc);
0957
0958     atomic_dec(&dcc->discard_cmd_cnt);
0959 }
0960
0961 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
0962                             struct discard_cmd *dc)
0963 {
0964     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
0965     unsigned long flags;
0966
0967     trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
0968
0969     spin_lock_irqsave(&dc->lock, flags);
0970     if (dc->bio_ref) {
0971         spin_unlock_irqrestore(&dc->lock, flags);
0972         return;
0973     }
0974     spin_unlock_irqrestore(&dc->lock, flags);
0975
0976     f2fs_bug_on(sbi, dc->ref);
0977
0978     if (dc->error == -EOPNOTSUPP)
0979         dc->error = 0;
0980
0981     if (dc->error)
0982         printk_ratelimited(
0983             "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
0984             KERN_INFO, sbi->sb->s_id,
0985             dc->lstart, dc->start, dc->len, dc->error);
0986     __detach_discard_cmd(dcc, dc);
0987 }
0988
0989 static void f2fs_submit_discard_endio(struct bio *bio)
0990 {
0991     struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
0992     unsigned long flags;
0993
0994     spin_lock_irqsave(&dc->lock, flags);
0995     if (!dc->error)
0996         dc->error = blk_status_to_errno(bio->bi_status);
0997     dc->bio_ref--;
0998     if (!dc->bio_ref && dc->state == D_SUBMIT) {
0999         dc->state = D_DONE;
1000         complete_all(&dc->wait);
1001     }
1002     spin_unlock_irqrestore(&dc->lock, flags);
1003     bio_put(bio);
1004 }
1005
1006 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1007                 block_t start, block_t end)
1008 {
1009 #ifdef CONFIG_F2FS_CHECK_FS
1010     struct seg_entry *sentry;
1011     unsigned int segno;
1012     block_t blk = start;
1013     unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1014     unsigned long *map;
1015
1016     while (blk < end) {
1017         segno = GET_SEGNO(sbi, blk);
1018         sentry = get_seg_entry(sbi, segno);
1019         offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1020
1021         if (end < START_BLOCK(sbi, segno + 1))
1022             size = GET_BLKOFF_FROM_SEG0(sbi, end);
1023         else
1024             size = max_blocks;
1025         map = (unsigned long *)(sentry->cur_valid_map);
1026         offset = __find_rev_next_bit(map, size, offset);
1027         f2fs_bug_on(sbi, offset != size);
1028         blk = START_BLOCK(sbi, segno + 1);
1029     }
1030 #endif
1031 }
1032
1033 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1034                 struct discard_policy *dpolicy,
1035                 int discard_type, unsigned int granularity)
1036 {
1037     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1038
1039     /* common policy */
1040     dpolicy->type = discard_type;
1041     dpolicy->sync = true;
1042     dpolicy->ordered = false;
1043     dpolicy->granularity = granularity;
1044
1045     dpolicy->max_requests = dcc->max_discard_request;
1046     dpolicy->io_aware_gran = MAX_PLIST_NUM;
1047     dpolicy->timeout = false;
1048
1049     if (discard_type == DPOLICY_BG) {
1050         dpolicy->min_interval = dcc->min_discard_issue_time;
1051         dpolicy->mid_interval = dcc->mid_discard_issue_time;
1052         dpolicy->max_interval = dcc->max_discard_issue_time;
1053         dpolicy->io_aware = true;
1054         dpolicy->sync = false;
1055         dpolicy->ordered = true;
1056         if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1057             dpolicy->granularity = 1;
1058             if (atomic_read(&dcc->discard_cmd_cnt))
1059                 dpolicy->max_interval =
1060                     dcc->min_discard_issue_time;
1061         }
1062     } else if (discard_type == DPOLICY_FORCE) {
1063         dpolicy->min_interval = dcc->min_discard_issue_time;
1064         dpolicy->mid_interval = dcc->mid_discard_issue_time;
1065         dpolicy->max_interval = dcc->max_discard_issue_time;
1066         dpolicy->io_aware = false;
1067     } else if (discard_type == DPOLICY_FSTRIM) {
1068         dpolicy->io_aware = false;
1069     } else if (discard_type == DPOLICY_UMOUNT) {
1070         dpolicy->io_aware = false;
1071         /* we need to issue all to keep CP_TRIMMED_FLAG */
1072         dpolicy->granularity = 1;
1073         dpolicy->timeout = true;
1074     }
1075 }
1076
1077 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1078                 struct block_device *bdev, block_t lstart,
1079                 block_t start, block_t len);
1080 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1081 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1082                         struct discard_policy *dpolicy,
1083                         struct discard_cmd *dc,
1084                         unsigned int *issued)
1085 {
1086     struct block_device *bdev = dc->bdev;
1087     unsigned int max_discard_blocks =
1088             SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1089     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1090     struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1091                     &(dcc->fstrim_list) : &(dcc->wait_list);
1092     blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1093     block_t lstart, start, len, total_len;
1094     int err = 0;
1095
1096     if (dc->state != D_PREP)
1097         return 0;
1098
1099     if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1100         return 0;
1101
1102     trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1103
1104     lstart = dc->lstart;
1105     start = dc->start;
1106     len = dc->len;
1107     total_len = len;
1108
1109     dc->len = 0;
1110
1111     while (total_len && *issued < dpolicy->max_requests && !err) {
1112         struct bio *bio = NULL;
1113         unsigned long flags;
1114         bool last = true;
1115
1116         if (len > max_discard_blocks) {
1117             len = max_discard_blocks;
1118             last = false;
1119         }
1120
1121         (*issued)++;
1122         if (*issued == dpolicy->max_requests)
1123             last = true;
1124
1125         dc->len += len;
1126
1127         if (time_to_inject(sbi, FAULT_DISCARD)) {
1128             f2fs_show_injection_info(sbi, FAULT_DISCARD);
1129             err = -EIO;
1130             goto submit;
1131         }
1132         err = __blkdev_issue_discard(bdev,
1133                     SECTOR_FROM_BLOCK(start),
1134                     SECTOR_FROM_BLOCK(len),
1135                     GFP_NOFS, &bio);
1136 submit:
1137         if (err) {
1138             spin_lock_irqsave(&dc->lock, flags);
1139             if (dc->state == D_PARTIAL)
1140                 dc->state = D_SUBMIT;
1141             spin_unlock_irqrestore(&dc->lock, flags);
1142
1143             break;
1144         }
1145
1146         f2fs_bug_on(sbi, !bio);
1147
1148         /*
1149          * should keep before submission to avoid D_DONE
1150          * right away
1151          */
1152         spin_lock_irqsave(&dc->lock, flags);
1153         if (last)
1154             dc->state = D_SUBMIT;
1155         else
1156             dc->state = D_PARTIAL;
1157         dc->bio_ref++;
1158         spin_unlock_irqrestore(&dc->lock, flags);
1159
1160         atomic_inc(&dcc->queued_discard);
1161         dc->queued++;
1162         list_move_tail(&dc->list, wait_list);
1163
1164         /* sanity check on discard range */
1165         __check_sit_bitmap(sbi, lstart, lstart + len);
1166
1167         bio->bi_private = dc;
1168         bio->bi_end_io = f2fs_submit_discard_endio;
1169         bio->bi_opf |= flag;
1170         submit_bio(bio);
1171
1172         atomic_inc(&dcc->issued_discard);
1173
1174         f2fs_update_iostat(sbi, FS_DISCARD, 1);
1175
1176         lstart += len;
1177         start += len;
1178         total_len -= len;
1179         len = total_len;
1180     }
1181
1182     if (!err && len) {
1183         dcc->undiscard_blks -= len;
1184         __update_discard_tree_range(sbi, bdev, lstart, start, len);
1185     }
1186     return err;
1187 }
1188
1189 static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1190                 struct block_device *bdev, block_t lstart,
1191                 block_t start, block_t len,
1192                 struct rb_node **insert_p,
1193                 struct rb_node *insert_parent)
1194 {
1195     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1196     struct rb_node **p;
1197     struct rb_node *parent = NULL;
1198     bool leftmost = true;
1199
1200     if (insert_p && insert_parent) {
1201         parent = insert_parent;
1202         p = insert_p;
1203         goto do_insert;
1204     }
1205
1206     p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1207                             lstart, &leftmost);
1208 do_insert:
1209     __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1210                                 p, leftmost);
1211 }
1212
1213 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1214                         struct discard_cmd *dc)
1215 {
1216     list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1217 }
1218
1219 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1220                 struct discard_cmd *dc, block_t blkaddr)
1221 {
1222     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1223     struct discard_info di = dc->di;
1224     bool modified = false;
1225
1226     if (dc->state == D_DONE || dc->len == 1) {
1227         __remove_discard_cmd(sbi, dc);
1228         return;
1229     }
1230
1231     dcc->undiscard_blks -= di.len;
1232
1233     if (blkaddr > di.lstart) {
1234         dc->len = blkaddr - dc->lstart;
1235         dcc->undiscard_blks += dc->len;
1236         __relocate_discard_cmd(dcc, dc);
1237         modified = true;
1238     }
1239
1240     if (blkaddr < di.lstart + di.len - 1) {
1241         if (modified) {
1242             __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1243                     di.start + blkaddr + 1 - di.lstart,
1244                     di.lstart + di.len - 1 - blkaddr,
1245                     NULL, NULL);
1246         } else {
1247             dc->lstart++;
1248             dc->len--;
1249             dc->start++;
1250             dcc->undiscard_blks += dc->len;
1251             __relocate_discard_cmd(dcc, dc);
1252         }
1253     }
1254 }
1255
1256 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1257                 struct block_device *bdev, block_t lstart,
1258                 block_t start, block_t len)
1259 {
1260     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1261     struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1262     struct discard_cmd *dc;
1263     struct discard_info di = {0};
1264     struct rb_node **insert_p = NULL, *insert_parent = NULL;
1265     unsigned int max_discard_blocks =
1266             SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1267     block_t end = lstart + len;
1268
1269     dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1270                     NULL, lstart,
1271                     (struct rb_entry **)&prev_dc,
1272                     (struct rb_entry **)&next_dc,
1273                     &insert_p, &insert_parent, true, NULL);
1274     if (dc)
1275         prev_dc = dc;
1276
1277     if (!prev_dc) {
1278         di.lstart = lstart;
1279         di.len = next_dc ? next_dc->lstart - lstart : len;
1280         di.len = min(di.len, len);
1281         di.start = start;
1282     }
1283
1284     while (1) {
1285         struct rb_node *node;
1286         bool merged = false;
1287         struct discard_cmd *tdc = NULL;
1288
1289         if (prev_dc) {
1290             di.lstart = prev_dc->lstart + prev_dc->len;
1291             if (di.lstart < lstart)
1292                 di.lstart = lstart;
1293             if (di.lstart >= end)
1294                 break;
1295
1296             if (!next_dc || next_dc->lstart > end)
1297                 di.len = end - di.lstart;
1298             else
1299                 di.len = next_dc->lstart - di.lstart;
1300             di.start = start + di.lstart - lstart;
1301         }
1302
1303         if (!di.len)
1304             goto next;
1305
1306         if (prev_dc && prev_dc->state == D_PREP &&
1307             prev_dc->bdev == bdev &&
1308             __is_discard_back_mergeable(&di, &prev_dc->di,
1309                             max_discard_blocks)) {
1310             prev_dc->di.len += di.len;
1311             dcc->undiscard_blks += di.len;
1312             __relocate_discard_cmd(dcc, prev_dc);
1313             di = prev_dc->di;
1314             tdc = prev_dc;
1315             merged = true;
1316         }
1317
1318         if (next_dc && next_dc->state == D_PREP &&
1319             next_dc->bdev == bdev &&
1320             __is_discard_front_mergeable(&di, &next_dc->di,
1321                             max_discard_blocks)) {
1322             next_dc->di.lstart = di.lstart;
1323             next_dc->di.len += di.len;
1324             next_dc->di.start = di.start;
1325             dcc->undiscard_blks += di.len;
1326             __relocate_discard_cmd(dcc, next_dc);
1327             if (tdc)
1328                 __remove_discard_cmd(sbi, tdc);
1329             merged = true;
1330         }
1331
1332         if (!merged) {
1333             __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1334                             di.len, NULL, NULL);
1335         }
1336  next:
1337         prev_dc = next_dc;
1338         if (!prev_dc)
1339             break;
1340
1341         node = rb_next(&prev_dc->rb_node);
1342         next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1343     }
1344 }
1345
1346 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1347         struct block_device *bdev, block_t blkstart, block_t blklen)
1348 {
1349     block_t lblkstart = blkstart;
1350
1351     if (!f2fs_bdev_support_discard(bdev))
1352         return 0;
1353
1354     trace_f2fs_queue_discard(bdev, blkstart, blklen);
1355
1356     if (f2fs_is_multi_device(sbi)) {
1357         int devi = f2fs_target_device_index(sbi, blkstart);
1358
1359         blkstart -= FDEV(devi).start_blk;
1360     }
1361     mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1362     __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1363     mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1364     return 0;
1365 }
1366
1367 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1368                     struct discard_policy *dpolicy)
1369 {
1370     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1371     struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1372     struct rb_node **insert_p = NULL, *insert_parent = NULL;
1373     struct discard_cmd *dc;
1374     struct blk_plug plug;
1375     unsigned int pos = dcc->next_pos;
1376     unsigned int issued = 0;
1377     bool io_interrupted = false;
1378
1379     mutex_lock(&dcc->cmd_lock);
1380     dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1381                     NULL, pos,
1382                     (struct rb_entry **)&prev_dc,
1383                     (struct rb_entry **)&next_dc,
1384                     &insert_p, &insert_parent, true, NULL);
1385     if (!dc)
1386         dc = next_dc;
1387
1388     blk_start_plug(&plug);
1389
1390     while (dc) {
1391         struct rb_node *node;
1392         int err = 0;
1393
1394         if (dc->state != D_PREP)
1395             goto next;
1396
1397         if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1398             io_interrupted = true;
1399             break;
1400         }
1401
1402         dcc->next_pos = dc->lstart + dc->len;
1403         err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1404
1405         if (issued >= dpolicy->max_requests)
1406             break;
1407 next:
1408         node = rb_next(&dc->rb_node);
1409         if (err)
1410             __remove_discard_cmd(sbi, dc);
1411         dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1412     }
1413
1414     blk_finish_plug(&plug);
1415
1416     if (!dc)
1417         dcc->next_pos = 0;
1418
1419     mutex_unlock(&dcc->cmd_lock);
1420
1421     if (!issued && io_interrupted)
1422         issued = -1;
1423
1424     return issued;
1425 }
1426 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1427                     struct discard_policy *dpolicy);
1428
1429 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1430                     struct discard_policy *dpolicy)
1431 {
1432     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1433     struct list_head *pend_list;
1434     struct discard_cmd *dc, *tmp;
1435     struct blk_plug plug;
1436     int i, issued;
1437     bool io_interrupted = false;
1438
1439     if (dpolicy->timeout)
1440         f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1441
1442 retry:
1443     issued = 0;
1444     for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1445         if (dpolicy->timeout &&
1446                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1447             break;
1448
1449         if (i + 1 < dpolicy->granularity)
1450             break;
1451
1452         if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1453             return __issue_discard_cmd_orderly(sbi, dpolicy);
1454
1455         pend_list = &dcc->pend_list[i];
1456
1457         mutex_lock(&dcc->cmd_lock);
1458         if (list_empty(pend_list))
1459             goto next;
1460         if (unlikely(dcc->rbtree_check))
1461             f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1462                             &dcc->root, false));
1463         blk_start_plug(&plug);
1464         list_for_each_entry_safe(dc, tmp, pend_list, list) {
1465             f2fs_bug_on(sbi, dc->state != D_PREP);
1466
1467             if (dpolicy->timeout &&
1468                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1469                 break;
1470
1471             if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1472                         !is_idle(sbi, DISCARD_TIME)) {
1473                 io_interrupted = true;
1474                 break;
1475             }
1476
1477             __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1478
1479             if (issued >= dpolicy->max_requests)
1480                 break;
1481         }
1482         blk_finish_plug(&plug);
1483 next:
1484         mutex_unlock(&dcc->cmd_lock);
1485
1486         if (issued >= dpolicy->max_requests || io_interrupted)
1487             break;
1488     }
1489
1490     if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1491         __wait_all_discard_cmd(sbi, dpolicy);
1492         goto retry;
1493     }
1494
1495     if (!issued && io_interrupted)
1496         issued = -1;
1497
1498     return issued;
1499 }
1500
1501 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1502 {
1503     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1504     struct list_head *pend_list;
1505     struct discard_cmd *dc, *tmp;
1506     int i;
1507     bool dropped = false;
1508
1509     mutex_lock(&dcc->cmd_lock);
1510     for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1511         pend_list = &dcc->pend_list[i];
1512         list_for_each_entry_safe(dc, tmp, pend_list, list) {
1513             f2fs_bug_on(sbi, dc->state != D_PREP);
1514             __remove_discard_cmd(sbi, dc);
1515             dropped = true;
1516         }
1517     }
1518     mutex_unlock(&dcc->cmd_lock);
1519
1520     return dropped;
1521 }
1522
1523 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1524 {
1525     __drop_discard_cmd(sbi);
1526 }
1527
1528 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1529                             struct discard_cmd *dc)
1530 {
1531     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1532     unsigned int len = 0;
1533
1534     wait_for_completion_io(&dc->wait);
1535     mutex_lock(&dcc->cmd_lock);
1536     f2fs_bug_on(sbi, dc->state != D_DONE);
1537     dc->ref--;
1538     if (!dc->ref) {
1539         if (!dc->error)
1540             len = dc->len;
1541         __remove_discard_cmd(sbi, dc);
1542     }
1543     mutex_unlock(&dcc->cmd_lock);
1544
1545     return len;
1546 }
1547
1548 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1549                         struct discard_policy *dpolicy,
1550                         block_t start, block_t end)
1551 {
1552     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1553     struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1554                     &(dcc->fstrim_list) : &(dcc->wait_list);
1555     struct discard_cmd *dc = NULL, *iter, *tmp;
1556     unsigned int trimmed = 0;
1557
1558 next:
1559     dc = NULL;
1560
1561     mutex_lock(&dcc->cmd_lock);
1562     list_for_each_entry_safe(iter, tmp, wait_list, list) {
1563         if (iter->lstart + iter->len <= start || end <= iter->lstart)
1564             continue;
1565         if (iter->len < dpolicy->granularity)
1566             continue;
1567         if (iter->state == D_DONE && !iter->ref) {
1568             wait_for_completion_io(&iter->wait);
1569             if (!iter->error)
1570                 trimmed += iter->len;
1571             __remove_discard_cmd(sbi, iter);
1572         } else {
1573             iter->ref++;
1574             dc = iter;
1575             break;
1576         }
1577     }
1578     mutex_unlock(&dcc->cmd_lock);
1579
1580     if (dc) {
1581         trimmed += __wait_one_discard_bio(sbi, dc);
1582         goto next;
1583     }
1584
1585     return trimmed;
1586 }
1587
1588 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1589                         struct discard_policy *dpolicy)
1590 {
1591     struct discard_policy dp;
1592     unsigned int discard_blks;
1593
1594     if (dpolicy)
1595         return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1596
1597     /* wait all */
1598     __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1599     discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1600     __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1601     discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1602
1603     return discard_blks;
1604 }
1605
1606 /* This should be covered by global mutex, &sit_i->sentry_lock */
1607 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1608 {
1609     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1610     struct discard_cmd *dc;
1611     bool need_wait = false;
1612
1613     mutex_lock(&dcc->cmd_lock);
1614     dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1615                             NULL, blkaddr);
1616     if (dc) {
1617         if (dc->state == D_PREP) {
1618             __punch_discard_cmd(sbi, dc, blkaddr);
1619         } else {
1620             dc->ref++;
1621             need_wait = true;
1622         }
1623     }
1624     mutex_unlock(&dcc->cmd_lock);
1625
1626     if (need_wait)
1627         __wait_one_discard_bio(sbi, dc);
1628 }
1629
1630 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1631 {
1632     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1633
1634     if (dcc && dcc->f2fs_issue_discard) {
1635         struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1636
1637         dcc->f2fs_issue_discard = NULL;
1638         kthread_stop(discard_thread);
1639     }
1640 }
1641
1642 /* This comes from f2fs_put_super */
1643 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1644 {
1645     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1646     struct discard_policy dpolicy;
1647     bool dropped;
1648
1649     __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1650                     dcc->discard_granularity);
1651     __issue_discard_cmd(sbi, &dpolicy);
1652     dropped = __drop_discard_cmd(sbi);
1653
1654     /* just to make sure there is no pending discard commands */
1655     __wait_all_discard_cmd(sbi, NULL);
1656
1657     f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1658     return dropped;
1659 }
1660
1661 static int issue_discard_thread(void *data)
1662 {
1663     struct f2fs_sb_info *sbi = data;
1664     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1665     wait_queue_head_t *q = &dcc->discard_wait_queue;
1666     struct discard_policy dpolicy;
1667     unsigned int wait_ms = dcc->min_discard_issue_time;
1668     int issued;
1669
1670     set_freezable();
1671
1672     do {
1673         if (sbi->gc_mode == GC_URGENT_HIGH ||
1674             !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1675             __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1676         else
1677             __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1678                         dcc->discard_granularity);
1679
1680         if (!atomic_read(&dcc->discard_cmd_cnt))
1681                wait_ms = dpolicy.max_interval;
1682
1683         wait_event_interruptible_timeout(*q,
1684                 kthread_should_stop() || freezing(current) ||
1685                 dcc->discard_wake,
1686                 msecs_to_jiffies(wait_ms));
1687
1688         if (dcc->discard_wake)
1689             dcc->discard_wake = 0;
1690
1691         /* clean up pending candidates before going to sleep */
1692         if (atomic_read(&dcc->queued_discard))
1693             __wait_all_discard_cmd(sbi, NULL);
1694
1695         if (try_to_freeze())
1696             continue;
1697         if (f2fs_readonly(sbi->sb))
1698             continue;
1699         if (kthread_should_stop())
1700             return 0;
1701         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1702             wait_ms = dpolicy.max_interval;
1703             continue;
1704         }
1705         if (!atomic_read(&dcc->discard_cmd_cnt))
1706             continue;
1707
1708         sb_start_intwrite(sbi->sb);
1709
1710         issued = __issue_discard_cmd(sbi, &dpolicy);
1711         if (issued > 0) {
1712             __wait_all_discard_cmd(sbi, &dpolicy);
1713             wait_ms = dpolicy.min_interval;
1714         } else if (issued == -1) {
1715             wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1716             if (!wait_ms)
1717                 wait_ms = dpolicy.mid_interval;
1718         } else {
1719             wait_ms = dpolicy.max_interval;
1720         }
1721
1722         sb_end_intwrite(sbi->sb);
1723
1724     } while (!kthread_should_stop());
1725     return 0;
1726 }
1727
1728 #ifdef CONFIG_BLK_DEV_ZONED
1729 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1730         struct block_device *bdev, block_t blkstart, block_t blklen)
1731 {
1732     sector_t sector, nr_sects;
1733     block_t lblkstart = blkstart;
1734     int devi = 0;
1735
1736     if (f2fs_is_multi_device(sbi)) {
1737         devi = f2fs_target_device_index(sbi, blkstart);
1738         if (blkstart < FDEV(devi).start_blk ||
1739             blkstart > FDEV(devi).end_blk) {
1740             f2fs_err(sbi, "Invalid block %x", blkstart);
1741             return -EIO;
1742         }
1743         blkstart -= FDEV(devi).start_blk;
1744     }
1745
1746     /* For sequential zones, reset the zone write pointer */
1747     if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1748         sector = SECTOR_FROM_BLOCK(blkstart);
1749         nr_sects = SECTOR_FROM_BLOCK(blklen);
1750
1751         if (sector & (bdev_zone_sectors(bdev) - 1) ||
1752                 nr_sects != bdev_zone_sectors(bdev)) {
1753             f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1754                  devi, sbi->s_ndevs ? FDEV(devi).path : "",
1755                  blkstart, blklen);
1756             return -EIO;
1757         }
1758         trace_f2fs_issue_reset_zone(bdev, blkstart);
1759         return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1760                     sector, nr_sects, GFP_NOFS);
1761     }
1762
1763     /* For conventional zones, use regular discard if supported */
1764     return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1765 }
1766 #endif
1767
1768 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1769         struct block_device *bdev, block_t blkstart, block_t blklen)
1770 {
1771 #ifdef CONFIG_BLK_DEV_ZONED
1772     if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1773         return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1774 #endif
1775     return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1776 }
1777
1778 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1779                 block_t blkstart, block_t blklen)
1780 {
1781     sector_t start = blkstart, len = 0;
1782     struct block_device *bdev;
1783     struct seg_entry *se;
1784     unsigned int offset;
1785     block_t i;
1786     int err = 0;
1787
1788     bdev = f2fs_target_device(sbi, blkstart, NULL);
1789
1790     for (i = blkstart; i < blkstart + blklen; i++, len++) {
1791         if (i != start) {
1792             struct block_device *bdev2 =
1793                 f2fs_target_device(sbi, i, NULL);
1794
1795             if (bdev2 != bdev) {
1796                 err = __issue_discard_async(sbi, bdev,
1797                         start, len);
1798                 if (err)
1799                     return err;
1800                 bdev = bdev2;
1801                 start = i;
1802                 len = 0;
1803             }
1804         }
1805
1806         se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1807         offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1808
1809         if (f2fs_block_unit_discard(sbi) &&
1810                 !f2fs_test_and_set_bit(offset, se->discard_map))
1811             sbi->discard_blks--;
1812     }
1813
1814     if (len)
1815         err = __issue_discard_async(sbi, bdev, start, len);
1816     return err;
1817 }
1818
1819 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1820                             bool check_only)
1821 {
1822     int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1823     int max_blocks = sbi->blocks_per_seg;
1824     struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1825     unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1826     unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1827     unsigned long *discard_map = (unsigned long *)se->discard_map;
1828     unsigned long *dmap = SIT_I(sbi)->tmp_map;
1829     unsigned int start = 0, end = -1;
1830     bool force = (cpc->reason & CP_DISCARD);
1831     struct discard_entry *de = NULL;
1832     struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1833     int i;
1834
1835     if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1836             !f2fs_block_unit_discard(sbi))
1837         return false;
1838
1839     if (!force) {
1840         if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1841             SM_I(sbi)->dcc_info->nr_discards >=
1842                 SM_I(sbi)->dcc_info->max_discards)
1843             return false;
1844     }
1845
1846     /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1847     for (i = 0; i < entries; i++)
1848         dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1849                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1850
1851     while (force || SM_I(sbi)->dcc_info->nr_discards <=
1852                 SM_I(sbi)->dcc_info->max_discards) {
1853         start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1854         if (start >= max_blocks)
1855             break;
1856
1857         end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1858         if (force && start && end != max_blocks
1859                     && (end - start) < cpc->trim_minlen)
1860             continue;
1861
1862         if (check_only)
1863             return true;
1864
1865         if (!de) {
1866             de = f2fs_kmem_cache_alloc(discard_entry_slab,
1867                         GFP_F2FS_ZERO, true, NULL);
1868             de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1869             list_add_tail(&de->list, head);
1870         }
1871
1872         for (i = start; i < end; i++)
1873             __set_bit_le(i, (void *)de->discard_map);
1874
1875         SM_I(sbi)->dcc_info->nr_discards += end - start;
1876     }
1877     return false;
1878 }
1879
1880 static void release_discard_addr(struct discard_entry *entry)
1881 {
1882     list_del(&entry->list);
1883     kmem_cache_free(discard_entry_slab, entry);
1884 }
1885
1886 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1887 {
1888     struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1889     struct discard_entry *entry, *this;
1890
1891     /* drop caches */
1892     list_for_each_entry_safe(entry, this, head, list)
1893         release_discard_addr(entry);
1894 }
1895
1896 /*
1897  * Should call f2fs_clear_prefree_segments after checkpoint is done.
1898  */
1899 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1900 {
1901     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1902     unsigned int segno;
1903
1904     mutex_lock(&dirty_i->seglist_lock);
1905     for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1906         __set_test_and_free(sbi, segno, false);
1907     mutex_unlock(&dirty_i->seglist_lock);
1908 }
1909
1910 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1911                         struct cp_control *cpc)
1912 {
1913     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1914     struct list_head *head = &dcc->entry_list;
1915     struct discard_entry *entry, *this;
1916     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1917     unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
1918     unsigned int start = 0, end = -1;
1919     unsigned int secno, start_segno;
1920     bool force = (cpc->reason & CP_DISCARD);
1921     bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
1922                         DISCARD_UNIT_SECTION;
1923
1924     if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
1925         section_alignment = true;
1926
1927     mutex_lock(&dirty_i->seglist_lock);
1928
1929     while (1) {
1930         int i;
1931
1932         if (section_alignment && end != -1)
1933             end--;
1934         start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
1935         if (start >= MAIN_SEGS(sbi))
1936             break;
1937         end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
1938                                 start + 1);
1939
1940         if (section_alignment) {
1941             start = rounddown(start, sbi->segs_per_sec);
1942             end = roundup(end, sbi->segs_per_sec);
1943         }
1944
1945         for (i = start; i < end; i++) {
1946             if (test_and_clear_bit(i, prefree_map))
1947                 dirty_i->nr_dirty[PRE]--;
1948         }
1949
1950         if (!f2fs_realtime_discard_enable(sbi))
1951             continue;
1952
1953         if (force && start >= cpc->trim_start &&
1954                     (end - 1) <= cpc->trim_end)
1955                 continue;
1956
1957         if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
1958             f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
1959                 (end - start) << sbi->log_blocks_per_seg);
1960             continue;
1961         }
1962 next:
1963         secno = GET_SEC_FROM_SEG(sbi, start);
1964         start_segno = GET_SEG_FROM_SEC(sbi, secno);
1965         if (!IS_CURSEC(sbi, secno) &&
1966             !get_valid_blocks(sbi, start, true))
1967             f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
1968                 sbi->segs_per_sec << sbi->log_blocks_per_seg);
1969
1970         start = start_segno + sbi->segs_per_sec;
1971         if (start < end)
1972             goto next;
1973         else
1974             end = start - 1;
1975     }
1976     mutex_unlock(&dirty_i->seglist_lock);
1977
1978     if (!f2fs_block_unit_discard(sbi))
1979         goto wakeup;
1980
1981     /* send small discards */
1982     list_for_each_entry_safe(entry, this, head, list) {
1983         unsigned int cur_pos = 0, next_pos, len, total_len = 0;
1984         bool is_valid = test_bit_le(0, entry->discard_map);
1985
1986 find_next:
1987         if (is_valid) {
1988             next_pos = find_next_zero_bit_le(entry->discard_map,
1989                     sbi->blocks_per_seg, cur_pos);
1990             len = next_pos - cur_pos;
1991
1992             if (f2fs_sb_has_blkzoned(sbi) ||
1993                 (force && len < cpc->trim_minlen))
1994                 goto skip;
1995
1996             f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
1997                                     len);
1998             total_len += len;
1999         } else {
2000             next_pos = find_next_bit_le(entry->discard_map,
2001                     sbi->blocks_per_seg, cur_pos);
2002         }
2003 skip:
2004         cur_pos = next_pos;
2005         is_valid = !is_valid;
2006
2007         if (cur_pos < sbi->blocks_per_seg)
2008             goto find_next;
2009
2010         release_discard_addr(entry);
2011         dcc->nr_discards -= total_len;
2012     }
2013
2014 wakeup:
2015     wake_up_discard_thread(sbi, false);
2016 }
2017
2018 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2019 {
2020     dev_t dev = sbi->sb->s_bdev->bd_dev;
2021     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2022     int err = 0;
2023
2024     if (!f2fs_realtime_discard_enable(sbi))
2025         return 0;
2026
2027     dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2028                 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2029     if (IS_ERR(dcc->f2fs_issue_discard))
2030         err = PTR_ERR(dcc->f2fs_issue_discard);
2031
2032     return err;
2033 }
2034
2035 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2036 {
2037     struct discard_cmd_control *dcc;
2038     int err = 0, i;
2039
2040     if (SM_I(sbi)->dcc_info) {
2041         dcc = SM_I(sbi)->dcc_info;
2042         goto init_thread;
2043     }
2044
2045     dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2046     if (!dcc)
2047         return -ENOMEM;
2048
2049     dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2050     if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2051         dcc->discard_granularity = sbi->blocks_per_seg;
2052     else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2053         dcc->discard_granularity = BLKS_PER_SEC(sbi);
2054
2055     INIT_LIST_HEAD(&dcc->entry_list);
2056     for (i = 0; i < MAX_PLIST_NUM; i++)
2057         INIT_LIST_HEAD(&dcc->pend_list[i]);
2058     INIT_LIST_HEAD(&dcc->wait_list);
2059     INIT_LIST_HEAD(&dcc->fstrim_list);
2060     mutex_init(&dcc->cmd_lock);
2061     atomic_set(&dcc->issued_discard, 0);
2062     atomic_set(&dcc->queued_discard, 0);
2063     atomic_set(&dcc->discard_cmd_cnt, 0);
2064     dcc->nr_discards = 0;
2065     dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2066     dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2067     dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2068     dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2069     dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2070     dcc->undiscard_blks = 0;
2071     dcc->next_pos = 0;
2072     dcc->root = RB_ROOT_CACHED;
2073     dcc->rbtree_check = false;
2074
2075     init_waitqueue_head(&dcc->discard_wait_queue);
2076     SM_I(sbi)->dcc_info = dcc;
2077 init_thread:
2078     err = f2fs_start_discard_thread(sbi);
2079     if (err) {
2080         kfree(dcc);
2081         SM_I(sbi)->dcc_info = NULL;
2082     }
2083
2084     return err;
2085 }
2086
2087 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2088 {
2089     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2090
2091     if (!dcc)
2092         return;
2093
2094     f2fs_stop_discard_thread(sbi);
2095
2096     /*
2097      * Recovery can cache discard commands, so in error path of
2098      * fill_super(), it needs to give a chance to handle them.
2099      */
2100     if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2101         f2fs_issue_discard_timeout(sbi);
2102
2103     kfree(dcc);
2104     SM_I(sbi)->dcc_info = NULL;
2105 }
2106
2107 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2108 {
2109     struct sit_info *sit_i = SIT_I(sbi);
2110
2111     if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2112         sit_i->dirty_sentries++;
2113         return false;
2114     }
2115
2116     return true;
2117 }
2118
2119 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2120                     unsigned int segno, int modified)
2121 {
2122     struct seg_entry *se = get_seg_entry(sbi, segno);
2123
2124     se->type = type;
2125     if (modified)
2126         __mark_sit_entry_dirty(sbi, segno);
2127 }
2128
2129 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2130                                 block_t blkaddr)
2131 {
2132     unsigned int segno = GET_SEGNO(sbi, blkaddr);
2133
2134     if (segno == NULL_SEGNO)
2135         return 0;
2136     return get_seg_entry(sbi, segno)->mtime;
2137 }
2138
2139 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2140                         unsigned long long old_mtime)
2141 {
2142     struct seg_entry *se;
2143     unsigned int segno = GET_SEGNO(sbi, blkaddr);
2144     unsigned long long ctime = get_mtime(sbi, false);
2145     unsigned long long mtime = old_mtime ? old_mtime : ctime;
2146
2147     if (segno == NULL_SEGNO)
2148         return;
2149
2150     se = get_seg_entry(sbi, segno);
2151
2152     if (!se->mtime)
2153         se->mtime = mtime;
2154     else
2155         se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2156                         se->valid_blocks + 1);
2157
2158     if (ctime > SIT_I(sbi)->max_mtime)
2159         SIT_I(sbi)->max_mtime = ctime;
2160 }
2161
2162 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2163 {
2164     struct seg_entry *se;
2165     unsigned int segno, offset;
2166     long int new_vblocks;
2167     bool exist;
2168 #ifdef CONFIG_F2FS_CHECK_FS
2169     bool mir_exist;
2170 #endif
2171
2172     segno = GET_SEGNO(sbi, blkaddr);
2173
2174     se = get_seg_entry(sbi, segno);
2175     new_vblocks = se->valid_blocks + del;
2176     offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2177
2178     f2fs_bug_on(sbi, (new_vblocks < 0 ||
2179             (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2180
2181     se->valid_blocks = new_vblocks;
2182
2183     /* Update valid block bitmap */
2184     if (del > 0) {
2185         exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2186 #ifdef CONFIG_F2FS_CHECK_FS
2187         mir_exist = f2fs_test_and_set_bit(offset,
2188                         se->cur_valid_map_mir);
2189         if (unlikely(exist != mir_exist)) {
2190             f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2191                  blkaddr, exist);
2192             f2fs_bug_on(sbi, 1);
2193         }
2194 #endif
2195         if (unlikely(exist)) {
2196             f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2197                  blkaddr);
2198             f2fs_bug_on(sbi, 1);
2199             se->valid_blocks--;
2200             del = 0;
2201         }
2202
2203         if (f2fs_block_unit_discard(sbi) &&
2204                 !f2fs_test_and_set_bit(offset, se->discard_map))
2205             sbi->discard_blks--;
2206
2207         /*
2208          * SSR should never reuse block which is checkpointed
2209          * or newly invalidated.
2210          */
2211         if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2212             if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2213                 se->ckpt_valid_blocks++;
2214         }
2215     } else {
2216         exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2217 #ifdef CONFIG_F2FS_CHECK_FS
2218         mir_exist = f2fs_test_and_clear_bit(offset,
2219                         se->cur_valid_map_mir);
2220         if (unlikely(exist != mir_exist)) {
2221             f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2222                  blkaddr, exist);
2223             f2fs_bug_on(sbi, 1);
2224         }
2225 #endif
2226         if (unlikely(!exist)) {
2227             f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2228                  blkaddr);
2229             f2fs_bug_on(sbi, 1);
2230             se->valid_blocks++;
2231             del = 0;
2232         } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2233             /*
2234              * If checkpoints are off, we must not reuse data that
2235              * was used in the previous checkpoint. If it was used
2236              * before, we must track that to know how much space we
2237              * really have.
2238              */
2239             if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2240                 spin_lock(&sbi->stat_lock);
2241                 sbi->unusable_block_count++;
2242                 spin_unlock(&sbi->stat_lock);
2243             }
2244         }
2245
2246         if (f2fs_block_unit_discard(sbi) &&
2247             f2fs_test_and_clear_bit(offset, se->discard_map))
2248             sbi->discard_blks++;
2249     }
2250     if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2251         se->ckpt_valid_blocks += del;
2252
2253     __mark_sit_entry_dirty(sbi, segno);
2254
2255     /* update total number of valid blocks to be written in ckpt area */
2256     SIT_I(sbi)->written_valid_blocks += del;
2257
2258     if (__is_large_section(sbi))
2259         get_sec_entry(sbi, segno)->valid_blocks += del;
2260 }
2261
2262 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2263 {
2264     unsigned int segno = GET_SEGNO(sbi, addr);
2265     struct sit_info *sit_i = SIT_I(sbi);
2266
2267     f2fs_bug_on(sbi, addr == NULL_ADDR);
2268     if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2269         return;
2270
2271     invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2272     f2fs_invalidate_compress_page(sbi, addr);
2273
2274     /* add it into sit main buffer */
2275     down_write(&sit_i->sentry_lock);
2276
2277     update_segment_mtime(sbi, addr, 0);
2278     update_sit_entry(sbi, addr, -1);
2279
2280     /* add it into dirty seglist */
2281     locate_dirty_segment(sbi, segno);
2282
2283     up_write(&sit_i->sentry_lock);
2284 }
2285
2286 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2287 {
2288     struct sit_info *sit_i = SIT_I(sbi);
2289     unsigned int segno, offset;
2290     struct seg_entry *se;
2291     bool is_cp = false;
2292
2293     if (!__is_valid_data_blkaddr(blkaddr))
2294         return true;
2295
2296     down_read(&sit_i->sentry_lock);
2297
2298     segno = GET_SEGNO(sbi, blkaddr);
2299     se = get_seg_entry(sbi, segno);
2300     offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2301
2302     if (f2fs_test_bit(offset, se->ckpt_valid_map))
2303         is_cp = true;
2304
2305     up_read(&sit_i->sentry_lock);
2306
2307     return is_cp;
2308 }
2309
2310 /*
2311  * This function should be resided under the curseg_mutex lock
2312  */
2313 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2314                     struct f2fs_summary *sum)
2315 {
2316     struct curseg_info *curseg = CURSEG_I(sbi, type);
2317     void *addr = curseg->sum_blk;
2318
2319     addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2320     memcpy(addr, sum, sizeof(struct f2fs_summary));
2321 }
2322
2323 /*
2324  * Calculate the number of current summary pages for writing
2325  */
2326 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2327 {
2328     int valid_sum_count = 0;
2329     int i, sum_in_page;
2330
2331     for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2332         if (sbi->ckpt->alloc_type[i] == SSR)
2333             valid_sum_count += sbi->blocks_per_seg;
2334         else {
2335             if (for_ra)
2336                 valid_sum_count += le16_to_cpu(
2337                     F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2338             else
2339                 valid_sum_count += curseg_blkoff(sbi, i);
2340         }
2341     }
2342
2343     sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2344             SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2345     if (valid_sum_count <= sum_in_page)
2346         return 1;
2347     else if ((valid_sum_count - sum_in_page) <=
2348         (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2349         return 2;
2350     return 3;
2351 }
2352
2353 /*
2354  * Caller should put this summary page
2355  */
2356 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2357 {
2358     if (unlikely(f2fs_cp_error(sbi)))
2359         return ERR_PTR(-EIO);
2360     return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2361 }
2362
2363 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2364                     void *src, block_t blk_addr)
2365 {
2366     struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2367
2368     memcpy(page_address(page), src, PAGE_SIZE);
2369     set_page_dirty(page);
2370     f2fs_put_page(page, 1);
2371 }
2372
2373 static void write_sum_page(struct f2fs_sb_info *sbi,
2374             struct f2fs_summary_block *sum_blk, block_t blk_addr)
2375 {
2376     f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2377 }
2378
2379 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2380                         int type, block_t blk_addr)
2381 {
2382     struct curseg_info *curseg = CURSEG_I(sbi, type);
2383     struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2384     struct f2fs_summary_block *src = curseg->sum_blk;
2385     struct f2fs_summary_block *dst;
2386
2387     dst = (struct f2fs_summary_block *)page_address(page);
2388     memset(dst, 0, PAGE_SIZE);
2389
2390     mutex_lock(&curseg->curseg_mutex);
2391
2392     down_read(&curseg->journal_rwsem);
2393     memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2394     up_read(&curseg->journal_rwsem);
2395
2396     memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2397     memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2398
2399     mutex_unlock(&curseg->curseg_mutex);
2400
2401     set_page_dirty(page);
2402     f2fs_put_page(page, 1);
2403 }
2404
2405 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2406                 struct curseg_info *curseg, int type)
2407 {
2408     unsigned int segno = curseg->segno + 1;
2409     struct free_segmap_info *free_i = FREE_I(sbi);
2410
2411     if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2412         return !test_bit(segno, free_i->free_segmap);
2413     return 0;
2414 }
2415
2416 /*
2417  * Find a new segment from the free segments bitmap to right order
2418  * This function should be returned with success, otherwise BUG
2419  */
2420 static void get_new_segment(struct f2fs_sb_info *sbi,
2421             unsigned int *newseg, bool new_sec, int dir)
2422 {
2423     struct free_segmap_info *free_i = FREE_I(sbi);
2424     unsigned int segno, secno, zoneno;
2425     unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2426     unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2427     unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2428     unsigned int left_start = hint;
2429     bool init = true;
2430     int go_left = 0;
2431     int i;
2432
2433     spin_lock(&free_i->segmap_lock);
2434
2435     if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2436         segno = find_next_zero_bit(free_i->free_segmap,
2437             GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2438         if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2439             goto got_it;
2440     }
2441 find_other_zone:
2442     secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2443     if (secno >= MAIN_SECS(sbi)) {
2444         if (dir == ALLOC_RIGHT) {
2445             secno = find_first_zero_bit(free_i->free_secmap,
2446                             MAIN_SECS(sbi));
2447             f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2448         } else {
2449             go_left = 1;
2450             left_start = hint - 1;
2451         }
2452     }
2453     if (go_left == 0)
2454         goto skip_left;
2455
2456     while (test_bit(left_start, free_i->free_secmap)) {
2457         if (left_start > 0) {
2458             left_start--;
2459             continue;
2460         }
2461         left_start = find_first_zero_bit(free_i->free_secmap,
2462                             MAIN_SECS(sbi));
2463         f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2464         break;
2465     }
2466     secno = left_start;
2467 skip_left:
2468     segno = GET_SEG_FROM_SEC(sbi, secno);
2469     zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2470
2471     /* give up on finding another zone */
2472     if (!init)
2473         goto got_it;
2474     if (sbi->secs_per_zone == 1)
2475         goto got_it;
2476     if (zoneno == old_zoneno)
2477         goto got_it;
2478     if (dir == ALLOC_LEFT) {
2479         if (!go_left && zoneno + 1 >= total_zones)
2480             goto got_it;
2481         if (go_left && zoneno == 0)
2482             goto got_it;
2483     }
2484     for (i = 0; i < NR_CURSEG_TYPE; i++)
2485         if (CURSEG_I(sbi, i)->zone == zoneno)
2486             break;
2487
2488     if (i < NR_CURSEG_TYPE) {
2489         /* zone is in user, try another */
2490         if (go_left)
2491             hint = zoneno * sbi->secs_per_zone - 1;
2492         else if (zoneno + 1 >= total_zones)
2493             hint = 0;
2494         else
2495             hint = (zoneno + 1) * sbi->secs_per_zone;
2496         init = false;
2497         goto find_other_zone;
2498     }
2499 got_it:
2500     /* set it as dirty segment in free segmap */
2501     f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2502     __set_inuse(sbi, segno);
2503     *newseg = segno;
2504     spin_unlock(&free_i->segmap_lock);
2505 }
2506
2507 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2508 {
2509     struct curseg_info *curseg = CURSEG_I(sbi, type);
2510     struct summary_footer *sum_footer;
2511     unsigned short seg_type = curseg->seg_type;
2512
2513     curseg->inited = true;
2514     curseg->segno = curseg->next_segno;
2515     curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2516     curseg->next_blkoff = 0;
2517     curseg->next_segno = NULL_SEGNO;
2518
2519     sum_footer = &(curseg->sum_blk->footer);
2520     memset(sum_footer, 0, sizeof(struct summary_footer));
2521
2522     sanity_check_seg_type(sbi, seg_type);
2523
2524     if (IS_DATASEG(seg_type))
2525         SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2526     if (IS_NODESEG(seg_type))
2527         SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2528     __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2529 }
2530
2531 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2532 {
2533     struct curseg_info *curseg = CURSEG_I(sbi, type);
2534     unsigned short seg_type = curseg->seg_type;
2535
2536     sanity_check_seg_type(sbi, seg_type);
2537     if (f2fs_need_rand_seg(sbi))
2538         return prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec);
2539
2540     /* if segs_per_sec is large than 1, we need to keep original policy. */
2541     if (__is_large_section(sbi))
2542         return curseg->segno;
2543
2544     /* inmem log may not locate on any segment after mount */
2545     if (!curseg->inited)
2546         return 0;
2547
2548     if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2549         return 0;
2550
2551     if (test_opt(sbi, NOHEAP) &&
2552         (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2553         return 0;
2554
2555     if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2556         return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2557
2558     /* find segments from 0 to reuse freed segments */
2559     if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2560         return 0;
2561
2562     return curseg->segno;
2563 }
2564
2565 /*
2566  * Allocate a current working segment.
2567  * This function always allocates a free segment in LFS manner.
2568  */
2569 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2570 {
2571     struct curseg_info *curseg = CURSEG_I(sbi, type);
2572     unsigned short seg_type = curseg->seg_type;
2573     unsigned int segno = curseg->segno;
2574     int dir = ALLOC_LEFT;
2575
2576     if (curseg->inited)
2577         write_sum_page(sbi, curseg->sum_blk,
2578                 GET_SUM_BLOCK(sbi, segno));
2579     if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2580         dir = ALLOC_RIGHT;
2581
2582     if (test_opt(sbi, NOHEAP))
2583         dir = ALLOC_RIGHT;
2584
2585     segno = __get_next_segno(sbi, type);
2586     get_new_segment(sbi, &segno, new_sec, dir);
2587     curseg->next_segno = segno;
2588     reset_curseg(sbi, type, 1);
2589     curseg->alloc_type = LFS;
2590     if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2591         curseg->fragment_remained_chunk =
2592                 prandom_u32() % sbi->max_fragment_chunk + 1;
2593 }
2594
2595 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2596                     int segno, block_t start)
2597 {
2598     struct seg_entry *se = get_seg_entry(sbi, segno);
2599     int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2600     unsigned long *target_map = SIT_I(sbi)->tmp_map;
2601     unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2602     unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2603     int i;
2604
2605     for (i = 0; i < entries; i++)
2606         target_map[i] = ckpt_map[i] | cur_map[i];
2607
2608     return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2609 }
2610
2611 /*
2612  * If a segment is written by LFS manner, next block offset is just obtained
2613  * by increasing the current block offset. However, if a segment is written by
2614  * SSR manner, next block offset obtained by calling __next_free_blkoff
2615  */
2616 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2617                 struct curseg_info *seg)
2618 {
2619     if (seg->alloc_type == SSR) {
2620         seg->next_blkoff =
2621             __next_free_blkoff(sbi, seg->segno,
2622                         seg->next_blkoff + 1);
2623     } else {
2624         seg->next_blkoff++;
2625         if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) {
2626             /* To allocate block chunks in different sizes, use random number */
2627             if (--seg->fragment_remained_chunk <= 0) {
2628                 seg->fragment_remained_chunk =
2629                    prandom_u32() % sbi->max_fragment_chunk + 1;
2630                 seg->next_blkoff +=
2631                    prandom_u32() % sbi->max_fragment_hole + 1;
2632             }
2633         }
2634     }
2635 }
2636
2637 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2638 {
2639     return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2640 }
2641
2642 /*
2643  * This function always allocates a used segment(from dirty seglist) by SSR
2644  * manner, so it should recover the existing segment information of valid blocks
2645  */
2646 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2647 {
2648     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2649     struct curseg_info *curseg = CURSEG_I(sbi, type);
2650     unsigned int new_segno = curseg->next_segno;
2651     struct f2fs_summary_block *sum_node;
2652     struct page *sum_page;
2653
2654     if (flush)
2655         write_sum_page(sbi, curseg->sum_blk,
2656                     GET_SUM_BLOCK(sbi, curseg->segno));
2657
2658     __set_test_and_inuse(sbi, new_segno);
2659
2660     mutex_lock(&dirty_i->seglist_lock);
2661     __remove_dirty_segment(sbi, new_segno, PRE);
2662     __remove_dirty_segment(sbi, new_segno, DIRTY);
2663     mutex_unlock(&dirty_i->seglist_lock);
2664
2665     reset_curseg(sbi, type, 1);
2666     curseg->alloc_type = SSR;
2667     curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2668
2669     sum_page = f2fs_get_sum_page(sbi, new_segno);
2670     if (IS_ERR(sum_page)) {
2671         /* GC won't be able to use stale summary pages by cp_error */
2672         memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2673         return;
2674     }
2675     sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2676     memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2677     f2fs_put_page(sum_page, 1);
2678 }
2679
2680 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2681                 int alloc_mode, unsigned long long age);
2682
2683 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2684                     int target_type, int alloc_mode,
2685                     unsigned long long age)
2686 {
2687     struct curseg_info *curseg = CURSEG_I(sbi, type);
2688
2689     curseg->seg_type = target_type;
2690
2691     if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2692         struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2693
2694         curseg->seg_type = se->type;
2695         change_curseg(sbi, type, true);
2696     } else {
2697         /* allocate cold segment by default */
2698         curseg->seg_type = CURSEG_COLD_DATA;
2699         new_curseg(sbi, type, true);
2700     }
2701     stat_inc_seg_type(sbi, curseg);
2702 }
2703
2704 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2705 {
2706     struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2707
2708     if (!sbi->am.atgc_enabled)
2709         return;
2710
2711     f2fs_down_read(&SM_I(sbi)->curseg_lock);
2712
2713     mutex_lock(&curseg->curseg_mutex);
2714     down_write(&SIT_I(sbi)->sentry_lock);
2715
2716     get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2717
2718     up_write(&SIT_I(sbi)->sentry_lock);
2719     mutex_unlock(&curseg->curseg_mutex);
2720
2721     f2fs_up_read(&SM_I(sbi)->curseg_lock);
2722
2723 }
2724 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2725 {
2726     __f2fs_init_atgc_curseg(sbi);
2727 }
2728
2729 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2730 {
2731     struct curseg_info *curseg = CURSEG_I(sbi, type);
2732
2733     mutex_lock(&curseg->curseg_mutex);
2734     if (!curseg->inited)
2735         goto out;
2736
2737     if (get_valid_blocks(sbi, curseg->segno, false)) {
2738         write_sum_page(sbi, curseg->sum_blk,
2739                 GET_SUM_BLOCK(sbi, curseg->segno));
2740     } else {
2741         mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2742         __set_test_and_free(sbi, curseg->segno, true);
2743         mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2744     }
2745 out:
2746     mutex_unlock(&curseg->curseg_mutex);
2747 }
2748
2749 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2750 {
2751     __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2752
2753     if (sbi->am.atgc_enabled)
2754         __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2755 }
2756
2757 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2758 {
2759     struct curseg_info *curseg = CURSEG_I(sbi, type);
2760
2761     mutex_lock(&curseg->curseg_mutex);
2762     if (!curseg->inited)
2763         goto out;
2764     if (get_valid_blocks(sbi, curseg->segno, false))
2765         goto out;
2766
2767     mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2768     __set_test_and_inuse(sbi, curseg->segno);
2769     mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2770 out:
2771     mutex_unlock(&curseg->curseg_mutex);
2772 }
2773
2774 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2775 {
2776     __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2777
2778     if (sbi->am.atgc_enabled)
2779         __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2780 }
2781
2782 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2783                 int alloc_mode, unsigned long long age)
2784 {
2785     struct curseg_info *curseg = CURSEG_I(sbi, type);
2786     const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2787     unsigned segno = NULL_SEGNO;
2788     unsigned short seg_type = curseg->seg_type;
2789     int i, cnt;
2790     bool reversed = false;
2791
2792     sanity_check_seg_type(sbi, seg_type);
2793
2794     /* f2fs_need_SSR() already forces to do this */
2795     if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2796         curseg->next_segno = segno;
2797         return 1;
2798     }
2799
2800     /* For node segments, let's do SSR more intensively */
2801     if (IS_NODESEG(seg_type)) {
2802         if (seg_type >= CURSEG_WARM_NODE) {
2803             reversed = true;
2804             i = CURSEG_COLD_NODE;
2805         } else {
2806             i = CURSEG_HOT_NODE;
2807         }
2808         cnt = NR_CURSEG_NODE_TYPE;
2809     } else {
2810         if (seg_type >= CURSEG_WARM_DATA) {
2811             reversed = true;
2812             i = CURSEG_COLD_DATA;
2813         } else {
2814             i = CURSEG_HOT_DATA;
2815         }
2816         cnt = NR_CURSEG_DATA_TYPE;
2817     }
2818
2819     for (; cnt-- > 0; reversed ? i-- : i++) {
2820         if (i == seg_type)
2821             continue;
2822         if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2823             curseg->next_segno = segno;
2824             return 1;
2825         }
2826     }
2827
2828     /* find valid_blocks=0 in dirty list */
2829     if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2830         segno = get_free_segment(sbi);
2831         if (segno != NULL_SEGNO) {
2832             curseg->next_segno = segno;
2833             return 1;
2834         }
2835     }
2836     return 0;
2837 }
2838
2839 /*
2840  * flush out current segment and replace it with new segment
2841  * This function should be returned with success, otherwise BUG
2842  */
2843 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2844                         int type, bool force)
2845 {
2846     struct curseg_info *curseg = CURSEG_I(sbi, type);
2847
2848     if (force)
2849         new_curseg(sbi, type, true);
2850     else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2851                     curseg->seg_type == CURSEG_WARM_NODE)
2852         new_curseg(sbi, type, false);
2853     else if (curseg->alloc_type == LFS &&
2854             is_next_segment_free(sbi, curseg, type) &&
2855             likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2856         new_curseg(sbi, type, false);
2857     else if (f2fs_need_SSR(sbi) &&
2858             get_ssr_segment(sbi, type, SSR, 0))
2859         change_curseg(sbi, type, true);
2860     else
2861         new_curseg(sbi, type, false);
2862
2863     stat_inc_seg_type(sbi, curseg);
2864 }
2865
2866 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2867                     unsigned int start, unsigned int end)
2868 {
2869     struct curseg_info *curseg = CURSEG_I(sbi, type);
2870     unsigned int segno;
2871
2872     f2fs_down_read(&SM_I(sbi)->curseg_lock);
2873     mutex_lock(&curseg->curseg_mutex);
2874     down_write(&SIT_I(sbi)->sentry_lock);
2875
2876     segno = CURSEG_I(sbi, type)->segno;
2877     if (segno < start || segno > end)
2878         goto unlock;
2879
2880     if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2881         change_curseg(sbi, type, true);
2882     else
2883         new_curseg(sbi, type, true);
2884
2885     stat_inc_seg_type(sbi, curseg);
2886
2887     locate_dirty_segment(sbi, segno);
2888 unlock:
2889     up_write(&SIT_I(sbi)->sentry_lock);
2890
2891     if (segno != curseg->segno)
2892         f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2893                 type, segno, curseg->segno);
2894
2895     mutex_unlock(&curseg->curseg_mutex);
2896     f2fs_up_read(&SM_I(sbi)->curseg_lock);
2897 }
2898
2899 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
2900                         bool new_sec, bool force)
2901 {
2902     struct curseg_info *curseg = CURSEG_I(sbi, type);
2903     unsigned int old_segno;
2904
2905     if (!curseg->inited)
2906         goto alloc;
2907
2908     if (force || curseg->next_blkoff ||
2909         get_valid_blocks(sbi, curseg->segno, new_sec))
2910         goto alloc;
2911
2912     if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
2913         return;
2914 alloc:
2915     old_segno = curseg->segno;
2916     SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
2917     locate_dirty_segment(sbi, old_segno);
2918 }
2919
2920 static void __allocate_new_section(struct f2fs_sb_info *sbi,
2921                         int type, bool force)
2922 {
2923     __allocate_new_segment(sbi, type, true, force);
2924 }
2925
2926 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
2927 {
2928     f2fs_down_read(&SM_I(sbi)->curseg_lock);
2929     down_write(&SIT_I(sbi)->sentry_lock);
2930     __allocate_new_section(sbi, type, force);
2931     up_write(&SIT_I(sbi)->sentry_lock);
2932     f2fs_up_read(&SM_I(sbi)->curseg_lock);
2933 }
2934
2935 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2936 {
2937     int i;
2938
2939     f2fs_down_read(&SM_I(sbi)->curseg_lock);
2940     down_write(&SIT_I(sbi)->sentry_lock);
2941     for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
2942         __allocate_new_segment(sbi, i, false, false);
2943     up_write(&SIT_I(sbi)->sentry_lock);
2944     f2fs_up_read(&SM_I(sbi)->curseg_lock);
2945 }
2946
2947 static const struct segment_allocation default_salloc_ops = {
2948     .allocate_segment = allocate_segment_by_default,
2949 };
2950
2951 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
2952                         struct cp_control *cpc)
2953 {
2954     __u64 trim_start = cpc->trim_start;
2955     bool has_candidate = false;
2956
2957     down_write(&SIT_I(sbi)->sentry_lock);
2958     for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
2959         if (add_discard_addrs(sbi, cpc, true)) {
2960             has_candidate = true;
2961             break;
2962         }
2963     }
2964     up_write(&SIT_I(sbi)->sentry_lock);
2965
2966     cpc->trim_start = trim_start;
2967     return has_candidate;
2968 }
2969
2970 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
2971                     struct discard_policy *dpolicy,
2972                     unsigned int start, unsigned int end)
2973 {
2974     struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2975     struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
2976     struct rb_node **insert_p = NULL, *insert_parent = NULL;
2977     struct discard_cmd *dc;
2978     struct blk_plug plug;
2979     int issued;
2980     unsigned int trimmed = 0;
2981
2982 next:
2983     issued = 0;
2984
2985     mutex_lock(&dcc->cmd_lock);
2986     if (unlikely(dcc->rbtree_check))
2987         f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
2988                             &dcc->root, false));
2989
2990     dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
2991                     NULL, start,
2992                     (struct rb_entry **)&prev_dc,
2993                     (struct rb_entry **)&next_dc,
2994                     &insert_p, &insert_parent, true, NULL);
2995     if (!dc)
2996         dc = next_dc;
2997
2998     blk_start_plug(&plug);
2999
3000     while (dc && dc->lstart <= end) {
3001         struct rb_node *node;
3002         int err = 0;
3003
3004         if (dc->len < dpolicy->granularity)
3005             goto skip;
3006
3007         if (dc->state != D_PREP) {
3008             list_move_tail(&dc->list, &dcc->fstrim_list);
3009             goto skip;
3010         }
3011
3012         err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3013
3014         if (issued >= dpolicy->max_requests) {
3015             start = dc->lstart + dc->len;
3016
3017             if (err)
3018                 __remove_discard_cmd(sbi, dc);
3019
3020             blk_finish_plug(&plug);
3021             mutex_unlock(&dcc->cmd_lock);
3022             trimmed += __wait_all_discard_cmd(sbi, NULL);
3023             f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3024             goto next;
3025         }
3026 skip:
3027         node = rb_next(&dc->rb_node);
3028         if (err)
3029             __remove_discard_cmd(sbi, dc);
3030         dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3031
3032         if (fatal_signal_pending(current))
3033             break;
3034     }
3035
3036     blk_finish_plug(&plug);
3037     mutex_unlock(&dcc->cmd_lock);
3038
3039     return trimmed;
3040 }
3041
3042 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3043 {
3044     __u64 start = F2FS_BYTES_TO_BLK(range->start);
3045     __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3046     unsigned int start_segno, end_segno;
3047     block_t start_block, end_block;
3048     struct cp_control cpc;
3049     struct discard_policy dpolicy;
3050     unsigned long long trimmed = 0;
3051     int err = 0;
3052     bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3053
3054     if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3055         return -EINVAL;
3056
3057     if (end < MAIN_BLKADDR(sbi))
3058         goto out;
3059
3060     if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3061         f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3062         return -EFSCORRUPTED;
3063     }
3064
3065     /* start/end segment number in main_area */
3066     start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3067     end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3068                         GET_SEGNO(sbi, end);
3069     if (need_align) {
3070         start_segno = rounddown(start_segno, sbi->segs_per_sec);
3071         end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3072     }
3073
3074     cpc.reason = CP_DISCARD;
3075     cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3076     cpc.trim_start = start_segno;
3077     cpc.trim_end = end_segno;
3078
3079     if (sbi->discard_blks == 0)
3080         goto out;
3081
3082     f2fs_down_write(&sbi->gc_lock);
3083     err = f2fs_write_checkpoint(sbi, &cpc);
3084     f2fs_up_write(&sbi->gc_lock);
3085     if (err)
3086         goto out;
3087
3088     /*
3089      * We filed discard candidates, but actually we don't need to wait for
3090      * all of them, since they'll be issued in idle time along with runtime
3091      * discard option. User configuration looks like using runtime discard
3092      * or periodic fstrim instead of it.
3093      */
3094     if (f2fs_realtime_discard_enable(sbi))
3095         goto out;
3096
3097     start_block = START_BLOCK(sbi, start_segno);
3098     end_block = START_BLOCK(sbi, end_segno + 1);
3099
3100     __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3101     trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3102                     start_block, end_block);
3103
3104     trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3105                     start_block, end_block);
3106 out:
3107     if (!err)
3108         range->len = F2FS_BLK_TO_BYTES(trimmed);
3109     return err;
3110 }
3111
3112 static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3113                     struct curseg_info *curseg)
3114 {
3115     return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3116                             curseg->segno);
3117 }
3118
3119 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3120 {
3121     switch (hint) {
3122     case WRITE_LIFE_SHORT:
3123         return CURSEG_HOT_DATA;
3124     case WRITE_LIFE_EXTREME:
3125         return CURSEG_COLD_DATA;
3126     default:
3127         return CURSEG_WARM_DATA;
3128     }
3129 }
3130
3131 static int __get_segment_type_2(struct f2fs_io_info *fio)
3132 {
3133     if (fio->type == DATA)
3134         return CURSEG_HOT_DATA;
3135     else
3136         return CURSEG_HOT_NODE;
3137 }
3138
3139 static int __get_segment_type_4(struct f2fs_io_info *fio)
3140 {
3141     if (fio->type == DATA) {
3142         struct inode *inode = fio->page->mapping->host;
3143
3144         if (S_ISDIR(inode->i_mode))
3145             return CURSEG_HOT_DATA;
3146         else
3147             return CURSEG_COLD_DATA;
3148     } else {
3149         if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3150             return CURSEG_WARM_NODE;
3151         else
3152             return CURSEG_COLD_NODE;
3153     }
3154 }
3155
3156 static int __get_segment_type_6(struct f2fs_io_info *fio)
3157 {
3158     if (fio->type == DATA) {
3159         struct inode *inode = fio->page->mapping->host;
3160
3161         if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3162             return CURSEG_COLD_DATA_PINNED;
3163
3164         if (page_private_gcing(fio->page)) {
3165             if (fio->sbi->am.atgc_enabled &&
3166                 (fio->io_type == FS_DATA_IO) &&
3167                 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3168                 return CURSEG_ALL_DATA_ATGC;
3169             else
3170                 return CURSEG_COLD_DATA;
3171         }
3172         if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3173             return CURSEG_COLD_DATA;
3174         if (file_is_hot(inode) ||
3175                 is_inode_flag_set(inode, FI_HOT_DATA) ||
3176                 f2fs_is_cow_file(inode))
3177             return CURSEG_HOT_DATA;
3178         return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3179     } else {
3180         if (IS_DNODE(fio->page))
3181             return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3182                         CURSEG_HOT_NODE;
3183         return CURSEG_COLD_NODE;
3184     }
3185 }
3186
3187 static int __get_segment_type(struct f2fs_io_info *fio)
3188 {
3189     int type = 0;
3190
3191     switch (F2FS_OPTION(fio->sbi).active_logs) {
3192     case 2:
3193         type = __get_segment_type_2(fio);
3194         break;
3195     case 4:
3196         type = __get_segment_type_4(fio);
3197         break;
3198     case 6:
3199         type = __get_segment_type_6(fio);
3200         break;
3201     default:
3202         f2fs_bug_on(fio->sbi, true);
3203     }
3204
3205     if (IS_HOT(type))
3206         fio->temp = HOT;
3207     else if (IS_WARM(type))
3208         fio->temp = WARM;
3209     else
3210         fio->temp = COLD;
3211     return type;
3212 }
3213
3214 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3215         block_t old_blkaddr, block_t *new_blkaddr,
3216         struct f2fs_summary *sum, int type,
3217         struct f2fs_io_info *fio)
3218 {
3219     struct sit_info *sit_i = SIT_I(sbi);
3220     struct curseg_info *curseg = CURSEG_I(sbi, type);
3221     unsigned long long old_mtime;
3222     bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3223     struct seg_entry *se = NULL;
3224
3225     f2fs_down_read(&SM_I(sbi)->curseg_lock);
3226
3227     mutex_lock(&curseg->curseg_mutex);
3228     down_write(&sit_i->sentry_lock);
3229
3230     if (from_gc) {
3231         f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3232         se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3233         sanity_check_seg_type(sbi, se->type);
3234         f2fs_bug_on(sbi, IS_NODESEG(se->type));
3235     }
3236     *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3237
3238     f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3239
3240     f2fs_wait_discard_bio(sbi, *new_blkaddr);
3241
3242     /*
3243      * __add_sum_entry should be resided under the curseg_mutex
3244      * because, this function updates a summary entry in the
3245      * current summary block.
3246      */
3247     __add_sum_entry(sbi, type, sum);
3248
3249     __refresh_next_blkoff(sbi, curseg);
3250
3251     stat_inc_block_count(sbi, curseg);
3252
3253     if (from_gc) {
3254         old_mtime = get_segment_mtime(sbi, old_blkaddr);
3255     } else {
3256         update_segment_mtime(sbi, old_blkaddr, 0);
3257         old_mtime = 0;
3258     }
3259     update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3260
3261     /*
3262      * SIT information should be updated before segment allocation,
3263      * since SSR needs latest valid block information.
3264      */
3265     update_sit_entry(sbi, *new_blkaddr, 1);
3266     if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3267         update_sit_entry(sbi, old_blkaddr, -1);
3268
3269     if (!__has_curseg_space(sbi, curseg)) {
3270         if (from_gc)
3271             get_atssr_segment(sbi, type, se->type,
3272                         AT_SSR, se->mtime);
3273         else
3274             sit_i->s_ops->allocate_segment(sbi, type, false);
3275     }
3276     /*
3277      * segment dirty status should be updated after segment allocation,
3278      * so we just need to update status only one time after previous
3279      * segment being closed.
3280      */
3281     locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3282     locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3283
3284     up_write(&sit_i->sentry_lock);
3285
3286     if (page && IS_NODESEG(type)) {
3287         fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3288
3289         f2fs_inode_chksum_set(sbi, page);
3290     }
3291
3292     if (fio) {
3293         struct f2fs_bio_info *io;
3294
3295         if (F2FS_IO_ALIGNED(sbi))
3296             fio->retry = false;
3297
3298         INIT_LIST_HEAD(&fio->list);
3299         fio->in_list = true;
3300         io = sbi->write_io[fio->type] + fio->temp;
3301         spin_lock(&io->io_lock);
3302         list_add_tail(&fio->list, &io->io_list);
3303         spin_unlock(&io->io_lock);
3304     }
3305
3306     mutex_unlock(&curseg->curseg_mutex);
3307
3308     f2fs_up_read(&SM_I(sbi)->curseg_lock);
3309 }
3310
3311 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3312                     block_t blkaddr, unsigned int blkcnt)
3313 {
3314     if (!f2fs_is_multi_device(sbi))
3315         return;
3316
3317     while (1) {
3318         unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3319         unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3320
3321         /* update device state for fsync */
3322         f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3323
3324         /* update device state for checkpoint */
3325         if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3326             spin_lock(&sbi->dev_lock);
3327             f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3328             spin_unlock(&sbi->dev_lock);
3329         }
3330
3331         if (blkcnt <= blks)
3332             break;
3333         blkcnt -= blks;
3334         blkaddr += blks;
3335     }
3336 }
3337
3338 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3339 {
3340     int type = __get_segment_type(fio);
3341     bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3342
3343     if (keep_order)
3344         f2fs_down_read(&fio->sbi->io_order_lock);
3345 reallocate:
3346     f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3347             &fio->new_blkaddr, sum, type, fio);
3348     if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3349         invalidate_mapping_pages(META_MAPPING(fio->sbi),
3350                     fio->old_blkaddr, fio->old_blkaddr);
3351         f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3352     }
3353
3354     /* writeout dirty page into bdev */
3355     f2fs_submit_page_write(fio);
3356     if (fio->retry) {
3357         fio->old_blkaddr = fio->new_blkaddr;
3358         goto reallocate;
3359     }
3360
3361     f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3362
3363     if (keep_order)
3364         f2fs_up_read(&fio->sbi->io_order_lock);
3365 }
3366
3367 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3368                     enum iostat_type io_type)
3369 {
3370     struct f2fs_io_info fio = {
3371         .sbi = sbi,
3372         .type = META,
3373         .temp = HOT,
3374         .op = REQ_OP_WRITE,
3375         .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3376         .old_blkaddr = page->index,
3377         .new_blkaddr = page->index,
3378         .page = page,
3379         .encrypted_page = NULL,
3380         .in_list = false,
3381     };
3382
3383     if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3384         fio.op_flags &= ~REQ_META;
3385
3386     set_page_writeback(page);
3387     ClearPageError(page);
3388     f2fs_submit_page_write(&fio);
3389
3390     stat_inc_meta_count(sbi, page->index);
3391     f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3392 }
3393
3394 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3395 {
3396     struct f2fs_summary sum;
3397
3398     set_summary(&sum, nid, 0, 0);
3399     do_write_page(&sum, fio);
3400
3401     f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3402 }
3403
3404 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3405                     struct f2fs_io_info *fio)
3406 {
3407     struct f2fs_sb_info *sbi = fio->sbi;
3408     struct f2fs_summary sum;
3409
3410     f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3411     set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3412     do_write_page(&sum, fio);
3413     f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3414
3415     f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3416 }
3417
3418 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3419 {
3420     int err;
3421     struct f2fs_sb_info *sbi = fio->sbi;
3422     unsigned int segno;
3423
3424     fio->new_blkaddr = fio->old_blkaddr;
3425     /* i/o temperature is needed for passing down write hints */
3426     __get_segment_type(fio);
3427
3428     segno = GET_SEGNO(sbi, fio->new_blkaddr);
3429
3430     if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3431         set_sbi_flag(sbi, SBI_NEED_FSCK);
3432         f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3433               __func__, segno);
3434         err = -EFSCORRUPTED;
3435         goto drop_bio;
3436     }
3437
3438     if (f2fs_cp_error(sbi)) {
3439         err = -EIO;
3440         goto drop_bio;
3441     }
3442
3443     if (fio->post_read)
3444         invalidate_mapping_pages(META_MAPPING(sbi),
3445                 fio->new_blkaddr, fio->new_blkaddr);
3446
3447     stat_inc_inplace_blocks(fio->sbi);
3448
3449     if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3450         err = f2fs_merge_page_bio(fio);
3451     else
3452         err = f2fs_submit_page_bio(fio);
3453     if (!err) {
3454         f2fs_update_device_state(fio->sbi, fio->ino,
3455                         fio->new_blkaddr, 1);
3456         f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3457     }
3458
3459     return err;
3460 drop_bio:
3461     if (fio->bio && *(fio->bio)) {
3462         struct bio *bio = *(fio->bio);
3463
3464         bio->bi_status = BLK_STS_IOERR;
3465         bio_endio(bio);
3466         *(fio->bio) = NULL;
3467     }
3468     return err;
3469 }
3470
3471 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3472                         unsigned int segno)
3473 {
3474     int i;
3475
3476     for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3477         if (CURSEG_I(sbi, i)->segno == segno)
3478             break;
3479     }
3480     return i;
3481 }
3482
3483 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3484                 block_t old_blkaddr, block_t new_blkaddr,
3485                 bool recover_curseg, bool recover_newaddr,
3486                 bool from_gc)
3487 {
3488     struct sit_info *sit_i = SIT_I(sbi);
3489     struct curseg_info *curseg;
3490     unsigned int segno, old_cursegno;
3491     struct seg_entry *se;
3492     int type;
3493     unsigned short old_blkoff;
3494     unsigned char old_alloc_type;
3495
3496     segno = GET_SEGNO(sbi, new_blkaddr);
3497     se = get_seg_entry(sbi, segno);
3498     type = se->type;
3499
3500     f2fs_down_write(&SM_I(sbi)->curseg_lock);
3501
3502     if (!recover_curseg) {
3503         /* for recovery flow */
3504         if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3505             if (old_blkaddr == NULL_ADDR)
3506                 type = CURSEG_COLD_DATA;
3507             else
3508                 type = CURSEG_WARM_DATA;
3509         }
3510     } else {
3511         if (IS_CURSEG(sbi, segno)) {
3512             /* se->type is volatile as SSR allocation */
3513             type = __f2fs_get_curseg(sbi, segno);
3514             f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3515         } else {
3516             type = CURSEG_WARM_DATA;
3517         }
3518     }
3519
3520     f2fs_bug_on(sbi, !IS_DATASEG(type));
3521     curseg = CURSEG_I(sbi, type);
3522
3523     mutex_lock(&curseg->curseg_mutex);
3524     down_write(&sit_i->sentry_lock);
3525
3526     old_cursegno = curseg->segno;
3527     old_blkoff = curseg->next_blkoff;
3528     old_alloc_type = curseg->alloc_type;
3529
3530     /* change the current segment */
3531     if (segno != curseg->segno) {
3532         curseg->next_segno = segno;
3533         change_curseg(sbi, type, true);
3534     }
3535
3536     curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3537     __add_sum_entry(sbi, type, sum);
3538
3539     if (!recover_curseg || recover_newaddr) {
3540         if (!from_gc)
3541             update_segment_mtime(sbi, new_blkaddr, 0);
3542         update_sit_entry(sbi, new_blkaddr, 1);
3543     }
3544     if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3545         invalidate_mapping_pages(META_MAPPING(sbi),
3546                     old_blkaddr, old_blkaddr);
3547         f2fs_invalidate_compress_page(sbi, old_blkaddr);
3548         if (!from_gc)
3549             update_segment_mtime(sbi, old_blkaddr, 0);
3550         update_sit_entry(sbi, old_blkaddr, -1);
3551     }
3552
3553     locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3554     locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3555
3556     locate_dirty_segment(sbi, old_cursegno);
3557
3558     if (recover_curseg) {
3559         if (old_cursegno != curseg->segno) {
3560             curseg->next_segno = old_cursegno;
3561             change_curseg(sbi, type, true);
3562         }
3563         curseg->next_blkoff = old_blkoff;
3564         curseg->alloc_type = old_alloc_type;
3565     }
3566
3567     up_write(&sit_i->sentry_lock);
3568     mutex_unlock(&curseg->curseg_mutex);
3569     f2fs_up_write(&SM_I(sbi)->curseg_lock);
3570 }
3571
3572 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3573                 block_t old_addr, block_t new_addr,
3574                 unsigned char version, bool recover_curseg,
3575                 bool recover_newaddr)
3576 {
3577     struct f2fs_summary sum;
3578
3579     set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3580
3581     f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3582                     recover_curseg, recover_newaddr, false);
3583
3584     f2fs_update_data_blkaddr(dn, new_addr);
3585 }
3586
3587 void f2fs_wait_on_page_writeback(struct page *page,
3588                 enum page_type type, bool ordered, bool locked)
3589 {
3590     if (PageWriteback(page)) {
3591         struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3592
3593         /* submit cached LFS IO */
3594         f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3595         /* sbumit cached IPU IO */
3596         f2fs_submit_merged_ipu_write(sbi, NULL, page);
3597         if (ordered) {
3598             wait_on_page_writeback(page);
3599             f2fs_bug_on(sbi, locked && PageWriteback(page));
3600         } else {
3601             wait_for_stable_page(page);
3602         }
3603     }
3604 }
3605
3606 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3607 {
3608     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3609     struct page *cpage;
3610
3611     if (!f2fs_post_read_required(inode))
3612         return;
3613
3614     if (!__is_valid_data_blkaddr(blkaddr))
3615         return;
3616
3617     cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3618     if (cpage) {
3619         f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3620         f2fs_put_page(cpage, 1);
3621     }
3622 }
3623
3624 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3625                                 block_t len)
3626 {
3627     struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3628     block_t i;
3629
3630     if (!f2fs_post_read_required(inode))
3631         return;
3632
3633     for (i = 0; i < len; i++)
3634         f2fs_wait_on_block_writeback(inode, blkaddr + i);
3635
3636     invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);
3637 }
3638
3639 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3640 {
3641     struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3642     struct curseg_info *seg_i;
3643     unsigned char *kaddr;
3644     struct page *page;
3645     block_t start;
3646     int i, j, offset;
3647
3648     start = start_sum_block(sbi);
3649
3650     page = f2fs_get_meta_page(sbi, start++);
3651     if (IS_ERR(page))
3652         return PTR_ERR(page);
3653     kaddr = (unsigned char *)page_address(page);
3654
3655     /* Step 1: restore nat cache */
3656     seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3657     memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3658
3659     /* Step 2: restore sit cache */
3660     seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3661     memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3662     offset = 2 * SUM_JOURNAL_SIZE;
3663
3664     /* Step 3: restore summary entries */
3665     for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3666         unsigned short blk_off;
3667         unsigned int segno;
3668
3669         seg_i = CURSEG_I(sbi, i);
3670         segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3671         blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3672         seg_i->next_segno = segno;
3673         reset_curseg(sbi, i, 0);
3674         seg_i->alloc_type = ckpt->alloc_type[i];
3675         seg_i->next_blkoff = blk_off;
3676
3677         if (seg_i->alloc_type == SSR)
3678             blk_off = sbi->blocks_per_seg;
3679
3680         for (j = 0; j < blk_off; j++) {
3681             struct f2fs_summary *s;
3682
3683             s = (struct f2fs_summary *)(kaddr + offset);
3684             seg_i->sum_blk->entries[j] = *s;
3685             offset += SUMMARY_SIZE;
3686             if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3687                         SUM_FOOTER_SIZE)
3688                 continue;
3689
3690             f2fs_put_page(page, 1);
3691             page = NULL;
3692
3693             page = f2fs_get_meta_page(sbi, start++);
3694             if (IS_ERR(page))
3695                 return PTR_ERR(page);
3696             kaddr = (unsigned char *)page_address(page);
3697             offset = 0;
3698         }
3699     }
3700     f2fs_put_page(page, 1);
3701     return 0;
3702 }
3703
3704 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3705 {
3706     struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3707     struct f2fs_summary_block *sum;
3708     struct curseg_info *curseg;
3709     struct page *new;
3710     unsigned short blk_off;
3711     unsigned int segno = 0;
3712     block_t blk_addr = 0;
3713     int err = 0;
3714
3715     /* get segment number and block addr */
3716     if (IS_DATASEG(type)) {
3717         segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3718         blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3719                             CURSEG_HOT_DATA]);
3720         if (__exist_node_summaries(sbi))
3721             blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3722         else
3723             blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3724     } else {
3725         segno = le32_to_cpu(ckpt->cur_node_segno[type -
3726                             CURSEG_HOT_NODE]);
3727         blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3728                             CURSEG_HOT_NODE]);
3729         if (__exist_node_summaries(sbi))
3730             blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3731                             type - CURSEG_HOT_NODE);
3732         else
3733             blk_addr = GET_SUM_BLOCK(sbi, segno);
3734     }
3735
3736     new = f2fs_get_meta_page(sbi, blk_addr);
3737     if (IS_ERR(new))
3738         return PTR_ERR(new);
3739     sum = (struct f2fs_summary_block *)page_address(new);
3740
3741     if (IS_NODESEG(type)) {
3742         if (__exist_node_summaries(sbi)) {
3743             struct f2fs_summary *ns = &sum->entries[0];
3744             int i;
3745
3746             for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3747                 ns->version = 0;
3748                 ns->ofs_in_node = 0;
3749             }
3750         } else {
3751             err = f2fs_restore_node_summary(sbi, segno, sum);
3752             if (err)
3753                 goto out;
3754         }
3755     }
3756
3757     /* set uncompleted segment to curseg */
3758     curseg = CURSEG_I(sbi, type);
3759     mutex_lock(&curseg->curseg_mutex);
3760
3761     /* update journal info */
3762     down_write(&curseg->journal_rwsem);
3763     memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3764     up_write(&curseg->journal_rwsem);
3765
3766     memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3767     memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3768     curseg->next_segno = segno;
3769     reset_curseg(sbi, type, 0);
3770     curseg->alloc_type = ckpt->alloc_type[type];
3771     curseg->next_blkoff = blk_off;
3772     mutex_unlock(&curseg->curseg_mutex);
3773 out:
3774     f2fs_put_page(new, 1);
3775     return err;
3776 }
3777
3778 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3779 {
3780     struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3781     struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3782     int type = CURSEG_HOT_DATA;
3783     int err;
3784
3785     if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3786         int npages = f2fs_npages_for_summary_flush(sbi, true);
3787
3788         if (npages >= 2)
3789             f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3790                             META_CP, true);
3791
3792         /* restore for compacted data summary */
3793         err = read_compacted_summaries(sbi);
3794         if (err)
3795             return err;
3796         type = CURSEG_HOT_NODE;
3797     }
3798
3799     if (__exist_node_summaries(sbi))
3800         f2fs_ra_meta_pages(sbi,
3801                 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
3802                 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3803
3804     for (; type <= CURSEG_COLD_NODE; type++) {
3805         err = read_normal_summaries(sbi, type);
3806         if (err)
3807             return err;
3808     }
3809
3810     /* sanity check for summary blocks */
3811     if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3812             sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3813         f2fs_err(sbi, "invalid journal entries nats %u sits %u",
3814              nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3815         return -EINVAL;
3816     }
3817
3818     return 0;
3819 }
3820
3821 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3822 {
3823     struct page *page;
3824     unsigned char *kaddr;
3825     struct f2fs_summary *summary;
3826     struct curseg_info *seg_i;
3827     int written_size = 0;
3828     int i, j;
3829
3830     page = f2fs_grab_meta_page(sbi, blkaddr++);
3831     kaddr = (unsigned char *)page_address(page);
3832     memset(kaddr, 0, PAGE_SIZE);
3833
3834     /* Step 1: write nat cache */
3835     seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3836     memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3837     written_size += SUM_JOURNAL_SIZE;
3838
3839     /* Step 2: write sit cache */
3840     seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3841     memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3842     written_size += SUM_JOURNAL_SIZE;
3843
3844     /* Step 3: write summary entries */
3845     for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3846         unsigned short blkoff;
3847
3848         seg_i = CURSEG_I(sbi, i);
3849         if (sbi->ckpt->alloc_type[i] == SSR)
3850             blkoff = sbi->blocks_per_seg;
3851         else
3852             blkoff = curseg_blkoff(sbi, i);
3853
3854         for (j = 0; j < blkoff; j++) {
3855             if (!page) {
3856                 page = f2fs_grab_meta_page(sbi, blkaddr++);
3857                 kaddr = (unsigned char *)page_address(page);
3858                 memset(kaddr, 0, PAGE_SIZE);
3859                 written_size = 0;
3860             }
3861             summary = (struct f2fs_summary *)(kaddr + written_size);
3862             *summary = seg_i->sum_blk->entries[j];
3863             written_size += SUMMARY_SIZE;
3864
3865             if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3866                             SUM_FOOTER_SIZE)
3867                 continue;
3868
3869             set_page_dirty(page);
3870             f2fs_put_page(page, 1);
3871             page = NULL;
3872         }
3873     }
3874     if (page) {
3875         set_page_dirty(page);
3876         f2fs_put_page(page, 1);
3877     }
3878 }
3879
3880 static void write_normal_summaries(struct f2fs_sb_info *sbi,
3881                     block_t blkaddr, int type)
3882 {
3883     int i, end;
3884
3885     if (IS_DATASEG(type))
3886         end = type + NR_CURSEG_DATA_TYPE;
3887     else
3888         end = type + NR_CURSEG_NODE_TYPE;
3889
3890     for (i = type; i < end; i++)
3891         write_current_sum_page(sbi, i, blkaddr + (i - type));
3892 }
3893
3894 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3895 {
3896     if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
3897         write_compacted_summaries(sbi, start_blk);
3898     else
3899         write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
3900 }
3901
3902 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3903 {
3904     write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
3905 }
3906
3907 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3908                     unsigned int val, int alloc)
3909 {
3910     int i;
3911
3912     if (type == NAT_JOURNAL) {
3913         for (i = 0; i < nats_in_cursum(journal); i++) {
3914             if (le32_to_cpu(nid_in_journal(journal, i)) == val)
3915                 return i;
3916         }
3917         if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
3918             return update_nats_in_cursum(journal, 1);
3919     } else if (type == SIT_JOURNAL) {
3920         for (i = 0; i < sits_in_cursum(journal); i++)
3921             if (le32_to_cpu(segno_in_journal(journal, i)) == val)
3922                 return i;
3923         if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
3924             return update_sits_in_cursum(journal, 1);
3925     }
3926     return -1;
3927 }
3928
3929 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
3930                     unsigned int segno)
3931 {
3932     return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
3933 }
3934
3935 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
3936                     unsigned int start)
3937 {
3938     struct sit_info *sit_i = SIT_I(sbi);
3939     struct page *page;
3940     pgoff_t src_off, dst_off;
3941
3942     src_off = current_sit_addr(sbi, start);
3943     dst_off = next_sit_addr(sbi, src_off);
3944
3945     page = f2fs_grab_meta_page(sbi, dst_off);
3946     seg_info_to_sit_page(sbi, page, start);
3947
3948     set_page_dirty(page);
3949     set_to_next_sit(sit_i, start);
3950
3951     return page;
3952 }
3953
3954 static struct sit_entry_set *grab_sit_entry_set(void)
3955 {
3956     struct sit_entry_set *ses =
3957             f2fs_kmem_cache_alloc(sit_entry_set_slab,
3958                         GFP_NOFS, true, NULL);
3959
3960     ses->entry_cnt = 0;
3961     INIT_LIST_HEAD(&ses->set_list);
3962     return ses;
3963 }
3964
3965 static void release_sit_entry_set(struct sit_entry_set *ses)
3966 {
3967     list_del(&ses->set_list);
3968     kmem_cache_free(sit_entry_set_slab, ses);
3969 }
3970
3971 static void adjust_sit_entry_set(struct sit_entry_set *ses,
3972                         struct list_head *head)
3973 {
3974     struct sit_entry_set *next = ses;
3975
3976     if (list_is_last(&ses->set_list, head))
3977         return;
3978
3979     list_for_each_entry_continue(next, head, set_list)
3980         if (ses->entry_cnt <= next->entry_cnt) {
3981             list_move_tail(&ses->set_list, &next->set_list);
3982             return;
3983         }
3984
3985     list_move_tail(&ses->set_list, head);
3986 }
3987
3988 static void add_sit_entry(unsigned int segno, struct list_head *head)
3989 {
3990     struct sit_entry_set *ses;
3991     unsigned int start_segno = START_SEGNO(segno);
3992
3993     list_for_each_entry(ses, head, set_list) {
3994         if (ses->start_segno == start_segno) {
3995             ses->entry_cnt++;
3996             adjust_sit_entry_set(ses, head);
3997             return;
3998         }
3999     }
4000
4001     ses = grab_sit_entry_set();
4002
4003     ses->start_segno = start_segno;
4004     ses->entry_cnt++;
4005     list_add(&ses->set_list, head);
4006 }
4007
4008 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4009 {
4010     struct f2fs_sm_info *sm_info = SM_I(sbi);
4011     struct list_head *set_list = &sm_info->sit_entry_set;
4012     unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4013     unsigned int segno;
4014
4015     for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4016         add_sit_entry(segno, set_list);
4017 }
4018
4019 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4020 {
4021     struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4022     struct f2fs_journal *journal = curseg->journal;
4023     int i;
4024
4025     down_write(&curseg->journal_rwsem);
4026     for (i = 0; i < sits_in_cursum(journal); i++) {
4027         unsigned int segno;
4028         bool dirtied;
4029
4030         segno = le32_to_cpu(segno_in_journal(journal, i));
4031         dirtied = __mark_sit_entry_dirty(sbi, segno);
4032
4033         if (!dirtied)
4034             add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4035     }
4036     update_sits_in_cursum(journal, -i);
4037     up_write(&curseg->journal_rwsem);
4038 }
4039
4040 /*
4041  * CP calls this function, which flushes SIT entries including sit_journal,
4042  * and moves prefree segs to free segs.
4043  */
4044 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4045 {
4046     struct sit_info *sit_i = SIT_I(sbi);
4047     unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4048     struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4049     struct f2fs_journal *journal = curseg->journal;
4050     struct sit_entry_set *ses, *tmp;
4051     struct list_head *head = &SM_I(sbi)->sit_entry_set;
4052     bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4053     struct seg_entry *se;
4054
4055     down_write(&sit_i->sentry_lock);
4056
4057     if (!sit_i->dirty_sentries)
4058         goto out;
4059
4060     /*
4061      * add and account sit entries of dirty bitmap in sit entry
4062      * set temporarily
4063      */
4064     add_sits_in_set(sbi);
4065
4066     /*
4067      * if there are no enough space in journal to store dirty sit
4068      * entries, remove all entries from journal and add and account
4069      * them in sit entry set.
4070      */
4071     if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4072                                 !to_journal)
4073         remove_sits_in_journal(sbi);
4074
4075     /*
4076      * there are two steps to flush sit entries:
4077      * #1, flush sit entries to journal in current cold data summary block.
4078      * #2, flush sit entries to sit page.
4079      */
4080     list_for_each_entry_safe(ses, tmp, head, set_list) {
4081         struct page *page = NULL;
4082         struct f2fs_sit_block *raw_sit = NULL;
4083         unsigned int start_segno = ses->start_segno;
4084         unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4085                         (unsigned long)MAIN_SEGS(sbi));
4086         unsigned int segno = start_segno;
4087
4088         if (to_journal &&
4089             !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4090             to_journal = false;
4091
4092         if (to_journal) {
4093             down_write(&curseg->journal_rwsem);
4094         } else {
4095             page = get_next_sit_page(sbi, start_segno);
4096             raw_sit = page_address(page);
4097         }
4098
4099         /* flush dirty sit entries in region of current sit set */
4100         for_each_set_bit_from(segno, bitmap, end) {
4101             int offset, sit_offset;
4102
4103             se = get_seg_entry(sbi, segno);
4104 #ifdef CONFIG_F2FS_CHECK_FS
4105             if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4106                         SIT_VBLOCK_MAP_SIZE))
4107                 f2fs_bug_on(sbi, 1);
4108 #endif
4109
4110             /* add discard candidates */
4111             if (!(cpc->reason & CP_DISCARD)) {
4112                 cpc->trim_start = segno;
4113                 add_discard_addrs(sbi, cpc, false);
4114             }
4115
4116             if (to_journal) {
4117                 offset = f2fs_lookup_journal_in_cursum(journal,
4118                             SIT_JOURNAL, segno, 1);
4119                 f2fs_bug_on(sbi, offset < 0);
4120                 segno_in_journal(journal, offset) =
4121                             cpu_to_le32(segno);
4122                 seg_info_to_raw_sit(se,
4123                     &sit_in_journal(journal, offset));
4124                 check_block_count(sbi, segno,
4125                     &sit_in_journal(journal, offset));
4126             } else {
4127                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4128                 seg_info_to_raw_sit(se,
4129                         &raw_sit->entries[sit_offset]);
4130                 check_block_count(sbi, segno,
4131                         &raw_sit->entries[sit_offset]);
4132             }
4133
4134             __clear_bit(segno, bitmap);
4135             sit_i->dirty_sentries--;
4136             ses->entry_cnt--;
4137         }
4138
4139         if (to_journal)
4140             up_write(&curseg->journal_rwsem);
4141         else
4142             f2fs_put_page(page, 1);
4143
4144         f2fs_bug_on(sbi, ses->entry_cnt);
4145         release_sit_entry_set(ses);
4146     }
4147
4148     f2fs_bug_on(sbi, !list_empty(head));
4149     f2fs_bug_on(sbi, sit_i->dirty_sentries);
4150 out:
4151     if (cpc->reason & CP_DISCARD) {
4152         __u64 trim_start = cpc->trim_start;
4153
4154         for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4155             add_discard_addrs(sbi, cpc, false);
4156
4157         cpc->trim_start = trim_start;
4158     }
4159     up_write(&sit_i->sentry_lock);
4160
4161     set_prefree_as_free_segments(sbi);
4162 }
4163
4164 static int build_sit_info(struct f2fs_sb_info *sbi)
4165 {
4166     struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4167     struct sit_info *sit_i;
4168     unsigned int sit_segs, start;
4169     char *src_bitmap, *bitmap;
4170     unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4171     unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4172
4173     /* allocate memory for SIT information */
4174     sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4175     if (!sit_i)
4176         return -ENOMEM;
4177
4178     SM_I(sbi)->sit_info = sit_i;
4179
4180     sit_i->sentries =
4181         f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4182                           MAIN_SEGS(sbi)),
4183                   GFP_KERNEL);
4184     if (!sit_i->sentries)
4185         return -ENOMEM;
4186
4187     main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4188     sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4189                                 GFP_KERNEL);
4190     if (!sit_i->dirty_sentries_bitmap)
4191         return -ENOMEM;
4192
4193 #ifdef CONFIG_F2FS_CHECK_FS
4194     bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4195 #else
4196     bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4197 #endif
4198     sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4199     if (!sit_i->bitmap)
4200         return -ENOMEM;
4201
4202     bitmap = sit_i->bitmap;
4203
4204     for (start = 0; start < MAIN_SEGS(sbi); start++) {
4205         sit_i->sentries[start].cur_valid_map = bitmap;
4206         bitmap += SIT_VBLOCK_MAP_SIZE;
4207
4208         sit_i->sentries[start].ckpt_valid_map = bitmap;
4209         bitmap += SIT_VBLOCK_MAP_SIZE;
4210
4211 #ifdef CONFIG_F2FS_CHECK_FS
4212         sit_i->sentries[start].cur_valid_map_mir = bitmap;
4213         bitmap += SIT_VBLOCK_MAP_SIZE;
4214 #endif
4215
4216         if (discard_map) {
4217             sit_i->sentries[start].discard_map = bitmap;
4218             bitmap += SIT_VBLOCK_MAP_SIZE;
4219         }
4220     }
4221
4222     sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4223     if (!sit_i->tmp_map)
4224         return -ENOMEM;
4225
4226     if (__is_large_section(sbi)) {
4227         sit_i->sec_entries =
4228             f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4229                               MAIN_SECS(sbi)),
4230                       GFP_KERNEL);
4231         if (!sit_i->sec_entries)
4232             return -ENOMEM;
4233     }
4234
4235     /* get information related with SIT */
4236     sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4237
4238     /* setup SIT bitmap from ckeckpoint pack */
4239     sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4240     src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4241
4242     sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4243     if (!sit_i->sit_bitmap)
4244         return -ENOMEM;
4245
4246 #ifdef CONFIG_F2FS_CHECK_FS
4247     sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4248                     sit_bitmap_size, GFP_KERNEL);
4249     if (!sit_i->sit_bitmap_mir)
4250         return -ENOMEM;
4251
4252     sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4253                     main_bitmap_size, GFP_KERNEL);
4254     if (!sit_i->invalid_segmap)
4255         return -ENOMEM;
4256 #endif
4257
4258     /* init SIT information */
4259     sit_i->s_ops = &default_salloc_ops;
4260
4261     sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4262     sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4263     sit_i->written_valid_blocks = 0;
4264     sit_i->bitmap_size = sit_bitmap_size;
4265     sit_i->dirty_sentries = 0;
4266     sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4267     sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4268     sit_i->mounted_time = ktime_get_boottime_seconds();
4269     init_rwsem(&sit_i->sentry_lock);
4270     return 0;
4271 }
4272
4273 static int build_free_segmap(struct f2fs_sb_info *sbi)
4274 {
4275     struct free_segmap_info *free_i;
4276     unsigned int bitmap_size, sec_bitmap_size;
4277
4278     /* allocate memory for free segmap information */
4279     free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4280     if (!free_i)
4281         return -ENOMEM;
4282
4283     SM_I(sbi)->free_info = free_i;
4284
4285     bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4286     free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4287     if (!free_i->free_segmap)
4288         return -ENOMEM;
4289
4290     sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4291     free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4292     if (!free_i->free_secmap)
4293         return -ENOMEM;
4294
4295     /* set all segments as dirty temporarily */
4296     memset(free_i->free_segmap, 0xff, bitmap_size);
4297     memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4298
4299     /* init free segmap information */
4300     free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4301     free_i->free_segments = 0;
4302     free_i->free_sections = 0;
4303     spin_lock_init(&free_i->segmap_lock);
4304     return 0;
4305 }
4306
4307 static int build_curseg(struct f2fs_sb_info *sbi)
4308 {
4309     struct curseg_info *array;
4310     int i;
4311
4312     array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4313                     sizeof(*array)), GFP_KERNEL);
4314     if (!array)
4315         return -ENOMEM;
4316
4317     SM_I(sbi)->curseg_array = array;
4318
4319     for (i = 0; i < NO_CHECK_TYPE; i++) {
4320         mutex_init(&array[i].curseg_mutex);
4321         array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4322         if (!array[i].sum_blk)
4323             return -ENOMEM;
4324         init_rwsem(&array[i].journal_rwsem);
4325         array[i].journal = f2fs_kzalloc(sbi,
4326                 sizeof(struct f2fs_journal), GFP_KERNEL);
4327         if (!array[i].journal)
4328             return -ENOMEM;
4329         if (i < NR_PERSISTENT_LOG)
4330             array[i].seg_type = CURSEG_HOT_DATA + i;
4331         else if (i == CURSEG_COLD_DATA_PINNED)
4332             array[i].seg_type = CURSEG_COLD_DATA;
4333         else if (i == CURSEG_ALL_DATA_ATGC)
4334             array[i].seg_type = CURSEG_COLD_DATA;
4335         array[i].segno = NULL_SEGNO;
4336         array[i].next_blkoff = 0;
4337         array[i].inited = false;
4338     }
4339     return restore_curseg_summaries(sbi);
4340 }
4341
4342 static int build_sit_entries(struct f2fs_sb_info *sbi)
4343 {
4344     struct sit_info *sit_i = SIT_I(sbi);
4345     struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4346     struct f2fs_journal *journal = curseg->journal;
4347     struct seg_entry *se;
4348     struct f2fs_sit_entry sit;
4349     int sit_blk_cnt = SIT_BLK_CNT(sbi);
4350     unsigned int i, start, end;
4351     unsigned int readed, start_blk = 0;
4352     int err = 0;
4353     block_t sit_valid_blocks[2] = {0, 0};
4354
4355     do {
4356         readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4357                             META_SIT, true);
4358
4359         start = start_blk * sit_i->sents_per_block;
4360         end = (start_blk + readed) * sit_i->sents_per_block;
4361
4362         for (; start < end && start < MAIN_SEGS(sbi); start++) {
4363             struct f2fs_sit_block *sit_blk;
4364             struct page *page;
4365
4366             se = &sit_i->sentries[start];
4367             page = get_current_sit_page(sbi, start);
4368             if (IS_ERR(page))
4369                 return PTR_ERR(page);
4370             sit_blk = (struct f2fs_sit_block *)page_address(page);
4371             sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4372             f2fs_put_page(page, 1);
4373
4374             err = check_block_count(sbi, start, &sit);
4375             if (err)
4376                 return err;
4377             seg_info_from_raw_sit(se, &sit);
4378
4379             if (se->type >= NR_PERSISTENT_LOG) {
4380                 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4381                             se->type, start);
4382                 return -EFSCORRUPTED;
4383             }
4384
4385             sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4386
4387             if (f2fs_block_unit_discard(sbi)) {
4388                 /* build discard map only one time */
4389                 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4390                     memset(se->discard_map, 0xff,
4391                         SIT_VBLOCK_MAP_SIZE);
4392                 } else {
4393                     memcpy(se->discard_map,
4394                         se->cur_valid_map,
4395                         SIT_VBLOCK_MAP_SIZE);
4396                     sbi->discard_blks +=
4397                         sbi->blocks_per_seg -
4398                         se->valid_blocks;
4399                 }
4400             }
4401
4402             if (__is_large_section(sbi))
4403                 get_sec_entry(sbi, start)->valid_blocks +=
4404                             se->valid_blocks;
4405         }
4406         start_blk += readed;
4407     } while (start_blk < sit_blk_cnt);
4408
4409     down_read(&curseg->journal_rwsem);
4410     for (i = 0; i < sits_in_cursum(journal); i++) {
4411         unsigned int old_valid_blocks;
4412
4413         start = le32_to_cpu(segno_in_journal(journal, i));
4414         if (start >= MAIN_SEGS(sbi)) {
4415             f2fs_err(sbi, "Wrong journal entry on segno %u",
4416                  start);
4417             err = -EFSCORRUPTED;
4418             break;
4419         }
4420
4421         se = &sit_i->sentries[start];
4422         sit = sit_in_journal(journal, i);
4423
4424         old_valid_blocks = se->valid_blocks;
4425
4426         sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4427
4428         err = check_block_count(sbi, start, &sit);
4429         if (err)
4430             break;
4431         seg_info_from_raw_sit(se, &sit);
4432
4433         if (se->type >= NR_PERSISTENT_LOG) {
4434             f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4435                             se->type, start);
4436             err = -EFSCORRUPTED;
4437             break;
4438         }
4439
4440         sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4441
4442         if (f2fs_block_unit_discard(sbi)) {
4443             if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4444                 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4445             } else {
4446                 memcpy(se->discard_map, se->cur_valid_map,
4447                             SIT_VBLOCK_MAP_SIZE);
4448                 sbi->discard_blks += old_valid_blocks;
4449                 sbi->discard_blks -= se->valid_blocks;
4450             }
4451         }
4452
4453         if (__is_large_section(sbi)) {
4454             get_sec_entry(sbi, start)->valid_blocks +=
4455                             se->valid_blocks;
4456             get_sec_entry(sbi, start)->valid_blocks -=
4457                             old_valid_blocks;
4458         }
4459     }
4460     up_read(&curseg->journal_rwsem);
4461
4462     if (err)
4463         return err;
4464
4465     if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4466         f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4467              sit_valid_blocks[NODE], valid_node_count(sbi));
4468         return -EFSCORRUPTED;
4469     }
4470
4471     if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
4472                 valid_user_blocks(sbi)) {
4473         f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
4474              sit_valid_blocks[DATA], sit_valid_blocks[NODE],
4475              valid_user_blocks(sbi));
4476         return -EFSCORRUPTED;
4477     }
4478
4479     return 0;
4480 }
4481
4482 static void init_free_segmap(struct f2fs_sb_info *sbi)
4483 {
4484     unsigned int start;
4485     int type;
4486     struct seg_entry *sentry;
4487
4488     for (start = 0; start < MAIN_SEGS(sbi); start++) {
4489         if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4490             continue;
4491         sentry = get_seg_entry(sbi, start);
4492         if (!sentry->valid_blocks)
4493             __set_free(sbi, start);
4494         else
4495             SIT_I(sbi)->written_valid_blocks +=
4496                         sentry->valid_blocks;
4497     }
4498
4499     /* set use the current segments */
4500     for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4501         struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4502
4503         __set_test_and_inuse(sbi, curseg_t->segno);
4504     }
4505 }
4506
4507 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4508 {
4509     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4510     struct free_segmap_info *free_i = FREE_I(sbi);
4511     unsigned int segno = 0, offset = 0, secno;
4512     block_t valid_blocks, usable_blks_in_seg;
4513
4514     while (1) {
4515         /* find dirty segment based on free segmap */
4516         segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4517         if (segno >= MAIN_SEGS(sbi))
4518             break;
4519         offset = segno + 1;
4520         valid_blocks = get_valid_blocks(sbi, segno, false);
4521         usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4522         if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4523             continue;
4524         if (valid_blocks > usable_blks_in_seg) {
4525             f2fs_bug_on(sbi, 1);
4526             continue;
4527         }
4528         mutex_lock(&dirty_i->seglist_lock);
4529         __locate_dirty_segment(sbi, segno, DIRTY);
4530         mutex_unlock(&dirty_i->seglist_lock);
4531     }
4532
4533     if (!__is_large_section(sbi))
4534         return;
4535
4536     mutex_lock(&dirty_i->seglist_lock);
4537     for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4538         valid_blocks = get_valid_blocks(sbi, segno, true);
4539         secno = GET_SEC_FROM_SEG(sbi, segno);
4540
4541         if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
4542             continue;
4543         if (IS_CURSEC(sbi, secno))
4544             continue;
4545         set_bit(secno, dirty_i->dirty_secmap);
4546     }
4547     mutex_unlock(&dirty_i->seglist_lock);
4548 }
4549
4550 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4551 {
4552     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4553     unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4554
4555     dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4556     if (!dirty_i->victim_secmap)
4557         return -ENOMEM;
4558
4559     dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4560     if (!dirty_i->pinned_secmap)
4561         return -ENOMEM;
4562
4563     dirty_i->pinned_secmap_cnt = 0;
4564     dirty_i->enable_pin_section = true;
4565     return 0;
4566 }
4567
4568 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4569 {
4570     struct dirty_seglist_info *dirty_i;
4571     unsigned int bitmap_size, i;
4572
4573     /* allocate memory for dirty segments list information */
4574     dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4575                                 GFP_KERNEL);
4576     if (!dirty_i)
4577         return -ENOMEM;
4578
4579     SM_I(sbi)->dirty_info = dirty_i;
4580     mutex_init(&dirty_i->seglist_lock);
4581
4582     bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4583
4584     for (i = 0; i < NR_DIRTY_TYPE; i++) {
4585         dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4586                                 GFP_KERNEL);
4587         if (!dirty_i->dirty_segmap[i])
4588             return -ENOMEM;
4589     }
4590
4591     if (__is_large_section(sbi)) {
4592         bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4593         dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4594                         bitmap_size, GFP_KERNEL);
4595         if (!dirty_i->dirty_secmap)
4596             return -ENOMEM;
4597     }
4598
4599     init_dirty_segmap(sbi);
4600     return init_victim_secmap(sbi);
4601 }
4602
4603 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4604 {
4605     int i;
4606
4607     /*
4608      * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4609      * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4610      */
4611     for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4612         struct curseg_info *curseg = CURSEG_I(sbi, i);
4613         struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4614         unsigned int blkofs = curseg->next_blkoff;
4615
4616         if (f2fs_sb_has_readonly(sbi) &&
4617             i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4618             continue;
4619
4620         sanity_check_seg_type(sbi, curseg->seg_type);
4621
4622         if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
4623             f2fs_err(sbi,
4624                  "Current segment has invalid alloc_type:%d",
4625                  curseg->alloc_type);
4626             return -EFSCORRUPTED;
4627         }
4628
4629         if (f2fs_test_bit(blkofs, se->cur_valid_map))
4630             goto out;
4631
4632         if (curseg->alloc_type == SSR)
4633             continue;
4634
4635         for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4636             if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4637                 continue;
4638 out:
4639             f2fs_err(sbi,
4640                  "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4641                  i, curseg->segno, curseg->alloc_type,
4642                  curseg->next_blkoff, blkofs);
4643             return -EFSCORRUPTED;
4644         }
4645     }
4646     return 0;
4647 }
4648
4649 #ifdef CONFIG_BLK_DEV_ZONED
4650
4651 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4652                     struct f2fs_dev_info *fdev,
4653                     struct blk_zone *zone)
4654 {
4655     unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4656     block_t zone_block, wp_block, last_valid_block;
4657     unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4658     int i, s, b, ret;
4659     struct seg_entry *se;
4660
4661     if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4662         return 0;
4663
4664     wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4665     wp_segno = GET_SEGNO(sbi, wp_block);
4666     wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4667     zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4668     zone_segno = GET_SEGNO(sbi, zone_block);
4669     zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4670
4671     if (zone_segno >= MAIN_SEGS(sbi))
4672         return 0;
4673
4674     /*
4675      * Skip check of zones cursegs point to, since
4676      * fix_curseg_write_pointer() checks them.
4677      */
4678     for (i = 0; i < NO_CHECK_TYPE; i++)
4679         if (zone_secno == GET_SEC_FROM_SEG(sbi,
4680                            CURSEG_I(sbi, i)->segno))
4681             return 0;
4682
4683     /*
4684      * Get last valid block of the zone.
4685      */
4686     last_valid_block = zone_block - 1;
4687     for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4688         segno = zone_segno + s;
4689         se = get_seg_entry(sbi, segno);
4690         for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4691             if (f2fs_test_bit(b, se->cur_valid_map)) {
4692                 last_valid_block = START_BLOCK(sbi, segno) + b;
4693                 break;
4694             }
4695         if (last_valid_block >= zone_block)
4696             break;
4697     }
4698
4699     /*
4700      * If last valid block is beyond the write pointer, report the
4701      * inconsistency. This inconsistency does not cause write error
4702      * because the zone will not be selected for write operation until
4703      * it get discarded. Just report it.
4704      */
4705     if (last_valid_block >= wp_block) {
4706         f2fs_notice(sbi, "Valid block beyond write pointer: "
4707                 "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4708                 GET_SEGNO(sbi, last_valid_block),
4709                 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4710                 wp_segno, wp_blkoff);
4711         return 0;
4712     }
4713
4714     /*
4715      * If there is no valid block in the zone and if write pointer is
4716      * not at zone start, reset the write pointer.
4717      */
4718     if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4719         f2fs_notice(sbi,
4720                 "Zone without valid block has non-zero write "
4721                 "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4722                 wp_segno, wp_blkoff);
4723         ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4724                     zone->len >> log_sectors_per_block);
4725         if (ret) {
4726             f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4727                  fdev->path, ret);
4728             return ret;
4729         }
4730     }
4731
4732     return 0;
4733 }
4734
4735 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4736                           block_t zone_blkaddr)
4737 {
4738     int i;
4739
4740     for (i = 0; i < sbi->s_ndevs; i++) {
4741         if (!bdev_is_zoned(FDEV(i).bdev))
4742             continue;
4743         if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4744                 zone_blkaddr <= FDEV(i).end_blk))
4745             return &FDEV(i);
4746     }
4747
4748     return NULL;
4749 }
4750
4751 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4752                   void *data)
4753 {
4754     memcpy(data, zone, sizeof(struct blk_zone));
4755     return 0;
4756 }
4757
4758 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4759 {
4760     struct curseg_info *cs = CURSEG_I(sbi, type);
4761     struct f2fs_dev_info *zbd;
4762     struct blk_zone zone;
4763     unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4764     block_t cs_zone_block, wp_block;
4765     unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4766     sector_t zone_sector;
4767     int err;
4768
4769     cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4770     cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4771
4772     zbd = get_target_zoned_dev(sbi, cs_zone_block);
4773     if (!zbd)
4774         return 0;
4775
4776     /* report zone for the sector the curseg points to */
4777     zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4778         << log_sectors_per_block;
4779     err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4780                   report_one_zone_cb, &zone);
4781     if (err != 1) {
4782         f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4783              zbd->path, err);
4784         return err;
4785     }
4786
4787     if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4788         return 0;
4789
4790     wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4791     wp_segno = GET_SEGNO(sbi, wp_block);
4792     wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4793     wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4794
4795     if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4796         wp_sector_off == 0)
4797         return 0;
4798
4799     f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4800             "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4801             type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4802
4803     f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4804             "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4805
4806     f2fs_allocate_new_section(sbi, type, true);
4807
4808     /* check consistency of the zone curseg pointed to */
4809     if (check_zone_write_pointer(sbi, zbd, &zone))
4810         return -EIO;
4811
4812     /* check newly assigned zone */
4813     cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4814     cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4815
4816     zbd = get_target_zoned_dev(sbi, cs_zone_block);
4817     if (!zbd)
4818         return 0;
4819
4820     zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4821         << log_sectors_per_block;
4822     err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4823                   report_one_zone_cb, &zone);
4824     if (err != 1) {
4825         f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4826              zbd->path, err);
4827         return err;
4828     }
4829
4830     if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4831         return 0;
4832
4833     if (zone.wp != zone.start) {
4834         f2fs_notice(sbi,
4835                 "New zone for curseg[%d] is not yet discarded. "
4836                 "Reset the zone: curseg[0x%x,0x%x]",
4837                 type, cs->segno, cs->next_blkoff);
4838         err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
4839                 zone_sector >> log_sectors_per_block,
4840                 zone.len >> log_sectors_per_block);
4841         if (err) {
4842             f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4843                  zbd->path, err);
4844             return err;
4845         }
4846     }
4847
4848     return 0;
4849 }
4850
4851 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4852 {
4853     int i, ret;
4854
4855     for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4856         ret = fix_curseg_write_pointer(sbi, i);
4857         if (ret)
4858             return ret;
4859     }
4860
4861     return 0;
4862 }
4863
4864 struct check_zone_write_pointer_args {
4865     struct f2fs_sb_info *sbi;
4866     struct f2fs_dev_info *fdev;
4867 };
4868
4869 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
4870                       void *data)
4871 {
4872     struct check_zone_write_pointer_args *args;
4873
4874     args = (struct check_zone_write_pointer_args *)data;
4875
4876     return check_zone_write_pointer(args->sbi, args->fdev, zone);
4877 }
4878
4879 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4880 {
4881     int i, ret;
4882     struct check_zone_write_pointer_args args;
4883
4884     for (i = 0; i < sbi->s_ndevs; i++) {
4885         if (!bdev_is_zoned(FDEV(i).bdev))
4886             continue;
4887
4888         args.sbi = sbi;
4889         args.fdev = &FDEV(i);
4890         ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
4891                       check_zone_write_pointer_cb, &args);
4892         if (ret < 0)
4893             return ret;
4894     }
4895
4896     return 0;
4897 }
4898
4899 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
4900                         unsigned int dev_idx)
4901 {
4902     if (!bdev_is_zoned(FDEV(dev_idx).bdev))
4903         return true;
4904     return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
4905 }
4906
4907 /* Return the zone index in the given device */
4908 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
4909                     int dev_idx)
4910 {
4911     block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
4912
4913     return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
4914                         sbi->log_blocks_per_blkz;
4915 }
4916
4917 /*
4918  * Return the usable segments in a section based on the zone's
4919  * corresponding zone capacity. Zone is equal to a section.
4920  */
4921 static inline unsigned int f2fs_usable_zone_segs_in_sec(
4922         struct f2fs_sb_info *sbi, unsigned int segno)
4923 {
4924     unsigned int dev_idx, zone_idx;
4925
4926     dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
4927     zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
4928
4929     /* Conventional zone's capacity is always equal to zone size */
4930     if (is_conv_zone(sbi, zone_idx, dev_idx))
4931         return sbi->segs_per_sec;
4932
4933     if (!sbi->unusable_blocks_per_sec)
4934         return sbi->segs_per_sec;
4935
4936     /* Get the segment count beyond zone capacity block */
4937     return sbi->segs_per_sec - (sbi->unusable_blocks_per_sec >>
4938                         sbi->log_blocks_per_seg);
4939 }
4940
4941 /*
4942  * Return the number of usable blocks in a segment. The number of blocks
4943  * returned is always equal to the number of blocks in a segment for
4944  * segments fully contained within a sequential zone capacity or a
4945  * conventional zone. For segments partially contained in a sequential
4946  * zone capacity, the number of usable blocks up to the zone capacity
4947  * is returned. 0 is returned in all other cases.
4948  */
4949 static inline unsigned int f2fs_usable_zone_blks_in_seg(
4950             struct f2fs_sb_info *sbi, unsigned int segno)
4951 {
4952     block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
4953     unsigned int zone_idx, dev_idx, secno;
4954
4955     secno = GET_SEC_FROM_SEG(sbi, segno);
4956     seg_start = START_BLOCK(sbi, segno);
4957     dev_idx = f2fs_target_device_index(sbi, seg_start);
4958     zone_idx = get_zone_idx(sbi, secno, dev_idx);
4959
4960     /*
4961      * Conventional zone's capacity is always equal to zone size,
4962      * so, blocks per segment is unchanged.
4963      */
4964     if (is_conv_zone(sbi, zone_idx, dev_idx))
4965         return sbi->blocks_per_seg;
4966
4967     if (!sbi->unusable_blocks_per_sec)
4968         return sbi->blocks_per_seg;
4969
4970     sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
4971     sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
4972
4973     /*
4974      * If segment starts before zone capacity and spans beyond
4975      * zone capacity, then usable blocks are from seg start to
4976      * zone capacity. If the segment starts after the zone capacity,
4977      * then there are no usable blocks.
4978      */
4979     if (seg_start >= sec_cap_blkaddr)
4980         return 0;
4981     if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
4982         return sec_cap_blkaddr - seg_start;
4983
4984     return sbi->blocks_per_seg;
4985 }
4986 #else
4987 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4988 {
4989     return 0;
4990 }
4991
4992 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4993 {
4994     return 0;
4995 }
4996
4997 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
4998                             unsigned int segno)
4999 {
5000     return 0;
5001 }
5002
5003 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
5004                             unsigned int segno)
5005 {
5006     return 0;
5007 }
5008 #endif
5009 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5010                     unsigned int segno)
5011 {
5012     if (f2fs_sb_has_blkzoned(sbi))
5013         return f2fs_usable_zone_blks_in_seg(sbi, segno);
5014
5015     return sbi->blocks_per_seg;
5016 }
5017
5018 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5019                     unsigned int segno)
5020 {
5021     if (f2fs_sb_has_blkzoned(sbi))
5022         return f2fs_usable_zone_segs_in_sec(sbi, segno);
5023
5024     return sbi->segs_per_sec;
5025 }
5026
5027 /*
5028  * Update min, max modified time for cost-benefit GC algorithm
5029  */
5030 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5031 {
5032     struct sit_info *sit_i = SIT_I(sbi);
5033     unsigned int segno;
5034
5035     down_write(&sit_i->sentry_lock);
5036
5037     sit_i->min_mtime = ULLONG_MAX;
5038
5039     for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5040         unsigned int i;
5041         unsigned long long mtime = 0;
5042
5043         for (i = 0; i < sbi->segs_per_sec; i++)
5044             mtime += get_seg_entry(sbi, segno + i)->mtime;
5045
5046         mtime = div_u64(mtime, sbi->segs_per_sec);
5047
5048         if (sit_i->min_mtime > mtime)
5049             sit_i->min_mtime = mtime;
5050     }
5051     sit_i->max_mtime = get_mtime(sbi, false);
5052     sit_i->dirty_max_mtime = 0;
5053     up_write(&sit_i->sentry_lock);
5054 }
5055
5056 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5057 {
5058     struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5059     struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5060     struct f2fs_sm_info *sm_info;
5061     int err;
5062
5063     sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5064     if (!sm_info)
5065         return -ENOMEM;
5066
5067     /* init sm info */
5068     sbi->sm_info = sm_info;
5069     sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5070     sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5071     sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5072     sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5073     sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5074     sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5075     sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5076     sm_info->rec_prefree_segments = sm_info->main_segments *
5077                     DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5078     if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5079         sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5080
5081     if (!f2fs_lfs_mode(sbi))
5082         sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5083     sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5084     sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5085     sm_info->min_seq_blocks = sbi->blocks_per_seg;
5086     sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5087     sm_info->min_ssr_sections = reserved_sections(sbi);
5088
5089     INIT_LIST_HEAD(&sm_info->sit_entry_set);
5090
5091     init_f2fs_rwsem(&sm_info->curseg_lock);
5092
5093     if (!f2fs_readonly(sbi->sb)) {
5094         err = f2fs_create_flush_cmd_control(sbi);
5095         if (err)
5096             return err;
5097     }
5098
5099     err = create_discard_cmd_control(sbi);
5100     if (err)
5101         return err;
5102
5103     err = build_sit_info(sbi);
5104     if (err)
5105         return err;
5106     err = build_free_segmap(sbi);
5107     if (err)
5108         return err;
5109     err = build_curseg(sbi);
5110     if (err)
5111         return err;
5112
5113     /* reinit free segmap based on SIT */
5114     err = build_sit_entries(sbi);
5115     if (err)
5116         return err;
5117
5118     init_free_segmap(sbi);
5119     err = build_dirty_segmap(sbi);
5120     if (err)
5121         return err;
5122
5123     err = sanity_check_curseg(sbi);
5124     if (err)
5125         return err;
5126
5127     init_min_max_mtime(sbi);
5128     return 0;
5129 }
5130
5131 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5132         enum dirty_type dirty_type)
5133 {
5134     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5135
5136     mutex_lock(&dirty_i->seglist_lock);
5137     kvfree(dirty_i->dirty_segmap[dirty_type]);
5138     dirty_i->nr_dirty[dirty_type] = 0;
5139     mutex_unlock(&dirty_i->seglist_lock);
5140 }
5141
5142 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5143 {
5144     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5145
5146     kvfree(dirty_i->pinned_secmap);
5147     kvfree(dirty_i->victim_secmap);
5148 }
5149
5150 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5151 {
5152     struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5153     int i;
5154
5155     if (!dirty_i)
5156         return;
5157
5158     /* discard pre-free/dirty segments list */
5159     for (i = 0; i < NR_DIRTY_TYPE; i++)
5160         discard_dirty_segmap(sbi, i);
5161
5162     if (__is_large_section(sbi)) {
5163         mutex_lock(&dirty_i->seglist_lock);
5164         kvfree(dirty_i->dirty_secmap);
5165         mutex_unlock(&dirty_i->seglist_lock);
5166     }
5167
5168     destroy_victim_secmap(sbi);
5169     SM_I(sbi)->dirty_info = NULL;
5170     kfree(dirty_i);
5171 }
5172
5173 static void destroy_curseg(struct f2fs_sb_info *sbi)
5174 {
5175     struct curseg_info *array = SM_I(sbi)->curseg_array;
5176     int i;
5177
5178     if (!array)
5179         return;
5180     SM_I(sbi)->curseg_array = NULL;
5181     for (i = 0; i < NR_CURSEG_TYPE; i++) {
5182         kfree(array[i].sum_blk);
5183         kfree(array[i].journal);
5184     }
5185     kfree(array);
5186 }
5187
5188 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5189 {
5190     struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5191
5192     if (!free_i)
5193         return;
5194     SM_I(sbi)->free_info = NULL;
5195     kvfree(free_i->free_segmap);
5196     kvfree(free_i->free_secmap);
5197     kfree(free_i);
5198 }
5199
5200 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5201 {
5202     struct sit_info *sit_i = SIT_I(sbi);
5203
5204     if (!sit_i)
5205         return;
5206
5207     if (sit_i->sentries)
5208         kvfree(sit_i->bitmap);
5209     kfree(sit_i->tmp_map);
5210
5211     kvfree(sit_i->sentries);
5212     kvfree(sit_i->sec_entries);
5213     kvfree(sit_i->dirty_sentries_bitmap);
5214
5215     SM_I(sbi)->sit_info = NULL;
5216     kvfree(sit_i->sit_bitmap);
5217 #ifdef CONFIG_F2FS_CHECK_FS
5218     kvfree(sit_i->sit_bitmap_mir);
5219     kvfree(sit_i->invalid_segmap);
5220 #endif
5221     kfree(sit_i);
5222 }
5223
5224 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5225 {
5226     struct f2fs_sm_info *sm_info = SM_I(sbi);
5227
5228     if (!sm_info)
5229         return;
5230     f2fs_destroy_flush_cmd_control(sbi, true);
5231     destroy_discard_cmd_control(sbi);
5232     destroy_dirty_segmap(sbi);
5233     destroy_curseg(sbi);
5234     destroy_free_segmap(sbi);
5235     destroy_sit_info(sbi);
5236     sbi->sm_info = NULL;
5237     kfree(sm_info);
5238 }
5239
5240 int __init f2fs_create_segment_manager_caches(void)
5241 {
5242     discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5243             sizeof(struct discard_entry));
5244     if (!discard_entry_slab)
5245         goto fail;
5246
5247     discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5248             sizeof(struct discard_cmd));
5249     if (!discard_cmd_slab)
5250         goto destroy_discard_entry;
5251
5252     sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5253             sizeof(struct sit_entry_set));
5254     if (!sit_entry_set_slab)
5255         goto destroy_discard_cmd;
5256
5257     revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5258             sizeof(struct revoke_entry));
5259     if (!revoke_entry_slab)
5260         goto destroy_sit_entry_set;
5261     return 0;
5262
5263 destroy_sit_entry_set:
5264     kmem_cache_destroy(sit_entry_set_slab);
5265 destroy_discard_cmd:
5266     kmem_cache_destroy(discard_cmd_slab);
5267 destroy_discard_entry:
5268     kmem_cache_destroy(discard_entry_slab);
5269 fail:
5270     return -ENOMEM;
5271 }
5272
5273 void f2fs_destroy_segment_manager_caches(void)
5274 {
5275     kmem_cache_destroy(sit_entry_set_slab);
5276     kmem_cache_destroy(discard_cmd_slab);
5277     kmem_cache_destroy(discard_entry_slab);
5278     kmem_cache_destroy(revoke_entry_slab);
5279 }