From 0a26f327e46c203229e72c823dfec71a2b405ec5 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 5 Jan 2023 12:51:45 -0800 Subject: block: make BLK_DEF_MAX_SECTORS unsigned This is used as an unsigned value, so define it that way to avoid having to cast it. Suggested-by: Christoph Hellwig Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Reviewed-by: Bart Van Assche Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20230105205146.3610282-2-kbusch@meta.com Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 43d4e073b111..2b85161e2256 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1095,11 +1095,12 @@ static inline bool bdev_is_partition(struct block_device *bdev) enum blk_default_limits { BLK_MAX_SEGMENTS = 128, BLK_SAFE_MAX_SECTORS = 255, - BLK_DEF_MAX_SECTORS = 2560, BLK_MAX_SEGMENT_SIZE = 65536, BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, }; +#define BLK_DEF_MAX_SECTORS 2560u + static inline unsigned long queue_segment_boundary(const struct request_queue *q) { return q->limits.seg_boundary_mask; -- cgit v1.2.3 From c9c77418a98273fe96835c42666f7427b3883f48 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 5 Jan 2023 12:51:46 -0800 Subject: block: save user max_sectors limit The user can set the max_sectors limit to any valid value via sysfs /sys/block//queue/max_sectors_kb attribute. If the device limits are ever rescanned, though, the limit reverts back to the potentially artificially low BLK_DEF_MAX_SECTORS value. Preserve the user's setting as the max_sectors limit as long as it's valid. The user can reset back to defaults by writing 0 to the sysfs file. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20230105205146.3610282-3-kbusch@meta.com Signed-off-by: Jens Axboe --- Documentation/ABI/stable/sysfs-block | 3 ++- block/blk-settings.c | 9 +++++++-- block/blk-sysfs.c | 21 +++++++++++++++------ include/linux/blkdev.h | 1 + 4 files changed, 25 insertions(+), 9 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block index cd14ecb3c9a5..ac1e519272aa 100644 --- a/Documentation/ABI/stable/sysfs-block +++ b/Documentation/ABI/stable/sysfs-block @@ -432,7 +432,8 @@ Contact: linux-block@vger.kernel.org Description: [RW] This is the maximum number of kilobytes that the block layer will allow for a filesystem request. Must be smaller than - or equal to the maximum size allowed by the hardware. + or equal to the maximum size allowed by the hardware. Write 0 + to use default kernel settings. What: /sys/block//queue/max_segment_size diff --git a/block/blk-settings.c b/block/blk-settings.c index 9875ca131eb0..9c9713c9269c 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -40,7 +40,7 @@ void blk_set_default_limits(struct queue_limits *lim) lim->virt_boundary_mask = 0; lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; - lim->max_dev_sectors = 0; + lim->max_user_sectors = lim->max_dev_sectors = 0; lim->chunk_sectors = 0; lim->max_write_zeroes_sectors = 0; lim->max_zone_append_sectors = 0; @@ -135,7 +135,12 @@ void blk_queue_max_hw_sectors(struct request_queue *q, unsigned int max_hw_secto limits->max_hw_sectors = max_hw_sectors; max_sectors = min_not_zero(max_hw_sectors, limits->max_dev_sectors); - max_sectors = min(max_sectors, BLK_DEF_MAX_SECTORS); + + if (limits->max_user_sectors) + max_sectors = min(max_sectors, limits->max_user_sectors); + else + max_sectors = min(max_sectors, BLK_DEF_MAX_SECTORS); + max_sectors = round_down(max_sectors, limits->logical_block_size >> SECTOR_SHIFT); limits->max_sectors = max_sectors; diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 93d9e9c9a6ea..5486b6c57f6b 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -239,19 +239,28 @@ static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page) static ssize_t queue_max_sectors_store(struct request_queue *q, const char *page, size_t count) { - unsigned long max_sectors_kb, + unsigned long var; + unsigned int max_sectors_kb, max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1, page_kb = 1 << (PAGE_SHIFT - 10); - ssize_t ret = queue_var_store(&max_sectors_kb, page, count); + ssize_t ret = queue_var_store(&var, page, count); if (ret < 0) return ret; - max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long) + max_sectors_kb = (unsigned int)var; + max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, q->limits.max_dev_sectors >> 1); - - if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb) - return -EINVAL; + if (max_sectors_kb == 0) { + q->limits.max_user_sectors = 0; + max_sectors_kb = min(max_hw_sectors_kb, + BLK_DEF_MAX_SECTORS >> 1); + } else { + if (max_sectors_kb > max_hw_sectors_kb || + max_sectors_kb < page_kb) + return -EINVAL; + q->limits.max_user_sectors = max_sectors_kb << 1; + } spin_lock_irq(&q->queue_lock); q->limits.max_sectors = max_sectors_kb << 1; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2b85161e2256..b87ed829ab94 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -288,6 +288,7 @@ struct queue_limits { unsigned int max_dev_sectors; unsigned int chunk_sectors; unsigned int max_sectors; + unsigned int max_user_sectors; unsigned int max_segment_size; unsigned int physical_block_size; unsigned int logical_block_size; -- cgit v1.2.3 From fea127b36c93d9afe49561b640fe1fc541dc3ba4 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Tue, 10 Jan 2023 15:36:33 +0100 Subject: block: remove superfluous check for request queue in bdev_is_zoned() Remove the superfluous request queue check in bdev_is_zoned() as bdev_get_queue() can never return NULL. Reviewed-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Bart Van Assche Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Signed-off-by: Pankaj Raghav Link: https://lore.kernel.org/r/20230110143635.77300-2-p.raghav@samsung.com Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b87ed829ab94..0956bc0fb5b0 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1285,12 +1285,7 @@ static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev) static inline bool bdev_is_zoned(struct block_device *bdev) { - struct request_queue *q = bdev_get_queue(bdev); - - if (q) - return blk_queue_is_zoned(q); - - return false; + return blk_queue_is_zoned(bdev_get_queue(bdev)); } static inline bool bdev_op_is_zoned_write(struct block_device *bdev, -- cgit v1.2.3 From e29b210021dcf8e03e0dcc035107afaeb55e6631 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Tue, 10 Jan 2023 15:36:34 +0100 Subject: block: add a new helper bdev_{is_zone_start, offset_from_zone_start} Instead of open coding to check for zone start, add a helper to improve readability and store the logic in one place. Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Signed-off-by: Pankaj Raghav Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20230110143635.77300-3-p.raghav@samsung.com Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/blk-zoned.c | 4 ++-- include/linux/blkdev.h | 12 ++++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-core.c b/block/blk-core.c index b5098355d8b2..6fa82291210e 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -570,7 +570,7 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, return BLK_STS_NOTSUPP; /* The bio sector must point to the start of a sequential zone */ - if (bio->bi_iter.bi_sector & (bdev_zone_sectors(bio->bi_bdev) - 1) || + if (!bdev_is_zone_start(bio->bi_bdev, bio->bi_iter.bi_sector) || !bio_zone_is_seq(bio)) return BLK_STS_IOERR; diff --git a/block/blk-zoned.c b/block/blk-zoned.c index db829401d8d0..614b575be899 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -277,10 +277,10 @@ int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, return -EINVAL; /* Check alignment (handle eventual smaller last zone) */ - if (sector & (zone_sectors - 1)) + if (!bdev_is_zone_start(bdev, sector)) return -EINVAL; - if ((nr_sectors & (zone_sectors - 1)) && end_sector != capacity) + if (!bdev_is_zone_start(bdev, nr_sectors) && end_sector != capacity) return -EINVAL; /* diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0956bc0fb5b0..7822c6f4c7bd 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1306,6 +1306,18 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev) return q->limits.chunk_sectors; } +static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev, + sector_t sector) +{ + return sector & (bdev_zone_sectors(bdev) - 1); +} + +static inline bool bdev_is_zone_start(struct block_device *bdev, + sector_t sector) +{ + return bdev_offset_from_zone_start(bdev, sector) == 0; +} + static inline int queue_dma_alignment(const struct request_queue *q) { return q ? q->limits.dma_alignment : 511; -- cgit v1.2.3 From d67ea690ce0983dadf59cd06facc18f3acc89cea Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Tue, 10 Jan 2023 15:36:35 +0100 Subject: block: introduce bdev_zone_no helper Add a generic bdev_zone_no() helper to calculate zone number for a given sector in a block device. This helper internally uses disk_zone_no() to find the zone number. Use the helper bdev_zone_no() to calculate nr of zones. This lets us make modifications to the math if needed in one place. Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Bart Van Assche Reviewed-by: Damien Le Moal Signed-off-by: Pankaj Raghav Link: https://lore.kernel.org/r/20230110143635.77300-4-p.raghav@samsung.com Signed-off-by: Jens Axboe --- drivers/nvme/target/zns.c | 3 +-- include/linux/blkdev.h | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index 1254cf57e008..7e4292d88016 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -254,8 +254,7 @@ static unsigned long nvmet_req_nr_zones_from_slba(struct nvmet_req *req) { unsigned int sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba); - return bdev_nr_zones(req->ns->bdev) - - (sect >> ilog2(bdev_zone_sectors(req->ns->bdev))); + return bdev_nr_zones(req->ns->bdev) - bdev_zone_no(req->ns->bdev, sect); } static unsigned long get_nr_zones_from_buf(struct nvmet_req *req, u32 bufsize) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7822c6f4c7bd..89f51d68c68a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1288,6 +1288,11 @@ static inline bool bdev_is_zoned(struct block_device *bdev) return blk_queue_is_zoned(bdev_get_queue(bdev)); } +static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) +{ + return disk_zone_no(bdev->bd_disk, sec); +} + static inline bool bdev_op_is_zoned_write(struct block_device *bdev, blk_opf_t op) { -- cgit v1.2.3 From f1c006f1c6850c14040f8337753a63119bba39b9 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 19 Jan 2023 19:03:50 +0800 Subject: blk-cgroup: synchronize pd_free_fn() from blkg_free_workfn() and blkcg_deactivate_policy() Currently parent pd can be freed before child pd: t1: remove cgroup C1 blkcg_destroy_blkgs blkg_destroy list_del_init(&blkg->q_node) // remove blkg from queue list percpu_ref_kill(&blkg->refcnt) blkg_release call_rcu t2: from t1 __blkg_release blkg_free schedule_work t4: deactivate policy blkcg_deactivate_policy pd_free_fn // parent of C1 is freed first t3: from t2 blkg_free_workfn pd_free_fn If policy(for example, ioc_timer_fn() from iocost) access parent pd from child pd after pd_offline_fn(), then UAF can be triggered. Fix the problem by delaying 'list_del_init(&blkg->q_node)' from blkg_destroy() to blkg_free_workfn(), and using a new disk level mutex to synchronize blkg_free_workfn() and blkcg_deactivate_policy(). Signed-off-by: Yu Kuai Acked-by: Tejun Heo Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230119110350.2287325-4-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 35 +++++++++++++++++++++++++++++------ include/linux/blkdev.h | 1 + 2 files changed, 30 insertions(+), 6 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 75f3c4460715..cb110fc51940 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -118,16 +118,32 @@ static void blkg_free_workfn(struct work_struct *work) { struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, free_work); + struct request_queue *q = blkg->q; int i; + /* + * pd_free_fn() can also be called from blkcg_deactivate_policy(), + * in order to make sure pd_free_fn() is called in order, the deletion + * of the list blkg->q_node is delayed to here from blkg_destroy(), and + * blkcg_mutex is used to synchronize blkg_free_workfn() and + * blkcg_deactivate_policy(). + */ + if (q) + mutex_lock(&q->blkcg_mutex); + for (i = 0; i < BLKCG_MAX_POLS; i++) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); if (blkg->parent) blkg_put(blkg->parent); - if (blkg->q) - blk_put_queue(blkg->q); + + if (q) { + list_del_init(&blkg->q_node); + mutex_unlock(&q->blkcg_mutex); + blk_put_queue(q); + } + free_percpu(blkg->iostat_cpu); percpu_ref_exit(&blkg->refcnt); kfree(blkg); @@ -462,9 +478,14 @@ static void blkg_destroy(struct blkcg_gq *blkg) lockdep_assert_held(&blkg->q->queue_lock); lockdep_assert_held(&blkcg->lock); - /* Something wrong if we are trying to remove same group twice */ - WARN_ON_ONCE(list_empty(&blkg->q_node)); - WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); + /* + * blkg stays on the queue list until blkg_free_workfn(), see details in + * blkg_free_workfn(), hence this function can be called from + * blkcg_destroy_blkgs() first and again from blkg_destroy_all() before + * blkg_free_workfn(). + */ + if (hlist_unhashed(&blkg->blkcg_node)) + return; for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -479,7 +500,6 @@ static void blkg_destroy(struct blkcg_gq *blkg) blkg->online = false; radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); - list_del_init(&blkg->q_node); hlist_del_init_rcu(&blkg->blkcg_node); /* @@ -1280,6 +1300,7 @@ int blkcg_init_disk(struct gendisk *disk) int ret; INIT_LIST_HEAD(&q->blkg_list); + mutex_init(&q->blkcg_mutex); new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) @@ -1520,6 +1541,7 @@ void blkcg_deactivate_policy(struct request_queue *q, if (queue_is_mq(q)) blk_mq_freeze_queue(q); + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); __clear_bit(pol->plid, q->blkcg_pols); @@ -1538,6 +1560,7 @@ void blkcg_deactivate_policy(struct request_queue *q, } spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 89f51d68c68a..b9637d63e6f0 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -485,6 +485,7 @@ struct request_queue { DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); struct blkcg_gq *root_blkg; struct list_head blkg_list; + struct mutex blkcg_mutex; #endif struct queue_limits limits; -- cgit v1.2.3 From 3222d8c2a7f888bf38b845b125e9470b12108a4d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 25 Jan 2023 14:34:36 +0100 Subject: block: remove ->rw_page The ->rw_page method is a special purpose bypass of the usual bio handling path that is limited to single-page reads and writes and synchronous which causes a lot of extra code in the drivers, callers and the block layer. The only remaining user is the MM swap code. Switch that swap code to simply submit a single-vec on-stack bio an synchronously wait on it based on a newly added QUEUE_FLAG_SYNCHRONOUS flag set by the drivers that currently implement ->rw_page instead. While this touches one extra cache line and executes extra code, it simplifies the block layer and drivers and ensures that all feastures are properly supported by all drivers, e.g. right now ->rw_page bypassed cgroup writeback entirely. [akpm@linux-foundation.org: fix comment typo, per Dan] Link: https://lkml.kernel.org/r/20230125133436.447864-8-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Cc: Dave Jiang Cc: Ira Weiny Cc: Jens Axboe Cc: Keith Busch Cc: Minchan Kim Cc: Sergey Senozhatsky Cc: Vishal Verma Signed-off-by: Andrew Morton --- block/bdev.c | 78 ------------------------------------------- drivers/block/brd.c | 15 +-------- drivers/block/zram/zram_drv.c | 61 +-------------------------------- drivers/nvdimm/btt.c | 16 +-------- drivers/nvdimm/pmem.c | 24 +------------ include/linux/blkdev.h | 12 ++++--- mm/page_io.c | 53 +++++++++++++++++------------ mm/swapfile.c | 2 +- 8 files changed, 44 insertions(+), 217 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/bdev.c b/block/bdev.c index edc110d90df4..1795c7d4b99e 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -304,84 +304,6 @@ out: } EXPORT_SYMBOL(thaw_bdev); -/** - * bdev_read_page() - Start reading a page from a block device - * @bdev: The device to read the page from - * @sector: The offset on the device to read the page to (need not be aligned) - * @page: The page to read - * - * On entry, the page should be locked. It will be unlocked when the page - * has been read. If the block driver implements rw_page synchronously, - * that will be true on exit from this function, but it need not be. - * - * Errors returned by this function are usually "soft", eg out of memory, or - * queue full; callers should try a different route to read this page rather - * than propagate an error back up the stack. - * - * Return: negative errno if an error occurs, 0 if submission was successful. - */ -int bdev_read_page(struct block_device *bdev, sector_t sector, - struct page *page) -{ - const struct block_device_operations *ops = bdev->bd_disk->fops; - int result = -EOPNOTSUPP; - - if (!ops->rw_page || bdev_get_integrity(bdev)) - return result; - - result = blk_queue_enter(bdev_get_queue(bdev), 0); - if (result) - return result; - result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, - REQ_OP_READ); - blk_queue_exit(bdev_get_queue(bdev)); - return result; -} - -/** - * bdev_write_page() - Start writing a page to a block device - * @bdev: The device to write the page to - * @sector: The offset on the device to write the page to (need not be aligned) - * @page: The page to write - * @wbc: The writeback_control for the write - * - * On entry, the page should be locked and not currently under writeback. - * On exit, if the write started successfully, the page will be unlocked and - * under writeback. If the write failed already (eg the driver failed to - * queue the page to the device), the page will still be locked. If the - * caller is a ->writepage implementation, it will need to unlock the page. - * - * Errors returned by this function are usually "soft", eg out of memory, or - * queue full; callers should try a different route to write this page rather - * than propagate an error back up the stack. - * - * Return: negative errno if an error occurs, 0 if submission was successful. - */ -int bdev_write_page(struct block_device *bdev, sector_t sector, - struct page *page, struct writeback_control *wbc) -{ - int result; - const struct block_device_operations *ops = bdev->bd_disk->fops; - - if (!ops->rw_page || bdev_get_integrity(bdev)) - return -EOPNOTSUPP; - result = blk_queue_enter(bdev_get_queue(bdev), 0); - if (result) - return result; - - set_page_writeback(page); - result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, - REQ_OP_WRITE); - if (result) { - end_page_writeback(page); - } else { - clean_page_buffers(page); - unlock_page(page); - } - blk_queue_exit(bdev_get_queue(bdev)); - return result; -} - /* * pseudo-fs */ diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 20acc4a1fd6d..37dce184eb56 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -309,23 +309,9 @@ static void brd_submit_bio(struct bio *bio) bio_endio(bio); } -static int brd_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, enum req_op op) -{ - struct brd_device *brd = bdev->bd_disk->private_data; - int err; - - if (PageTransHuge(page)) - return -ENOTSUPP; - err = brd_do_bvec(brd, page, PAGE_SIZE, 0, op, sector); - page_endio(page, op_is_write(op), err); - return err; -} - static const struct block_device_operations brd_fops = { .owner = THIS_MODULE, .submit_bio = brd_submit_bio, - .rw_page = brd_rw_page, }; /* @@ -411,6 +397,7 @@ static int brd_alloc(int i) /* Tell the block layer that this is not a rotational device */ blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); + blk_queue_flag_set(QUEUE_FLAG_SYNCHRONOUS, disk->queue); blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, disk->queue); err = add_disk(disk); if (err) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 5d1088a645e3..25526707f607 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1453,10 +1453,6 @@ static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, /* Slot should be unlocked before the function call */ zram_slot_unlock(zram, index); - /* A null bio means rw_page was used, we must fallback to bio */ - if (!bio) - return -EOPNOTSUPP; - ret = zram_bvec_read_from_bdev(zram, page, index, bio, partial_io); } @@ -2081,61 +2077,6 @@ static void zram_slot_free_notify(struct block_device *bdev, zram_slot_unlock(zram, index); } -static int zram_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, enum req_op op) -{ - int offset, ret; - u32 index; - struct zram *zram; - struct bio_vec bv; - unsigned long start_time; - - if (PageTransHuge(page)) - return -ENOTSUPP; - zram = bdev->bd_disk->private_data; - - if (!valid_io_request(zram, sector, PAGE_SIZE)) { - atomic64_inc(&zram->stats.invalid_io); - ret = -EINVAL; - goto out; - } - - index = sector >> SECTORS_PER_PAGE_SHIFT; - offset = (sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; - - bv.bv_page = page; - bv.bv_len = PAGE_SIZE; - bv.bv_offset = 0; - - start_time = bdev_start_io_acct(bdev->bd_disk->part0, - SECTORS_PER_PAGE, op, jiffies); - ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL); - bdev_end_io_acct(bdev->bd_disk->part0, op, start_time); -out: - /* - * If I/O fails, just return error(ie, non-zero) without - * calling page_endio. - * It causes resubmit the I/O with bio request by upper functions - * of rw_page(e.g., swap_readpage, __swap_writepage) and - * bio->bi_end_io does things to handle the error - * (e.g., SetPageError, set_page_dirty and extra works). - */ - if (unlikely(ret < 0)) - return ret; - - switch (ret) { - case 0: - page_endio(page, op_is_write(op), 0); - break; - case 1: - ret = 0; - break; - default: - WARN_ON(1); - } - return ret; -} - static void zram_destroy_comps(struct zram *zram) { u32 prio; @@ -2290,7 +2231,6 @@ static const struct block_device_operations zram_devops = { .open = zram_open, .submit_bio = zram_submit_bio, .swap_slot_free_notify = zram_slot_free_notify, - .rw_page = zram_rw_page, .owner = THIS_MODULE }; @@ -2389,6 +2329,7 @@ static int zram_add(void) set_capacity(zram->disk, 0); /* zram devices sort of resembles non-rotational disks */ blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue); + blk_queue_flag_set(QUEUE_FLAG_SYNCHRONOUS, zram->disk->queue); blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue); /* diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 0297b7882e33..d5593b0dc700 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1482,20 +1482,6 @@ static void btt_submit_bio(struct bio *bio) bio_endio(bio); } -static int btt_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, enum req_op op) -{ - struct btt *btt = bdev->bd_disk->private_data; - int rc; - - rc = btt_do_bvec(btt, NULL, page, thp_size(page), 0, op, sector); - if (rc == 0) - page_endio(page, op_is_write(op), 0); - - return rc; -} - - static int btt_getgeo(struct block_device *bd, struct hd_geometry *geo) { /* some standard values */ @@ -1508,7 +1494,6 @@ static int btt_getgeo(struct block_device *bd, struct hd_geometry *geo) static const struct block_device_operations btt_fops = { .owner = THIS_MODULE, .submit_bio = btt_submit_bio, - .rw_page = btt_rw_page, .getgeo = btt_getgeo, }; @@ -1530,6 +1515,7 @@ static int btt_blk_init(struct btt *btt) blk_queue_logical_block_size(btt->btt_disk->queue, btt->sector_size); blk_queue_max_hw_sectors(btt->btt_disk->queue, UINT_MAX); blk_queue_flag_set(QUEUE_FLAG_NONROT, btt->btt_disk->queue); + blk_queue_flag_set(QUEUE_FLAG_SYNCHRONOUS, btt->btt_disk->queue); if (btt_meta_size(btt)) { rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt)); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 96e6e9a5f235..ceea55f621cc 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -238,28 +238,6 @@ static void pmem_submit_bio(struct bio *bio) bio_endio(bio); } -static int pmem_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, enum req_op op) -{ - struct pmem_device *pmem = bdev->bd_disk->private_data; - blk_status_t rc; - - if (op_is_write(op)) - rc = pmem_do_write(pmem, page, 0, sector, thp_size(page)); - else - rc = pmem_do_read(pmem, page, 0, sector, thp_size(page)); - /* - * The ->rw_page interface is subtle and tricky. The core - * retries on any error, so we can only invoke page_endio() in - * the successful completion case. Otherwise, we'll see crashes - * caused by double completion. - */ - if (rc == 0) - page_endio(page, op_is_write(op), 0); - - return blk_status_to_errno(rc); -} - /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */ __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, void **kaddr, @@ -310,7 +288,6 @@ __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff, static const struct block_device_operations pmem_fops = { .owner = THIS_MODULE, .submit_bio = pmem_submit_bio, - .rw_page = pmem_rw_page, }; static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, @@ -565,6 +542,7 @@ static int pmem_attach_disk(struct device *dev, blk_queue_logical_block_size(q, pmem_sector_size(ndns)); blk_queue_max_hw_sectors(q, UINT_MAX); blk_queue_flag_set(QUEUE_FLAG_NONROT, q); + blk_queue_flag_set(QUEUE_FLAG_SYNCHRONOUS, q); if (pmem->pfn_flags & PFN_MAP) blk_queue_flag_set(QUEUE_FLAG_DAX, q); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 43d4e073b111..c5e59965b145 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -554,6 +554,7 @@ struct request_queue { #define QUEUE_FLAG_IO_STAT 7 /* do disk/partitions IO accounting */ #define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */ #define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */ +#define QUEUE_FLAG_SYNCHRONOUS 11 /* always completes in submit context */ #define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */ #define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */ #define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */ @@ -1250,6 +1251,12 @@ static inline bool bdev_nonrot(struct block_device *bdev) return blk_queue_nonrot(bdev_get_queue(bdev)); } +static inline bool bdev_synchronous(struct block_device *bdev) +{ + return test_bit(QUEUE_FLAG_SYNCHRONOUS, + &bdev_get_queue(bdev)->queue_flags); +} + static inline bool bdev_stable_writes(struct block_device *bdev) { return test_bit(QUEUE_FLAG_STABLE_WRITES, @@ -1382,7 +1389,6 @@ struct block_device_operations { unsigned int flags); int (*open) (struct block_device *, fmode_t); void (*release) (struct gendisk *, fmode_t); - int (*rw_page)(struct block_device *, sector_t, struct page *, enum req_op); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); unsigned int (*check_events) (struct gendisk *disk, @@ -1417,10 +1423,6 @@ extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t, #define blkdev_compat_ptr_ioctl NULL #endif -extern int bdev_read_page(struct block_device *, sector_t, struct page *); -extern int bdev_write_page(struct block_device *, sector_t, struct page *, - struct writeback_control *); - static inline void blk_wake_io_task(struct task_struct *waiter) { /* diff --git a/mm/page_io.c b/mm/page_io.c index 0a1a3b831344..a805117f7fd7 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -27,7 +27,7 @@ #include #include "swap.h" -static void end_swap_bio_write(struct bio *bio) +static void __end_swap_bio_write(struct bio *bio) { struct page *page = bio_first_page_all(bio); @@ -48,6 +48,11 @@ static void end_swap_bio_write(struct bio *bio) ClearPageReclaim(page); } end_page_writeback(page); +} + +static void end_swap_bio_write(struct bio *bio) +{ + __end_swap_bio_write(bio); bio_put(bio); } @@ -326,15 +331,31 @@ static void swap_writepage_fs(struct page *page, struct writeback_control *wbc) *wbc->swap_plug = sio; } -static void swap_writepage_bdev(struct page *page, +static void swap_writepage_bdev_sync(struct page *page, struct writeback_control *wbc, struct swap_info_struct *sis) { - struct bio *bio; + struct bio_vec bv; + struct bio bio; - if (!bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc)) { - count_swpout_vm_event(page); - return; - } + bio_init(&bio, sis->bdev, &bv, 1, + REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc)); + bio.bi_iter.bi_sector = swap_page_sector(page); + bio_add_page(&bio, page, thp_size(page), 0); + + bio_associate_blkg_from_page(&bio, page); + count_swpout_vm_event(page); + + set_page_writeback(page); + unlock_page(page); + + submit_bio_wait(&bio); + __end_swap_bio_write(&bio); +} + +static void swap_writepage_bdev_async(struct page *page, + struct writeback_control *wbc, struct swap_info_struct *sis) +{ + struct bio *bio; bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc), @@ -362,8 +383,10 @@ void __swap_writepage(struct page *page, struct writeback_control *wbc) */ if (data_race(sis->flags & SWP_FS_OPS)) swap_writepage_fs(page, wbc); + else if (sis->flags & SWP_SYNCHRONOUS_IO) + swap_writepage_bdev_sync(page, wbc, sis); else - swap_writepage_bdev(page, wbc, sis); + swap_writepage_bdev_async(page, wbc, sis); } void swap_write_unplug(struct swap_iocb *sio) @@ -447,12 +470,6 @@ static void swap_readpage_bdev_sync(struct page *page, struct bio_vec bv; struct bio bio; - if ((sis->flags & SWP_SYNCHRONOUS_IO) && - !bdev_read_page(sis->bdev, swap_page_sector(page), page)) { - count_vm_event(PSWPIN); - return; - } - bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ); bio.bi_iter.bi_sector = swap_page_sector(page); bio_add_page(&bio, page, thp_size(page), 0); @@ -472,12 +489,6 @@ static void swap_readpage_bdev_async(struct page *page, { struct bio *bio; - if ((sis->flags & SWP_SYNCHRONOUS_IO) && - !bdev_read_page(sis->bdev, swap_page_sector(page), page)) { - count_vm_event(PSWPIN); - return; - } - bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); bio->bi_iter.bi_sector = swap_page_sector(page); bio->bi_end_io = end_swap_bio_read; @@ -513,7 +524,7 @@ void swap_readpage(struct page *page, bool synchronous, struct swap_iocb **plug) unlock_page(page); } else if (data_race(sis->flags & SWP_FS_OPS)) { swap_readpage_fs(page, plug); - } else if (synchronous) { + } else if (synchronous || (sis->flags & SWP_SYNCHRONOUS_IO)) { swap_readpage_bdev_sync(page, sis); } else { swap_readpage_bdev_async(page, sis); diff --git a/mm/swapfile.c b/mm/swapfile.c index af151679d13a..888aed774fb6 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3071,7 +3071,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) if (p->bdev && bdev_stable_writes(p->bdev)) p->flags |= SWP_STABLE_WRITES; - if (p->bdev && p->bdev->bd_disk->fops->rw_page) + if (p->bdev && bdev_synchronous(p->bdev)) p->flags |= SWP_SYNCHRONOUS_IO; if (p->bdev && bdev_nonrot(p->bdev)) { -- cgit v1.2.3 From 3f13ab7c80fdb0ada86a8e3e818960bc1ccbaa59 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 3 Feb 2023 16:04:00 +0100 Subject: blk-cgroup: move the cgroup information to struct gendisk cgroup information only makes sense on a live gendisk that allows file system I/O (which includes the raw block device). So move over the cgroup related members. Signed-off-by: Christoph Hellwig Reviewed-by: Andreas Herrmann Acked-by: Tejun Heo Link: https://lore.kernel.org/r/20230203150400.3199230-20-hch@lst.de Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 4 +-- block/blk-cgroup.c | 66 +++++++++++++++++++++++++------------------------- block/blk-cgroup.h | 4 +-- block/blk-iolatency.c | 2 +- block/blk-throttle.c | 16 +++++++----- include/linux/blkdev.h | 12 ++++----- 6 files changed, 54 insertions(+), 50 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 37333c164ed4..4fdbbec71647 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -1001,7 +1001,7 @@ void bfq_end_wr_async(struct bfq_data *bfqd) { struct blkcg_gq *blkg; - list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { + list_for_each_entry(blkg, &bfqd->queue->disk->blkg_list, entry) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); bfq_end_wr_async_queues(bfqd, bfqg); @@ -1295,7 +1295,7 @@ struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) if (ret) return NULL; - return blkg_to_bfqg(bfqd->queue->root_blkg); + return blkg_to_bfqg(bfqd->queue->disk->root_blkg); } struct blkcg_policy blkcg_policy_bfq = { diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 45a683e88bca..8faeca6022be 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -108,10 +108,10 @@ static struct cgroup_subsys_state *blkcg_css(void) return task_css(current, io_cgrp_id); } -static bool blkcg_policy_enabled(struct request_queue *q, +static bool blkcg_policy_enabled(struct gendisk *disk, const struct blkcg_policy *pol) { - return pol && test_bit(pol->plid, q->blkcg_pols); + return pol && test_bit(pol->plid, disk->blkcg_pols); } static void blkg_free(struct blkcg_gq *blkg) @@ -121,18 +121,18 @@ static void blkg_free(struct blkcg_gq *blkg) /* * pd_free_fn() can also be called from blkcg_deactivate_policy(), * in order to make sure pd_free_fn() is called in order, the deletion - * of the list blkg->q_node is delayed to here from blkg_destroy(), and + * of the list blkg->entry is delayed to here from blkg_destroy(), and * blkcg_mutex is used to synchronize blkg_free_workfn() and * blkcg_deactivate_policy(). */ - mutex_lock(&blkg->disk->queue->blkcg_mutex); + mutex_lock(&blkg->disk->blkcg_mutex); for (i = 0; i < BLKCG_MAX_POLS; i++) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); if (blkg->parent) blkg_put(blkg->parent); - list_del_init(&blkg->q_node); - mutex_unlock(&blkg->disk->queue->blkcg_mutex); + list_del_init(&blkg->entry); + mutex_unlock(&blkg->disk->blkcg_mutex); put_disk(blkg->disk); free_percpu(blkg->iostat_cpu); @@ -256,7 +256,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, get_device(disk_to_dev(disk)); blkg->disk = disk; - INIT_LIST_HEAD(&blkg->q_node); + INIT_LIST_HEAD(&blkg->entry); spin_lock_init(&blkg->async_bio_lock); bio_list_init(&blkg->async_bios); INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); @@ -272,7 +272,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd; - if (!blkcg_policy_enabled(disk->queue, pol)) + if (!blkcg_policy_enabled(disk, pol)) continue; /* alloc per-policy data and attach it to blkg */ @@ -358,7 +358,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); if (likely(!ret)) { hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); - list_add(&blkg->q_node, &disk->queue->blkg_list); + list_add(&blkg->entry, &disk->blkg_list); for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -430,7 +430,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, while (true) { struct blkcg *pos = blkcg; struct blkcg *parent = blkcg_parent(blkcg); - struct blkcg_gq *ret_blkg = q->root_blkg; + struct blkcg_gq *ret_blkg = disk->root_blkg; while (parent) { blkg = blkg_lookup(parent, disk); @@ -512,7 +512,7 @@ static void blkg_destroy_all(struct gendisk *disk) restart: spin_lock_irq(&q->queue_lock); - list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { + list_for_each_entry_safe(blkg, n, &disk->blkg_list, entry) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -531,7 +531,7 @@ restart: } } - q->root_blkg = NULL; + disk->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); } @@ -606,7 +606,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { spin_lock_irq(&blkg->disk->queue->queue_lock); - if (blkcg_policy_enabled(blkg->disk->queue, pol)) + if (blkcg_policy_enabled(blkg->disk, pol)) total += prfill(sf, blkg->pd[pol->plid], data); spin_unlock_irq(&blkg->disk->queue->queue_lock); } @@ -714,7 +714,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(q, pol)) { + if (!blkcg_policy_enabled(disk, pol)) { ret = -EOPNOTSUPP; goto fail_unlock; } @@ -757,7 +757,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(q, pol)) { + if (!blkcg_policy_enabled(disk, pol)) { blkg_free(new_blkg); ret = -EOPNOTSUPP; goto fail_preloaded; @@ -937,7 +937,7 @@ static void blkcg_fill_root_iostats(void) class_dev_iter_init(&iter, &block_class, NULL, &disk_type); while ((dev = class_dev_iter_next(&iter))) { struct block_device *bdev = dev_to_bdev(dev); - struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; + struct blkcg_gq *blkg = bdev->bd_disk->root_blkg; struct blkg_iostat tmp; int cpu; unsigned long flags; @@ -1284,8 +1284,8 @@ int blkcg_init_disk(struct gendisk *disk) bool preloaded; int ret; - INIT_LIST_HEAD(&q->blkg_list); - mutex_init(&q->blkcg_mutex); + INIT_LIST_HEAD(&disk->blkg_list); + mutex_init(&disk->blkcg_mutex); new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) @@ -1299,7 +1299,7 @@ int blkcg_init_disk(struct gendisk *disk) blkg = blkg_create(&blkcg_root, disk, new_blkg); if (IS_ERR(blkg)) goto err_unlock; - q->root_blkg = blkg; + disk->root_blkg = blkg; spin_unlock_irq(&q->queue_lock); if (preloaded) @@ -1412,7 +1412,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) struct blkcg_gq *blkg, *pinned_blkg = NULL; int ret; - if (blkcg_policy_enabled(q, pol)) + if (blkcg_policy_enabled(disk, pol)) return 0; if (queue_is_mq(q)) @@ -1421,7 +1421,7 @@ retry: spin_lock_irq(&q->queue_lock); /* blkg_list is pushed at the head, reverse walk to allocate parents first */ - list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { + list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) { struct blkg_policy_data *pd; if (blkg->pd[pol->plid]) @@ -1466,16 +1466,16 @@ retry: /* all allocated, init in the same order */ if (pol->pd_init_fn) - list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) + list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) pol->pd_init_fn(blkg->pd[pol->plid]); - list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { + list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) { if (pol->pd_online_fn) pol->pd_online_fn(blkg->pd[pol->plid]); blkg->pd[pol->plid]->online = true; } - __set_bit(pol->plid, q->blkcg_pols); + __set_bit(pol->plid, disk->blkcg_pols); ret = 0; spin_unlock_irq(&q->queue_lock); @@ -1491,7 +1491,7 @@ out: enomem: /* alloc failed, nothing's initialized yet, free everything */ spin_lock_irq(&q->queue_lock); - list_for_each_entry(blkg, &q->blkg_list, q_node) { + list_for_each_entry(blkg, &disk->blkg_list, entry) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1521,18 +1521,18 @@ void blkcg_deactivate_policy(struct gendisk *disk, struct request_queue *q = disk->queue; struct blkcg_gq *blkg; - if (!blkcg_policy_enabled(q, pol)) + if (!blkcg_policy_enabled(disk, pol)) return; if (queue_is_mq(q)) blk_mq_freeze_queue(q); - mutex_lock(&q->blkcg_mutex); + mutex_lock(&disk->blkcg_mutex); spin_lock_irq(&q->queue_lock); - __clear_bit(pol->plid, q->blkcg_pols); + __clear_bit(pol->plid, disk->blkcg_pols); - list_for_each_entry(blkg, &q->blkg_list, q_node) { + list_for_each_entry(blkg, &disk->blkg_list, entry) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1546,7 +1546,7 @@ void blkcg_deactivate_policy(struct gendisk *disk, } spin_unlock_irq(&q->queue_lock); - mutex_unlock(&q->blkcg_mutex); + mutex_unlock(&disk->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q); @@ -1943,7 +1943,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, * Associate @bio with the blkg found by combining the css's blkg and the * request_queue of the @bio. An association failure is handled by walking up * the blkg tree. Therefore, the blkg associated can be anything between @blkg - * and q->root_blkg. This situation only happens when a cgroup is dying and + * and disk->root_blkg. This situation only happens when a cgroup is dying and * then the remaining bios will spill to the closest alive blkg. * * A reference will be taken on the blkg and will be released when @bio is @@ -1958,8 +1958,8 @@ void bio_associate_blkg_from_css(struct bio *bio, if (css && css->parent) { bio->bi_blkg = blkg_tryget_closest(bio, css); } else { - blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg); - bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg; + blkg_get(bio->bi_bdev->bd_disk->root_blkg); + bio->bi_blkg = bio->bi_bdev->bd_disk->root_blkg; } } EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 151f24de2539..e442b406ca0d 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -54,7 +54,7 @@ struct blkg_iostat_set { /* association between a blk cgroup and a request queue */ struct blkcg_gq { struct gendisk *disk; - struct list_head q_node; + struct list_head entry; struct hlist_node blkcg_node; struct blkcg *blkcg; @@ -250,7 +250,7 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, WARN_ON_ONCE(!rcu_read_lock_held()); if (blkcg == &blkcg_root) - return disk->queue->root_blkg; + return disk->root_blkg; blkg = rcu_dereference(blkcg->blkg_hint); if (blkg && blkg->disk == disk) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index bc0d217f5c17..5d5aa1e526b7 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -665,7 +665,7 @@ static void blkiolatency_timer_fn(struct timer_list *t) rcu_read_lock(); blkg_for_each_descendant_pre(blkg, pos_css, - blkiolat->rqos.disk->queue->root_blkg) { + blkiolat->rqos.disk->root_blkg) { struct iolatency_grp *iolat; struct child_latency_info *lat_info; unsigned long flags; diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 74bb1e753ea0..902203bdddb4 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -451,7 +451,8 @@ static void blk_throtl_update_limit_valid(struct throtl_data *td) bool low_valid = false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || @@ -1180,7 +1181,7 @@ static void throtl_pending_timer_fn(struct timer_list *t) spin_lock_irq(&q->queue_lock); - if (!q->root_blkg) + if (!q->disk->root_blkg) goto out_unlock; if (throtl_can_upgrade(td, NULL)) @@ -1322,7 +1323,8 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) * blk-throttle. */ blkg_for_each_descendant_pre(blkg, pos_css, - global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { + global ? tg->td->queue->disk->root_blkg : + tg_to_blkg(tg)) { struct throtl_grp *this_tg = blkg_to_tg(blkg); struct throtl_grp *parent_tg; @@ -1717,7 +1719,7 @@ void blk_throtl_cancel_bios(struct gendisk *disk) * path need RCU protection and to prevent warning from lockdep. */ rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; @@ -1871,7 +1873,8 @@ static bool throtl_can_upgrade(struct throtl_data *td, return false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg == this_tg) @@ -1917,7 +1920,8 @@ static void throtl_upgrade_state(struct throtl_data *td) td->low_upgrade_time = jiffies; td->scale = 0; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b9637d63e6f0..79aec4ebadb9 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -163,6 +163,12 @@ struct gendisk { struct timer_rand_state *random; atomic_t sync_io; /* RAID */ struct disk_events *ev; +#ifdef CONFIG_BLK_CGROUP + DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); + struct blkcg_gq *root_blkg; + struct list_head blkg_list; + struct mutex blkcg_mutex; +#endif /* CONFIG_BLK_CGROUP */ #ifdef CONFIG_BLK_DEV_INTEGRITY struct kobject integrity_kobj; #endif /* CONFIG_BLK_DEV_INTEGRITY */ @@ -481,12 +487,6 @@ struct request_queue { struct blk_mq_tags *sched_shared_tags; struct list_head icq_list; -#ifdef CONFIG_BLK_CGROUP - DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); - struct blkcg_gq *root_blkg; - struct list_head blkg_list; - struct mutex blkcg_mutex; -#endif struct queue_limits limits; -- cgit v1.2.3 From 1231039db31cf0703996d0b1797c2702e25a110a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 14 Feb 2023 19:33:04 +0100 Subject: Revert "blk-cgroup: move the cgroup information to struct gendisk" This reverts commit 3f13ab7c80fdb0ada86a8e3e818960bc1ccbaa59 as a patch it depends on caused a few problems. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230214183308.1658775-2-hch@lst.de Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 4 +-- block/blk-cgroup.c | 66 +++++++++++++++++++++++++------------------------- block/blk-cgroup.h | 4 +-- block/blk-iolatency.c | 2 +- block/blk-throttle.c | 16 +++++------- include/linux/blkdev.h | 12 ++++----- 6 files changed, 50 insertions(+), 54 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 624530643a05..935a497b5ded 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -999,7 +999,7 @@ void bfq_end_wr_async(struct bfq_data *bfqd) { struct blkcg_gq *blkg; - list_for_each_entry(blkg, &bfqd->queue->disk->blkg_list, entry) { + list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); bfq_end_wr_async_queues(bfqd, bfqg); @@ -1293,7 +1293,7 @@ struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) if (ret) return NULL; - return blkg_to_bfqg(bfqd->queue->disk->root_blkg); + return blkg_to_bfqg(bfqd->queue->root_blkg); } struct blkcg_policy blkcg_policy_bfq = { diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 935028912e7a..1653786644ea 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -108,10 +108,10 @@ static struct cgroup_subsys_state *blkcg_css(void) return task_css(current, io_cgrp_id); } -static bool blkcg_policy_enabled(struct gendisk *disk, +static bool blkcg_policy_enabled(struct request_queue *q, const struct blkcg_policy *pol) { - return pol && test_bit(pol->plid, disk->blkcg_pols); + return pol && test_bit(pol->plid, q->blkcg_pols); } static void blkg_free_workfn(struct work_struct *work) @@ -123,18 +123,18 @@ static void blkg_free_workfn(struct work_struct *work) /* * pd_free_fn() can also be called from blkcg_deactivate_policy(), * in order to make sure pd_free_fn() is called in order, the deletion - * of the list blkg->entry is delayed to here from blkg_destroy(), and + * of the list blkg->q_node is delayed to here from blkg_destroy(), and * blkcg_mutex is used to synchronize blkg_free_workfn() and * blkcg_deactivate_policy(). */ - mutex_lock(&blkg->disk->blkcg_mutex); + mutex_lock(&blkg->disk->queue->blkcg_mutex); for (i = 0; i < BLKCG_MAX_POLS; i++) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); if (blkg->parent) blkg_put(blkg->parent); - list_del_init(&blkg->entry); - mutex_unlock(&blkg->disk->blkcg_mutex); + list_del_init(&blkg->q_node); + mutex_unlock(&blkg->disk->queue->blkcg_mutex); put_disk(blkg->disk); free_percpu(blkg->iostat_cpu); @@ -269,7 +269,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, get_device(disk_to_dev(disk)); blkg->disk = disk; - INIT_LIST_HEAD(&blkg->entry); + INIT_LIST_HEAD(&blkg->q_node); spin_lock_init(&blkg->async_bio_lock); bio_list_init(&blkg->async_bios); INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); @@ -285,7 +285,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd; - if (!blkcg_policy_enabled(disk, pol)) + if (!blkcg_policy_enabled(disk->queue, pol)) continue; /* alloc per-policy data and attach it to blkg */ @@ -371,7 +371,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); if (likely(!ret)) { hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); - list_add(&blkg->entry, &disk->blkg_list); + list_add(&blkg->q_node, &disk->queue->blkg_list); for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -444,7 +444,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, while (true) { struct blkcg *pos = blkcg; struct blkcg *parent = blkcg_parent(blkcg); - struct blkcg_gq *ret_blkg = disk->root_blkg; + struct blkcg_gq *ret_blkg = q->root_blkg; while (parent) { blkg = blkg_lookup(parent, disk); @@ -526,7 +526,7 @@ static void blkg_destroy_all(struct gendisk *disk) restart: spin_lock_irq(&q->queue_lock); - list_for_each_entry_safe(blkg, n, &disk->blkg_list, entry) { + list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -545,7 +545,7 @@ restart: } } - disk->root_blkg = NULL; + q->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); } @@ -620,7 +620,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { spin_lock_irq(&blkg->disk->queue->queue_lock); - if (blkcg_policy_enabled(blkg->disk, pol)) + if (blkcg_policy_enabled(blkg->disk->queue, pol)) total += prfill(sf, blkg->pd[pol->plid], data); spin_unlock_irq(&blkg->disk->queue->queue_lock); } @@ -728,7 +728,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(disk, pol)) { + if (!blkcg_policy_enabled(q, pol)) { ret = -EOPNOTSUPP; goto fail_unlock; } @@ -771,7 +771,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(disk, pol)) { + if (!blkcg_policy_enabled(q, pol)) { blkg_free(new_blkg); ret = -EOPNOTSUPP; goto fail_preloaded; @@ -951,7 +951,7 @@ static void blkcg_fill_root_iostats(void) class_dev_iter_init(&iter, &block_class, NULL, &disk_type); while ((dev = class_dev_iter_next(&iter))) { struct block_device *bdev = dev_to_bdev(dev); - struct blkcg_gq *blkg = bdev->bd_disk->root_blkg; + struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; struct blkg_iostat tmp; int cpu; unsigned long flags; @@ -1298,8 +1298,8 @@ int blkcg_init_disk(struct gendisk *disk) bool preloaded; int ret; - INIT_LIST_HEAD(&disk->blkg_list); - mutex_init(&disk->blkcg_mutex); + INIT_LIST_HEAD(&q->blkg_list); + mutex_init(&q->blkcg_mutex); new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) @@ -1313,7 +1313,7 @@ int blkcg_init_disk(struct gendisk *disk) blkg = blkg_create(&blkcg_root, disk, new_blkg); if (IS_ERR(blkg)) goto err_unlock; - disk->root_blkg = blkg; + q->root_blkg = blkg; spin_unlock_irq(&q->queue_lock); if (preloaded) @@ -1426,7 +1426,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) struct blkcg_gq *blkg, *pinned_blkg = NULL; int ret; - if (blkcg_policy_enabled(disk, pol)) + if (blkcg_policy_enabled(q, pol)) return 0; if (queue_is_mq(q)) @@ -1435,7 +1435,7 @@ retry: spin_lock_irq(&q->queue_lock); /* blkg_list is pushed at the head, reverse walk to allocate parents first */ - list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) { + list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { struct blkg_policy_data *pd; if (blkg->pd[pol->plid]) @@ -1480,16 +1480,16 @@ retry: /* all allocated, init in the same order */ if (pol->pd_init_fn) - list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) + list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) pol->pd_init_fn(blkg->pd[pol->plid]); - list_for_each_entry_reverse(blkg, &disk->blkg_list, entry) { + list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { if (pol->pd_online_fn) pol->pd_online_fn(blkg->pd[pol->plid]); blkg->pd[pol->plid]->online = true; } - __set_bit(pol->plid, disk->blkcg_pols); + __set_bit(pol->plid, q->blkcg_pols); ret = 0; spin_unlock_irq(&q->queue_lock); @@ -1505,7 +1505,7 @@ out: enomem: /* alloc failed, nothing's initialized yet, free everything */ spin_lock_irq(&q->queue_lock); - list_for_each_entry(blkg, &disk->blkg_list, entry) { + list_for_each_entry(blkg, &q->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1535,18 +1535,18 @@ void blkcg_deactivate_policy(struct gendisk *disk, struct request_queue *q = disk->queue; struct blkcg_gq *blkg; - if (!blkcg_policy_enabled(disk, pol)) + if (!blkcg_policy_enabled(q, pol)) return; if (queue_is_mq(q)) blk_mq_freeze_queue(q); - mutex_lock(&disk->blkcg_mutex); + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); - __clear_bit(pol->plid, disk->blkcg_pols); + __clear_bit(pol->plid, q->blkcg_pols); - list_for_each_entry(blkg, &disk->blkg_list, entry) { + list_for_each_entry(blkg, &q->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1560,7 +1560,7 @@ void blkcg_deactivate_policy(struct gendisk *disk, } spin_unlock_irq(&q->queue_lock); - mutex_unlock(&disk->blkcg_mutex); + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q); @@ -1957,7 +1957,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, * Associate @bio with the blkg found by combining the css's blkg and the * request_queue of the @bio. An association failure is handled by walking up * the blkg tree. Therefore, the blkg associated can be anything between @blkg - * and disk->root_blkg. This situation only happens when a cgroup is dying and + * and q->root_blkg. This situation only happens when a cgroup is dying and * then the remaining bios will spill to the closest alive blkg. * * A reference will be taken on the blkg and will be released when @bio is @@ -1972,8 +1972,8 @@ void bio_associate_blkg_from_css(struct bio *bio, if (css && css->parent) { bio->bi_blkg = blkg_tryget_closest(bio, css); } else { - blkg_get(bio->bi_bdev->bd_disk->root_blkg); - bio->bi_blkg = bio->bi_bdev->bd_disk->root_blkg; + blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg); + bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg; } } EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index e442b406ca0d..151f24de2539 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -54,7 +54,7 @@ struct blkg_iostat_set { /* association between a blk cgroup and a request queue */ struct blkcg_gq { struct gendisk *disk; - struct list_head entry; + struct list_head q_node; struct hlist_node blkcg_node; struct blkcg *blkcg; @@ -250,7 +250,7 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, WARN_ON_ONCE(!rcu_read_lock_held()); if (blkcg == &blkcg_root) - return disk->root_blkg; + return disk->queue->root_blkg; blkg = rcu_dereference(blkcg->blkg_hint); if (blkg && blkg->disk == disk) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 5d5aa1e526b7..bc0d217f5c17 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -665,7 +665,7 @@ static void blkiolatency_timer_fn(struct timer_list *t) rcu_read_lock(); blkg_for_each_descendant_pre(blkg, pos_css, - blkiolat->rqos.disk->root_blkg) { + blkiolat->rqos.disk->queue->root_blkg) { struct iolatency_grp *iolat; struct child_latency_info *lat_info; unsigned long flags; diff --git a/block/blk-throttle.c b/block/blk-throttle.c index e7bd7050d684..21c8d5e871ea 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -451,8 +451,7 @@ static void blk_throtl_update_limit_valid(struct throtl_data *td) bool low_valid = false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, - td->queue->disk->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || @@ -1181,7 +1180,7 @@ static void throtl_pending_timer_fn(struct timer_list *t) spin_lock_irq(&q->queue_lock); - if (!q->disk->root_blkg) + if (!q->root_blkg) goto out_unlock; if (throtl_can_upgrade(td, NULL)) @@ -1323,8 +1322,7 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) * blk-throttle. */ blkg_for_each_descendant_pre(blkg, pos_css, - global ? tg->td->queue->disk->root_blkg : - tg_to_blkg(tg)) { + global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { struct throtl_grp *this_tg = blkg_to_tg(blkg); struct throtl_grp *parent_tg; @@ -1719,7 +1717,7 @@ void blk_throtl_cancel_bios(struct gendisk *disk) * path need RCU protection and to prevent warning from lockdep. */ rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, disk->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; @@ -1873,8 +1871,7 @@ static bool throtl_can_upgrade(struct throtl_data *td, return false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, - td->queue->disk->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg == this_tg) @@ -1920,8 +1917,7 @@ static void throtl_upgrade_state(struct throtl_data *td) td->low_upgrade_time = jiffies; td->scale = 0; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, - td->queue->disk->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 79aec4ebadb9..b9637d63e6f0 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -163,12 +163,6 @@ struct gendisk { struct timer_rand_state *random; atomic_t sync_io; /* RAID */ struct disk_events *ev; -#ifdef CONFIG_BLK_CGROUP - DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); - struct blkcg_gq *root_blkg; - struct list_head blkg_list; - struct mutex blkcg_mutex; -#endif /* CONFIG_BLK_CGROUP */ #ifdef CONFIG_BLK_DEV_INTEGRITY struct kobject integrity_kobj; #endif /* CONFIG_BLK_DEV_INTEGRITY */ @@ -487,6 +481,12 @@ struct request_queue { struct blk_mq_tags *sched_shared_tags; struct list_head icq_list; +#ifdef CONFIG_BLK_CGROUP + DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); + struct blkcg_gq *root_blkg; + struct list_head blkg_list; + struct mutex blkcg_mutex; +#endif struct queue_limits limits; -- cgit v1.2.3 From 9e0c7efa5ea231d85c0d41693a5115b3b971717c Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Fri, 3 Feb 2023 11:40:29 +0900 Subject: block: remove more NULL checks after bdev_get_queue() bdev_get_queue() never returns NULL. Several commits [1][2] have been made before to remove such superfluous checks, but some still remained. For places where bdev_get_queue() is called solely for NULL checks, it is removed entirely. [1] commit ec9fd2a13d74 ("blk-lib: don't check bdev_get_queue() NULL check") [2] commit fea127b36c93 ("block: remove superfluous check for request queue in bdev_is_zoned()") Signed-off-by: Juhyung Park Reviewed-by: Pankaj Raghav Link: https://lore.kernel.org/r/20230203024029.48260-1-qkrwngud825@gmail.com Signed-off-by: Jens Axboe --- block/blk-zoned.c | 10 ---------- include/linux/blkdev.h | 7 +------ kernel/trace/blktrace.c | 6 +----- 3 files changed, 2 insertions(+), 21 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 614b575be899..fce9082384d6 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -334,17 +334,12 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, { void __user *argp = (void __user *)arg; struct zone_report_args args; - struct request_queue *q; struct blk_zone_report rep; int ret; if (!argp) return -EINVAL; - q = bdev_get_queue(bdev); - if (!q) - return -ENXIO; - if (!bdev_is_zoned(bdev)) return -ENOTTY; @@ -391,7 +386,6 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { void __user *argp = (void __user *)arg; - struct request_queue *q; struct blk_zone_range zrange; enum req_op op; int ret; @@ -399,10 +393,6 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, if (!argp) return -EINVAL; - q = bdev_get_queue(bdev); - if (!q) - return -ENXIO; - if (!bdev_is_zoned(bdev)) return -ENOTTY; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b9637d63e6f0..89dd9b02b45b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1276,12 +1276,7 @@ static inline bool bdev_nowait(struct block_device *bdev) static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev) { - struct request_queue *q = bdev_get_queue(bdev); - - if (q) - return blk_queue_zoned_model(q); - - return BLK_ZONED_NONE; + return blk_queue_zoned_model(bdev_get_queue(bdev)); } static inline bool bdev_is_zoned(struct block_device *bdev) diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 5743be559415..d5d94510afd3 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -729,14 +729,10 @@ EXPORT_SYMBOL_GPL(blk_trace_startstop); **/ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) { - struct request_queue *q; + struct request_queue *q = bdev_get_queue(bdev); int ret, start = 0; char b[BDEVNAME_SIZE]; - q = bdev_get_queue(bdev); - if (!q) - return -ENXIO; - mutex_lock(&q->debugfs_mutex); switch (cmd) { -- cgit v1.2.3