From d50059ecf078224b00b0b341173fd35dbbd0eea3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2002 16:49:35 -0800 Subject: [PATCH] end_io bouncing o Split blk_queue_bounce() into a slow and fast path. The fast path is inlined, only if we actually need to check the bio for possible bounces (and bounce) do we enter __blk_queue_bounce() slow path. o Fix a nasty bug that could cause corruption for file systems not using PAGE_CACHE_SIZE blok size! We were not correctly setting the 'to' bv_offset correctly. o Add BIO_BOUNCE flag. Later patches will use this for debug checking. --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 42e81a4a0cab..6437522be832 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -268,7 +268,7 @@ extern unsigned long blk_max_low_pfn, blk_max_pfn; #define BLK_BOUNCE_ISA (ISA_DMA_THRESHOLD) extern int init_emergency_isa_pool(void); -void blk_queue_bounce(request_queue_t *q, struct bio **bio); +inline void blk_queue_bounce(request_queue_t *q, struct bio **bio); #define rq_for_each_bio(bio, rq) \ if ((rq->bio)) \ -- cgit v1.2.3 From 0717c0a96379ba634411b99a806587acad9f9fee Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2002 17:15:53 -0800 Subject: [PATCH] make queue prep_rq_fn() a bit more powerful Extend q->prep_rq_fn() to return one of three values: o BLKPREP_OK: request is good, return it o BLKPREP_KILL: request is bad, end it completely o BLKPREP_DEFER: request is good, but we can't take it now We maintain compatability with old prep functions (if any, outside of ide-cd). This change is needed or SCSI to use prep function for command init, if sg table allocation fails we can just defer the request. --- drivers/block/elevator.c | 34 +++++++++++++++++++++------------- include/linux/blkdev.h | 7 +++++++ 2 files changed, 28 insertions(+), 13 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c index ff3825209510..0dd77b16dbe5 100644 --- a/drivers/block/elevator.c +++ b/drivers/block/elevator.c @@ -303,8 +303,14 @@ static inline struct request *__elv_next_request(request_queue_t *q) struct request *elv_next_request(request_queue_t *q) { struct request *rq; + int ret; while ((rq = __elv_next_request(q))) { + /* + * just mark as started even if we don't start it, a request + * that has been delayed should not be passed by new incoming + * requests + */ rq->flags |= REQ_STARTED; if (&rq->queuelist == q->last_merge) @@ -313,20 +319,22 @@ struct request *elv_next_request(request_queue_t *q) if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn) break; - /* - * all ok, break and return it - */ - if (!q->prep_rq_fn(q, rq)) + ret = q->prep_rq_fn(q, rq); + if (ret == BLKPREP_OK) { break; - - /* - * prep said no-go, kill it - */ - blkdev_dequeue_request(rq); - if (end_that_request_first(rq, 0, rq->nr_sectors)) - BUG(); - - end_that_request_last(rq); + } else if (ret == BLKPREP_DEFER) { + rq = NULL; + break; + } else if (ret == BLKPREP_KILL) { + blkdev_dequeue_request(rq); + rq->flags |= REQ_QUIET; + while (end_that_request_first(rq, 0, rq->nr_sectors)) + ; + end_that_request_last(rq); + } else { + printk("%s: bad return=%d\n", __FUNCTION__, ret); + break; + } } return rq; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6437522be832..7ac2b3bb88e5 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -254,6 +254,13 @@ struct request_queue */ #define blk_queue_headactive(q, head_active) +/* + * q->prep_rq_fn return values + */ +#define BLKPREP_OK 0 /* serve it */ +#define BLKPREP_KILL 1 /* fatal error, kill */ +#define BLKPREP_DEFER 2 /* leave on queue */ + extern unsigned long blk_max_low_pfn, blk_max_pfn; /* -- cgit v1.2.3 From ad519c6902fb66990ad206cdf2c38da259f0dee9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2002 17:15:58 -0800 Subject: [PATCH] queue dma alignment Make it possible for a device to specify the dma alignment restrictions it has. This will be used by future infrastructure when mapping in user pages, and allows us to dma to on ATAPI even though user address and length is not sector size aligned. --- drivers/block/ll_rw_blk.c | 17 +++++++++++++++++ include/linux/blkdev.h | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c index e4db6ecf4faf..45d1b8ee1ce6 100644 --- a/drivers/block/ll_rw_blk.c +++ b/drivers/block/ll_rw_blk.c @@ -242,6 +242,7 @@ void blk_queue_make_request(request_queue_t * q, make_request_fn * mfn) q->backing_dev_info.state = 0; blk_queue_max_sectors(q, MAX_SECTORS); blk_queue_hardsect_size(q, 512); + blk_queue_dma_alignment(q, 511); /* * by default assume old behaviour and bounce for any highmem page @@ -408,6 +409,21 @@ void blk_queue_segment_boundary(request_queue_t *q, unsigned long mask) q->seg_boundary_mask = mask; } +/** + * blk_queue_dma_alignment - set dma length and memory alignment + * @q: the request queue for the device + * @dma_mask: alignment mask + * + * description: + * set required memory and length aligment for direct dma transactions. + * this is used when buiding direct io requests for the queue. + * + **/ +void blk_queue_dma_alignment(request_queue_t *q, int mask) +{ + q->dma_alignment = mask; +} + void blk_queue_assign_lock(request_queue_t *q, spinlock_t *lock) { spin_lock_init(lock); @@ -2124,6 +2140,7 @@ EXPORT_SYMBOL(blk_queue_max_hw_segments); EXPORT_SYMBOL(blk_queue_max_segment_size); EXPORT_SYMBOL(blk_queue_hardsect_size); EXPORT_SYMBOL(blk_queue_segment_boundary); +EXPORT_SYMBOL(blk_queue_dma_alignment); EXPORT_SYMBOL(blk_rq_map_sg); EXPORT_SYMBOL(blk_nohighio); EXPORT_SYMBOL(blk_dump_rq_flags); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7ac2b3bb88e5..cd9b016a8993 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -215,6 +215,7 @@ struct request_queue unsigned int max_segment_size; unsigned long seg_boundary_mask; + unsigned int dma_alignment; wait_queue_head_t queue_wait; @@ -346,6 +347,7 @@ extern void blk_queue_segment_boundary(request_queue_t *, unsigned long); extern void blk_queue_assign_lock(request_queue_t *, spinlock_t *); extern void blk_queue_prep_rq(request_queue_t *, prep_rq_fn *pfn); extern void blk_queue_merge_bvec(request_queue_t *, merge_bvec_fn *); +extern void blk_queue_dma_alignment(request_queue_t *, int); extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev); extern int blk_rq_map_sg(request_queue_t *, struct request *, struct scatterlist *); @@ -392,6 +394,21 @@ static inline int bdev_hardsect_size(struct block_device *bdev) return queue_hardsect_size(bdev_get_queue(bdev)); } +static inline int queue_dma_alignment(request_queue_t *q) +{ + int retval = 511; + + if (q && q->dma_alignment) + retval = q->dma_alignment; + + return retval; +} + +static inline int bdev_dma_aligment(struct block_device *bdev) +{ + return queue_dma_alignment(bdev_get_queue(bdev)); +} + #define blk_finished_io(nsects) do { } while (0) #define blk_started_io(nsects) do { } while (0) -- cgit v1.2.3 From 6f04a530fa6fbcc8ad1329ce2c6b135a7b7742b6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2002 17:39:19 -0800 Subject: [PATCH] request references and list deletion/insertion checking o Always use list_del_init() on request queuelist, this allows us to sanity check the integrity of the request on insertion and removal. So we can complain loudly instead of silently corrupting memory. o Add references to requests. This is cheap, since we dont have to use an atomic variable for it (all puts are inside queue lock). We've had a bug in IDE for years where we want to inspect request state after io completion, but this is not possible to do race free right now. REQ_BLOCK_PC and sgio will need this too, for checking io residual etc. This is not just a theoretical race, I've seen it happen. --- drivers/block/ll_rw_blk.c | 51 ++++++++++++++++++++++++++++++++++++----------- include/linux/blk.h | 4 +++- include/linux/blkdev.h | 2 ++ 3 files changed, 44 insertions(+), 13 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c index edf890268759..1c4e78b2928d 100644 --- a/drivers/block/ll_rw_blk.c +++ b/drivers/block/ll_rw_blk.c @@ -565,7 +565,7 @@ void blk_queue_end_tag(request_queue_t *q, struct request *rq) return; } - list_del(&rq->queuelist); + list_del_init(&rq->queuelist); rq->flags &= ~REQ_QUEUED; rq->tag = -1; @@ -649,7 +649,7 @@ void blk_queue_invalidate_tags(request_queue_t *q) if (rq->tag == -1) { printk("bad tag found on list\n"); - list_del(&rq->queuelist); + list_del_init(&rq->queuelist); rq->flags &= ~REQ_QUEUED; } else blk_queue_end_tag(q, rq); @@ -1132,7 +1132,7 @@ static int __blk_cleanup_queue(struct request_list *list) while (!list_empty(head)) { rq = list_entry(head->next, struct request, queuelist); - list_del(&rq->queuelist); + list_del_init(&rq->queuelist); kmem_cache_free(request_cachep, rq); i++; } @@ -1292,13 +1292,20 @@ static struct request *get_request(request_queue_t *q, int rw) if (!list_empty(&rl->free)) { rq = blkdev_free_rq(&rl->free); - list_del(&rq->queuelist); + list_del_init(&rq->queuelist); + rq->ref_count = 1; rl->count--; if (rl->count < queue_congestion_on_threshold()) set_queue_congested(q, rw); rq->flags = 0; rq->rq_status = RQ_ACTIVE; + rq->errors = 0; rq->special = NULL; + rq->buffer = NULL; + rq->data = NULL; + rq->sense = NULL; + rq->waiting = NULL; + rq->bio = rq->biotail = NULL; rq->q = q; rq->rl = rl; } @@ -1497,13 +1504,14 @@ static inline void add_request(request_queue_t * q, struct request * req, __elv_add_request_pos(q, req, insert_here); } -/* - * Must be called with queue lock held and interrupts disabled - */ -void blk_put_request(struct request *req) +void __blk_put_request(request_queue_t *q, struct request *req) { struct request_list *rl = req->rl; - request_queue_t *q = req->q; + + if (unlikely(--req->ref_count)) + return; + if (unlikely(!q)) + return; req->rq_status = RQ_INACTIVE; req->q = NULL; @@ -1516,6 +1524,8 @@ void blk_put_request(struct request *req) if (rl) { int rw = 0; + BUG_ON(!list_empty(&req->queuelist)); + list_add(&req->queuelist, &rl->free); if (rl == &q->rq[WRITE]) @@ -1533,6 +1543,23 @@ void blk_put_request(struct request *req) } } +void blk_put_request(struct request *req) +{ + request_queue_t *q = req->q; + + /* + * if req->q isn't set, this request didnt originate from the + * block layer, so it's safe to just disregard it + */ + if (q) { + unsigned long flags; + + spin_lock_irqsave(q->queue_lock, flags); + __blk_put_request(q, req); + spin_unlock_irqrestore(q->queue_lock, flags); + } +} + /** * blk_congestion_wait - wait for a queue to become uncongested * @rw: READ or WRITE @@ -1591,7 +1618,7 @@ static void attempt_merge(request_queue_t *q, struct request *req, elv_merge_requests(q, req, next); blkdev_dequeue_request(next); - blk_put_request(next); + __blk_put_request(q, next); } } @@ -1784,7 +1811,7 @@ get_rq: add_request(q, req, insert_here); out: if (freereq) - blk_put_request(freereq); + __blk_put_request(q, freereq); spin_unlock_irq(q->queue_lock); return 0; @@ -2069,7 +2096,7 @@ void end_that_request_last(struct request *req) if (req->waiting) complete(req->waiting); - blk_put_request(req); + __blk_put_request(req); } int __init blk_dev_init(void) diff --git a/include/linux/blk.h b/include/linux/blk.h index 775d021167c1..b225609f2ee7 100644 --- a/include/linux/blk.h +++ b/include/linux/blk.h @@ -44,7 +44,9 @@ struct request *elv_next_request(request_queue_t *q); static inline void blkdev_dequeue_request(struct request *req) { - list_del(&req->queuelist); + BUG_ON(list_empty(&req->queuelist)); + + list_del_init(&req->queuelist); if (req->q) elv_remove_request(req->q, req); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index cd9b016a8993..de87dee16f83 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -26,6 +26,8 @@ struct request { struct list_head queuelist; /* looking for ->queue? you must _not_ * access it directly, use * blkdev_dequeue_request! */ + int ref_count; + void *elevator_private; unsigned char cmd[16]; -- cgit v1.2.3