summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2017-01-26 12:40:07 -0700
committerJens Axboe <axboe@fb.com>2017-01-27 08:20:35 -0700
commitc13660a08c8b3bb49def4374bfd414aaaa564662 (patch)
treec7a61741aba75e320838865f176092c07af37f95
parent50e1dab86aa2c10cbca2f754aae9542169403141 (diff)
downloadlinux-0-day-c13660a08c8b3bb49def4374bfd414aaaa564662.tar.gz
blk-mq-sched: change ->dispatch_requests() to ->dispatch_request()
When we invoke dispatch_requests(), the scheduler empties everything into the passed in list. This isn't always a good thing, since it means that we remove items that we could have potentially merged with. Change the function to dispatch single requests at the time. If we do that, we can backoff exactly at the point where the device can't consume more IO, and leave the rest with the scheduler for better merging and future dispatch decision making. Signed-off-by: Jens Axboe <axboe@fb.com> Reviewed-by: Omar Sandoval <osandov@fb.com> Tested-by: Hannes Reinecke <hare@suse.com>
-rw-r--r--block/blk-mq-sched.c23
-rw-r--r--block/blk-mq.c2
-rw-r--r--block/mq-deadline.c10
-rw-r--r--include/linux/elevator.h2
4 files changed, 23 insertions, 14 deletions
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index fcc0e89..c27613d 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -201,15 +201,22 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
* leave them there for as long as we can. Mark the hw queue as
* needing a restart in that case.
*/
- if (list_empty(&rq_list)) {
- if (e && e->type->ops.mq.dispatch_requests)
- e->type->ops.mq.dispatch_requests(hctx, &rq_list);
- else
- blk_mq_flush_busy_ctxs(hctx, &rq_list);
- } else
+ if (!list_empty(&rq_list)) {
blk_mq_sched_mark_restart(hctx);
-
- blk_mq_dispatch_rq_list(hctx, &rq_list);
+ blk_mq_dispatch_rq_list(hctx, &rq_list);
+ } else if (!e || !e->type->ops.mq.dispatch_request) {
+ blk_mq_flush_busy_ctxs(hctx, &rq_list);
+ blk_mq_dispatch_rq_list(hctx, &rq_list);
+ } else {
+ do {
+ struct request *rq;
+
+ rq = e->type->ops.mq.dispatch_request(hctx);
+ if (!rq)
+ break;
+ list_add(&rq->queuelist, &rq_list);
+ } while (blk_mq_dispatch_rq_list(hctx, &rq_list));
+ }
}
void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 21795c6..301ae29 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -998,7 +998,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
*/
if (!list_empty(list)) {
spin_lock(&hctx->lock);
- list_splice(list, &hctx->dispatch);
+ list_splice_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock);
/*
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index a01986d..d93ec71 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -287,14 +287,16 @@ done:
return rq;
}
-static void dd_dispatch_requests(struct blk_mq_hw_ctx *hctx,
- struct list_head *rq_list)
+static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
{
struct deadline_data *dd = hctx->queue->elevator->elevator_data;
+ struct request *rq;
spin_lock(&dd->lock);
- blk_mq_sched_move_to_dispatch(hctx, rq_list, __dd_dispatch_request);
+ rq = __dd_dispatch_request(hctx);
spin_unlock(&dd->lock);
+
+ return rq;
}
static void dd_exit_queue(struct elevator_queue *e)
@@ -517,7 +519,7 @@ static struct elv_fs_entry deadline_attrs[] = {
static struct elevator_type mq_deadline = {
.ops.mq = {
.insert_requests = dd_insert_requests,
- .dispatch_requests = dd_dispatch_requests,
+ .dispatch_request = dd_dispatch_request,
.next_request = elv_rb_latter_request,
.former_request = elv_rb_former_request,
.bio_merge = dd_bio_merge,
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index ecb96fd..b5825c4 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -92,7 +92,7 @@ struct elevator_mq_ops {
struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *);
void (*put_request)(struct request *);
void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
- void (*dispatch_requests)(struct blk_mq_hw_ctx *, struct list_head *);
+ struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
bool (*has_work)(struct blk_mq_hw_ctx *);
void (*completed_request)(struct blk_mq_hw_ctx *, struct request *);
void (*started_request)(struct request *);