summaryrefslogtreecommitdiff
path: root/io_uring/net.c
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring/net.c')
-rw-r--r--io_uring/net.c160
1 files changed, 75 insertions, 85 deletions
diff --git a/io_uring/net.c b/io_uring/net.c
index d69f2afa4f7a..f99b90c762fc 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -10,6 +10,7 @@
#include <uapi/linux/io_uring.h>
+#include "filetable.h"
#include "io_uring.h"
#include "kbuf.h"
#include "alloc_cache.h"
@@ -178,10 +179,8 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)
io_vec_free(&hdr->vec);
- if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) {
- req->async_data = NULL;
- req->flags &= ~(REQ_F_ASYNC_DATA|REQ_F_NEED_CLEANUP);
- }
+ if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))
+ io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
}
static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
@@ -433,7 +432,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (req->opcode == IORING_OP_SENDMSG)
return -EINVAL;
sr->msg_flags |= MSG_WAITALL;
- req->buf_list = NULL;
req->flags |= REQ_F_MULTISHOT;
}
@@ -494,29 +492,29 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
return nbufs;
}
-static int io_net_kbuf_recyle(struct io_kiocb *req,
+static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
struct io_async_msghdr *kmsg, int len)
{
req->flags |= REQ_F_BL_NO_RECYCLE;
if (req->flags & REQ_F_BUFFERS_COMMIT)
- io_kbuf_commit(req, req->buf_list, len, io_bundle_nbufs(kmsg, len));
+ io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));
return IOU_RETRY;
}
-static inline bool io_send_finish(struct io_kiocb *req, int *ret,
+static inline bool io_send_finish(struct io_kiocb *req,
struct io_async_msghdr *kmsg,
- unsigned issue_flags)
+ struct io_br_sel *sel)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
- bool bundle_finished = *ret <= 0;
+ bool bundle_finished = sel->val <= 0;
unsigned int cflags;
if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
- cflags = io_put_kbuf(req, *ret, issue_flags);
+ cflags = io_put_kbuf(req, sel->val, sel->buf_list);
goto finish;
}
- cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret), issue_flags);
+ cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));
if (bundle_finished || req->flags & REQ_F_BL_EMPTY)
goto finish;
@@ -525,15 +523,15 @@ static inline bool io_send_finish(struct io_kiocb *req, int *ret,
* Fill CQE for this receive and see if we should keep trying to
* receive from this socket.
*/
- if (io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) {
+ if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
io_mshot_prep_retry(req, kmsg);
return false;
}
/* Otherwise stop bundle and use the current result. */
finish:
- io_req_set_res(req, *ret, cflags);
- *ret = IOU_COMPLETE;
+ io_req_set_res(req, sel->val, cflags);
+ sel->val = IOU_COMPLETE;
return true;
}
@@ -571,7 +569,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
kmsg->msg.msg_controllen = 0;
kmsg->msg.msg_control = NULL;
sr->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return -EAGAIN;
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
@@ -587,17 +585,16 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
}
static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
- struct io_async_msghdr *kmsg)
+ struct io_br_sel *sel, struct io_async_msghdr *kmsg)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
-
- int ret;
struct buf_sel_arg arg = {
.iovs = &kmsg->fast_iov,
.max_len = min_not_zero(sr->len, INT_MAX),
.nr_iovs = 1,
.buf_group = sr->buf_group,
};
+ int ret;
if (kmsg->vec.iovec) {
arg.nr_iovs = kmsg->vec.nr;
@@ -610,7 +607,7 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
else
arg.mode |= KBUF_MODE_EXPAND;
- ret = io_buffers_select(req, &arg, issue_flags);
+ ret = io_buffers_select(req, &arg, sel, issue_flags);
if (unlikely(ret < 0))
return ret;
@@ -639,6 +636,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data;
+ struct io_br_sel sel = { };
struct socket *sock;
unsigned flags;
int min_ret = 0;
@@ -657,8 +655,9 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
flags |= MSG_DONTWAIT;
retry_bundle:
+ sel.buf_list = NULL;
if (io_do_buffer_select(req)) {
- ret = io_send_select_buffer(req, issue_flags, kmsg);
+ ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);
if (ret)
return ret;
}
@@ -682,7 +681,7 @@ retry_bundle:
sr->len -= ret;
sr->buf += ret;
sr->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
@@ -693,11 +692,12 @@ retry_bundle:
else if (sr->done_io)
ret = sr->done_io;
- if (!io_send_finish(req, &ret, kmsg, issue_flags))
+ sel.val = ret;
+ if (!io_send_finish(req, kmsg, &sel))
goto retry_bundle;
io_req_msg_cleanup(req, issue_flags);
- return ret;
+ return sel.val;
}
static int io_recvmsg_mshot_prep(struct io_kiocb *req,
@@ -794,18 +794,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
req->flags |= REQ_F_NOWAIT;
if (sr->msg_flags & MSG_ERRQUEUE)
req->flags |= REQ_F_CLEAR_POLLIN;
- if (req->flags & REQ_F_BUFFER_SELECT) {
- /*
- * Store the buffer group for this multishot receive separately,
- * as if we end up doing an io-wq based issue that selects a
- * buffer, it has to be committed immediately and that will
- * clear ->buf_list. This means we lose the link to the buffer
- * list, and the eventual buffer put on completion then cannot
- * restore it.
- */
+ if (req->flags & REQ_F_BUFFER_SELECT)
sr->buf_group = req->buf_index;
- req->buf_list = NULL;
- }
sr->mshot_total_len = sr->mshot_len = 0;
if (sr->flags & IORING_RECV_MULTISHOT) {
if (!(req->flags & REQ_F_BUFFER_SELECT))
@@ -846,9 +836,10 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
* Returns true if it is actually finished, or false if it should run
* again (for multishot).
*/
-static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
+static inline bool io_recv_finish(struct io_kiocb *req,
struct io_async_msghdr *kmsg,
- bool mshot_finished, unsigned issue_flags)
+ struct io_br_sel *sel, bool mshot_finished,
+ unsigned issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
unsigned int cflags = 0;
@@ -856,13 +847,13 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
if (kmsg->msg.msg_inq > 0)
cflags |= IORING_CQE_F_SOCK_NONEMPTY;
- if (*ret > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
+ if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
/*
* If sr->len hits zero, the limit has been reached. Mark
* mshot as finished, and flag MSHOT_DONE as well to prevent
* a potential bundle from being retried.
*/
- sr->mshot_total_len -= min_t(int, *ret, sr->mshot_total_len);
+ sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len);
if (!sr->mshot_total_len) {
sr->flags |= IORING_RECV_MSHOT_DONE;
mshot_finished = true;
@@ -870,13 +861,12 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
}
if (sr->flags & IORING_RECVSEND_BUNDLE) {
- size_t this_ret = *ret - sr->done_io;
+ size_t this_ret = sel->val - sr->done_io;
- cflags |= io_put_kbufs(req, this_ret, io_bundle_nbufs(kmsg, this_ret),
- issue_flags);
+ cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret));
if (sr->flags & IORING_RECV_RETRY)
cflags = req->cqe.flags | (cflags & CQE_F_MASK);
- if (sr->mshot_len && *ret >= sr->mshot_len)
+ if (sr->mshot_len && sel->val >= sr->mshot_len)
sr->flags |= IORING_RECV_MSHOT_CAP;
/* bundle with no more immediate buffers, we're done */
if (req->flags & REQ_F_BL_EMPTY)
@@ -895,7 +885,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
return false;
}
} else {
- cflags |= io_put_kbuf(req, *ret, issue_flags);
+ cflags |= io_put_kbuf(req, sel->val, sel->buf_list);
}
/*
@@ -903,8 +893,8 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
* receive from this socket.
*/
if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
- io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) {
- *ret = IOU_RETRY;
+ io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
+ sel->val = IOU_RETRY;
io_mshot_prep_retry(req, kmsg);
/* Known not-empty or unknown state, retry */
if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
@@ -916,15 +906,15 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
sr->nr_multishot_loops = 0;
sr->flags &= ~IORING_RECV_MSHOT_CAP;
if (issue_flags & IO_URING_F_MULTISHOT)
- *ret = IOU_REQUEUE;
+ sel->val = IOU_REQUEUE;
}
return true;
}
/* Finish the request / stop multishot. */
finish:
- io_req_set_res(req, *ret, cflags);
- *ret = IOU_COMPLETE;
+ io_req_set_res(req, sel->val, cflags);
+ sel->val = IOU_COMPLETE;
io_req_msg_cleanup(req, issue_flags);
return true;
}
@@ -1017,6 +1007,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data;
+ struct io_br_sel sel = { };
struct socket *sock;
unsigned flags;
int ret, min_ret = 0;
@@ -1036,23 +1027,23 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
flags |= MSG_DONTWAIT;
retry_multishot:
+ sel.buf_list = NULL;
if (io_do_buffer_select(req)) {
- void __user *buf;
size_t len = sr->len;
- buf = io_buffer_select(req, &len, sr->buf_group, issue_flags);
- if (!buf)
+ sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
+ if (!sel.addr)
return -ENOBUFS;
if (req->flags & REQ_F_APOLL_MULTISHOT) {
- ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
+ ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
if (ret) {
- io_kbuf_recycle(req, issue_flags);
+ io_kbuf_recycle(req, sel.buf_list, issue_flags);
return ret;
}
}
- iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
+ iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
}
kmsg->msg.msg_get_inq = 1;
@@ -1071,14 +1062,12 @@ retry_multishot:
if (ret < min_ret) {
if (ret == -EAGAIN && force_nonblock) {
- if (issue_flags & IO_URING_F_MULTISHOT)
- io_kbuf_recycle(req, issue_flags);
-
+ io_kbuf_recycle(req, sel.buf_list, issue_flags);
return IOU_RETRY;
}
if (ret > 0 && io_net_retry(sock, flags)) {
sr->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
@@ -1092,16 +1081,17 @@ retry_multishot:
else if (sr->done_io)
ret = sr->done_io;
else
- io_kbuf_recycle(req, issue_flags);
+ io_kbuf_recycle(req, sel.buf_list, issue_flags);
- if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
+ sel.val = ret;
+ if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))
goto retry_multishot;
- return ret;
+ return sel.val;
}
static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
- size_t *len, unsigned int issue_flags)
+ struct io_br_sel *sel, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
int ret;
@@ -1126,15 +1116,15 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
arg.mode |= KBUF_MODE_FREE;
}
- if (*len)
- arg.max_len = *len;
+ if (sel->val)
+ arg.max_len = sel->val;
else if (kmsg->msg.msg_inq > 1)
- arg.max_len = min_not_zero(*len, (size_t) kmsg->msg.msg_inq);
+ arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq);
/* if mshot limited, ensure we don't go over */
if (sr->flags & IORING_RECV_MSHOT_LIM)
arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
- ret = io_buffers_peek(req, &arg);
+ ret = io_buffers_peek(req, &arg, sel);
if (unlikely(ret < 0))
return ret;
@@ -1155,14 +1145,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
arg.out_len);
} else {
- void __user *buf;
+ size_t len = sel->val;
- *len = sr->len;
- buf = io_buffer_select(req, len, sr->buf_group, issue_flags);
- if (!buf)
+ *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
+ if (!sel->addr)
return -ENOBUFS;
- sr->buf = buf;
- sr->len = *len;
+ sr->buf = sel->addr;
+ sr->len = len;
map_ubuf:
ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
&kmsg->msg.msg_iter);
@@ -1177,11 +1166,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr *kmsg = req->async_data;
+ struct io_br_sel sel;
struct socket *sock;
unsigned flags;
int ret, min_ret = 0;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
- size_t len = sr->len;
bool mshot_finished;
if (!(req->flags & REQ_F_POLLED) &&
@@ -1197,9 +1186,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
flags |= MSG_DONTWAIT;
retry_multishot:
+ sel.buf_list = NULL;
if (io_do_buffer_select(req)) {
- ret = io_recv_buf_select(req, kmsg, &len, issue_flags);
- if (unlikely(ret)) {
+ sel.val = sr->len;
+ ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);
+ if (unlikely(ret < 0)) {
kmsg->msg.msg_inq = -1;
goto out_free;
}
@@ -1215,16 +1206,14 @@ retry_multishot:
ret = sock_recvmsg(sock, &kmsg->msg, flags);
if (ret < min_ret) {
if (ret == -EAGAIN && force_nonblock) {
- if (issue_flags & IO_URING_F_MULTISHOT)
- io_kbuf_recycle(req, issue_flags);
-
+ io_kbuf_recycle(req, sel.buf_list, issue_flags);
return IOU_RETRY;
}
if (ret > 0 && io_net_retry(sock, flags)) {
sr->len -= ret;
sr->buf += ret;
sr->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
@@ -1240,12 +1229,13 @@ out_free:
else if (sr->done_io)
ret = sr->done_io;
else
- io_kbuf_recycle(req, issue_flags);
+ io_kbuf_recycle(req, sel.buf_list, issue_flags);
- if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
+ sel.val = ret;
+ if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))
goto retry_multishot;
- return ret;
+ return sel.val;
}
int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -1505,7 +1495,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
zc->len -= ret;
zc->buf += ret;
zc->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return -EAGAIN;
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
@@ -1575,7 +1565,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
if (ret > 0 && io_net_retry(sock, flags)) {
sr->done_io += ret;
- return io_net_kbuf_recyle(req, kmsg, ret);
+ return -EAGAIN;
}
if (ret == -ERESTARTSYS)
ret = -EINTR;
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-04 20:40:59 +0000