1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
// SPDX-License-Identifier: GPL-2.0
#ifndef IOU_ZC_RX_H
#define IOU_ZC_RX_H
#include <linux/io_uring_types.h>
#include <linux/dma-buf.h>
#include <linux/socket.h>
#include <net/page_pool/types.h>
#include <net/net_trackers.h>
#define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
#define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE |\
ZCRX_FEATURE_NOTIFICATION)
#define ZCRX_NOTIF_TYPE_MASK ((1U << ZCRX_NOTIF_NO_BUFFERS) | (1U << ZCRX_NOTIF_COPY))
struct io_zcrx_mem {
unsigned long size;
bool is_dmabuf;
struct page **pages;
unsigned long nr_folios;
struct sg_table page_sg_table;
unsigned long account_pages;
struct sg_table *sgt;
struct dma_buf_attachment *attach;
struct dma_buf *dmabuf;
};
struct io_zcrx_area {
struct net_iov_area nia;
struct io_zcrx_ifq *ifq;
atomic_t *user_refs;
bool is_mapped;
u16 area_id;
/* freelist */
spinlock_t freelist_lock ____cacheline_aligned_in_smp;
u32 free_count;
u32 *freelist;
struct io_zcrx_mem mem;
};
struct zcrx_rq {
spinlock_t lock;
struct io_uring *ring;
struct io_uring_zcrx_rqe *rqes;
u32 cached_head;
u32 nr_entries;
};
struct io_zcrx_ifq {
struct io_zcrx_area *area;
unsigned niov_shift;
struct user_struct *user;
struct mm_struct *mm_account;
bool kern_readable;
struct zcrx_rq rq ____cacheline_aligned_in_smp;
u32 if_rxq;
struct device *dev;
struct net_device *netdev;
netdevice_tracker netdev_tracker;
refcount_t refs;
/* counts userspace facing users like io_uring */
refcount_t user_refs;
/*
* Page pool and net configuration lock, can be taken deeper in the
* net stack.
*/
struct mutex pp_lock;
struct io_mapped_region rq_region;
spinlock_t ctx_lock;
struct io_ring_ctx *master_ctx;
u32 allowed_notif_mask;
u32 fired_notifs;
u64 notif_data;
struct zcrx_notif_stats *notif_stats;
};
#if defined(CONFIG_IO_URING_ZCRX)
int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
int io_register_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg);
void io_unregister_zcrx(struct io_ring_ctx *ctx);
void io_terminate_zcrx(struct io_ring_ctx *ctx);
int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
struct socket *sock, unsigned int flags,
unsigned issue_flags, unsigned int *len);
struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
unsigned int id);
#else
static inline int io_register_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg)
{
return -EOPNOTSUPP;
}
static inline void io_unregister_zcrx(struct io_ring_ctx *ctx)
{
}
static inline void io_terminate_zcrx(struct io_ring_ctx *ctx)
{
}
static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
struct socket *sock, unsigned int flags,
unsigned issue_flags, unsigned int *len)
{
return -EOPNOTSUPP;
}
static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
unsigned int id)
{
return NULL;
}
static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
void __user *arg, unsigned nr_arg)
{
return -EOPNOTSUPP;
}
#endif
int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
#endif
|
