1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.channel.DefaultFileRegion;
19 import io.netty.channel.unix.Buffer;
20 import io.netty.util.internal.ObjectUtil;
21 import io.netty.util.internal.logging.InternalLogger;
22 import io.netty.util.internal.logging.InternalLoggerFactory;
23 import io.netty.channel.unix.FileDescriptor;
24 import io.netty.channel.unix.PeerCredentials;
25 import io.netty.channel.unix.Unix;
26 import io.netty.util.internal.ClassInitializerUtil;
27 import io.netty.util.internal.NativeLibraryLoader;
28 import io.netty.util.internal.PlatformDependent;
29 import io.netty.util.internal.SystemPropertyUtil;
30 import io.netty.util.internal.ThrowableUtil;
31
32 import java.io.File;
33 import java.io.IOException;
34 import java.nio.channels.Selector;
35 import java.nio.file.Path;
36 import java.util.Arrays;
37 import java.util.Locale;
38
39 final class Native {
40 private static final InternalLogger logger = InternalLoggerFactory.getInstance(Native.class);
41
42 static {
43 Selector selector = null;
44 try {
45
46
47
48
49 selector = Selector.open();
50 } catch (IOException ignore) {
51
52 }
53
54
55
56
57
58
59 ClassInitializerUtil.tryLoadClasses(
60 Native.class,
61
62 PeerCredentials.class, java.io.FileDescriptor.class
63 );
64
65 File tmpDir = PlatformDependent.tmpdir();
66 Path tmpFile = tmpDir.toPath().resolve("netty_io_uring.tmp");
67 try {
68
69
70 Native.createFile(tmpFile.toString());
71 } catch (UnsatisfiedLinkError ignore) {
72
73 loadNativeLibrary();
74 } finally {
75 tmpFile.toFile().delete();
76 try {
77 if (selector != null) {
78 selector.close();
79 }
80 } catch (IOException ignore) {
81
82 }
83 }
84 Unix.registerInternal(Native::registerUnix);
85 }
86
87 static final int SOCK_NONBLOCK = NativeStaticallyReferencedJniMethods.sockNonblock();
88 static final int SOCK_CLOEXEC = NativeStaticallyReferencedJniMethods.sockCloexec();
89 static final short AF_INET = (short) NativeStaticallyReferencedJniMethods.afInet();
90 static final short AF_INET6 = (short) NativeStaticallyReferencedJniMethods.afInet6();
91 static final short AF_UNIX = (short) NativeStaticallyReferencedJniMethods.afUnix();
92 static final int SIZEOF_SOCKADDR_STORAGE = NativeStaticallyReferencedJniMethods.sizeofSockaddrStorage();
93 static final int SIZEOF_SOCKADDR_UN = NativeStaticallyReferencedJniMethods.sizeofSockaddrUn();
94 static final int SOCKADDR_UN_OFFSETOF_SUN_FAMILY =
95 NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunFamily();
96 static final int SOCKADDR_UN_OFFSETOF_SUN_PATH =
97 NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunPath();
98 static final int MAX_SUN_PATH_LEN =
99 NativeStaticallyReferencedJniMethods.maxSunPathLen();
100 static final int SIZEOF_SOCKADDR_IN = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn();
101 static final int SIZEOF_SOCKADDR_IN6 = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn6();
102 static final int SOCKADDR_IN_OFFSETOF_SIN_FAMILY =
103 NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinFamily();
104 static final int SOCKADDR_IN_OFFSETOF_SIN_PORT = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinPort();
105 static final int SOCKADDR_IN_OFFSETOF_SIN_ADDR = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinAddr();
106 static final int IN_ADDRESS_OFFSETOF_S_ADDR = NativeStaticallyReferencedJniMethods.inAddressOffsetofSAddr();
107 static final int SOCKADDR_IN6_OFFSETOF_SIN6_FAMILY =
108 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Family();
109 static final int SOCKADDR_IN6_OFFSETOF_SIN6_PORT =
110 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Port();
111 static final int SOCKADDR_IN6_OFFSETOF_SIN6_FLOWINFO =
112 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Flowinfo();
113 static final int SOCKADDR_IN6_OFFSETOF_SIN6_ADDR =
114 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Addr();
115 static final int SOCKADDR_IN6_OFFSETOF_SIN6_SCOPE_ID =
116 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6ScopeId();
117 static final int IN6_ADDRESS_OFFSETOF_S6_ADDR = NativeStaticallyReferencedJniMethods.in6AddressOffsetofS6Addr();
118 static final int SIZEOF_SIZE_T = NativeStaticallyReferencedJniMethods.sizeofSizeT();
119 static final int SIZEOF_INT = NativeStaticallyReferencedJniMethods.sizeofInt();
120 static final int SIZEOF_IOVEC = NativeStaticallyReferencedJniMethods.sizeofIovec();
121 static final int CMSG_SPACE = NativeStaticallyReferencedJniMethods.cmsgSpace();
122 static final int CMSG_SPACE_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgSpaceForFd();
123 static final int CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsgLen();
124 static final int CMSG_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgLenForFd();
125 static final int MSG_CONTROL_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.msgControlLenForFd();
126 static final int CMSG_OFFSETOF_CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLen();
127 static final int CMSG_OFFSETOF_CMSG_LEVEL = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLevel();
128 static final int CMSG_OFFSETOF_CMSG_TYPE = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgType();
129
130 static final int IO_URING_BUFFER_RING_TAIL = NativeStaticallyReferencedJniMethods.ioUringBufferRingOffsetTail();
131
132 static final int IOVEC_OFFSETOF_IOV_BASE = NativeStaticallyReferencedJniMethods.iovecOffsetofIovBase();
133 static final int IOVEC_OFFSETOF_IOV_LEN = NativeStaticallyReferencedJniMethods.iovecOffsetofIovLen();
134 static final int SIZEOF_MSGHDR = NativeStaticallyReferencedJniMethods.sizeofMsghdr();
135 static final int MSGHDR_OFFSETOF_MSG_NAME = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgName();
136 static final int MSGHDR_OFFSETOF_MSG_NAMELEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgNamelen();
137 static final int MSGHDR_OFFSETOF_MSG_IOV = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIov();
138 static final int MSGHDR_OFFSETOF_MSG_IOVLEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIovlen();
139 static final int MSGHDR_OFFSETOF_MSG_CONTROL = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControl();
140 static final int MSGHDR_OFFSETOF_MSG_CONTROLLEN =
141 NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControllen();
142 static final int MSGHDR_OFFSETOF_MSG_FLAGS = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgFlags();
143 static final int POLLIN = NativeStaticallyReferencedJniMethods.pollin();
144 static final int POLLOUT = NativeStaticallyReferencedJniMethods.pollout();
145 static final int POLLRDHUP = NativeStaticallyReferencedJniMethods.pollrdhup();
146 static final int ERRNO_ECANCELED_NEGATIVE = -NativeStaticallyReferencedJniMethods.ecanceled();
147 static final int ERRNO_ETIME_NEGATIVE = -NativeStaticallyReferencedJniMethods.etime();
148 static final int ERRNO_NOBUFS_NEGATIVE = -NativeStaticallyReferencedJniMethods.enobufs();
149
150 static final int PAGE_SIZE = NativeStaticallyReferencedJniMethods.pageSize();
151 static final int MAX_SKB_FRAGS = NativeStaticallyReferencedJniMethods.maxSkbFrags();
152
153 static final int SIZEOF_IOURING_BUF = NativeStaticallyReferencedJniMethods.sizeofIoUringBuf();
154 static final int IOURING_BUFFER_OFFSETOF_ADDR = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetAddr();
155 static final int IOURING_BUFFER_OFFSETOF_LEN = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetLen();
156 static final int IOURING_BUFFER_OFFSETOF_BID = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetBid();
157
158
159
160
161 static final byte IORING_OP_NOP = 0;
162 static final byte IORING_OP_READV = 1;
163 static final byte IORING_OP_WRITEV = 2;
164 static final byte IORING_OP_FSYNC = 3;
165 static final byte IORING_OP_READ_FIXED = 4;
166 static final byte IORING_OP_WRITE_FIXED = 5;
167 static final byte IORING_OP_POLL_ADD = 6;
168 static final byte IORING_OP_POLL_REMOVE = 7;
169 static final byte IORING_OP_SYNC_FILE_RANGE = 8;
170 static final byte IORING_OP_SENDMSG = 9;
171 static final byte IORING_OP_RECVMSG = 10;
172 static final byte IORING_OP_TIMEOUT = 11;
173 static final byte IORING_OP_TIMEOUT_REMOVE = 12;
174 static final byte IORING_OP_ACCEPT = 13;
175 static final byte IORING_OP_ASYNC_CANCEL = 14;
176 static final byte IORING_OP_LINK_TIMEOUT = 15;
177 static final byte IORING_OP_CONNECT = 16;
178 static final byte IORING_OP_FALLOCATE = 17;
179 static final byte IORING_OP_OPENAT = 18;
180 static final byte IORING_OP_CLOSE = 19;
181 static final byte IORING_OP_FILES_UPDATE = 20;
182 static final byte IORING_OP_STATX = 21;
183 static final byte IORING_OP_READ = 22;
184 static final byte IORING_OP_WRITE = 23;
185 static final byte IORING_OP_FADVISE = 24;
186 static final byte IORING_OP_MADVISE = 25;
187 static final byte IORING_OP_SEND = 26;
188 static final byte IORING_OP_RECV = 27;
189 static final byte IORING_OP_OPENAT2 = 28;
190 static final byte IORING_OP_EPOLL_CTL = 29;
191 static final byte IORING_OP_SPLICE = 30;
192 static final byte IORING_OP_PROVIDE_BUFFERS = 31;
193 static final byte IORING_OP_REMOVE_BUFFERS = 32;
194 static final byte IORING_OP_TEE = 33;
195 static final byte IORING_OP_SHUTDOWN = 34;
196 static final byte IORING_OP_RENAMEAT = 35;
197 static final byte IORING_OP_UNLINKAT = 36;
198 static final byte IORING_OP_MKDIRAT = 37;
199 static final byte IORING_OP_SYMLINKAT = 38;
200 static final byte IORING_OP_LINKAT = 39;
201 static final byte IORING_OP_MSG_RING = 40;
202 static final byte IORING_OP_FSETXATTR = 41;
203 static final byte IORING_OP_SETXATTR = 42;
204 static final byte IORING_OP_FGETXATTR = 43;
205 static final byte IORING_OP_GETXATTR = 44;
206 static final byte IORING_OP_SOCKET = 45;
207 static final byte IORING_OP_URING_CMD = 46;
208 static final byte IORING_OP_SEND_ZC = 47;
209 static final byte IORING_OP_SENDMSG_ZC = 48;
210 static final byte IORING_OP_READ_MULTISHOT = 49;
211 static final byte IORING_OP_WAITID = 50;
212 static final byte IORING_OP_FUTEX_WAIT = 51;
213 static final byte IORING_OP_FUTEX_WAKE = 52;
214 static final byte IORING_OP_FUTEX_WAITV = 53;
215 static final byte IORING_OP_FIXED_FD_INSTALL = 54;
216 static final byte IORING_OP_FTRUNCATE = 55;
217 static final byte IORING_OP_BIND = 56;
218 static final int IORING_CQE_F_BUFFER = 1 << 0;
219 static final int IORING_CQE_F_MORE = 1 << 1;
220 static final int IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
221 static final int IORING_CQE_F_NOTIF = 1 << 3;
222 static final int IORING_CQE_F_BUF_MORE = 1 << 4;
223 static final int IORING_CQE_F_SKIP = 1 << 5;
224 static final int IORING_CQE_F_32 = 1 << 15;
225 static final int IORING_SETUP_CQSIZE = 1 << 3;
226 static final int IORING_SETUP_CLAMP = 1 << 4;
227
228 static final int IORING_SETUP_R_DISABLED = 1 << 6;
229 static final int IORING_SETUP_SUBMIT_ALL = 1 << 7;
230 static final int IORING_SETUP_CQE32 = 1 << 11;
231
232 static final int IORING_SETUP_TASKRUN_FLAG = 1 << 9;
233 static final int IORING_SETUP_SINGLE_ISSUER = 1 << 12;
234 static final int IORING_SETUP_DEFER_TASKRUN = 1 << 13;
235 static final int IORING_SETUP_NO_SQARRAY = 1 << 16;
236 static final int IORING_SETUP_CQE_MIXED = 1 << 18;
237 static final int IORING_CQE_BUFFER_SHIFT = 16;
238
239 static final short IORING_POLL_ADD_MULTI = 1 << 0;
240
241 static final short IORING_RECVSEND_POLL_FIRST = 1 << 0;
242 static final short IORING_RECVSEND_BUNDLE = 1 << 4;
243 static final short IORING_RECV_MULTISHOT = 1 << 1;
244 static final short IORING_SEND_ZC_REPORT_USAGE = 1 << 3;
245
246 static final int IORING_NOTIF_USAGE_ZC_COPIED = 1 << 31;
247
248 static final short IORING_ACCEPT_MULTISHOT = 1 << 0;
249 static final short IORING_ACCEPT_DONTWAIT = 1 << 1;
250 static final short IORING_ACCEPT_POLL_FIRST = 1 << 2;
251
252 static final short IORING_NOP_CQE32 = 1 << 5;
253
254 static final int IORING_FEAT_NODROP = 1 << 1;
255 static final int IORING_FEAT_SUBMIT_STABLE = 1 << 2;
256 static final int IORING_FEAT_RECVSEND_BUNDLE = 1 << 14;
257 static final int IORING_FEAT_NO_IOWAIT = 1 << 17;
258
259 static final int IORING_SQ_NEED_WAKEUP = 1 << 0;
260 static final int IORING_SQ_CQ_OVERFLOW = 1 << 1;
261 static final int IORING_SQ_TASKRUN = 1 << 2;
262
263 static final int SPLICE_F_MOVE = 1;
264
265 static final int IOU_PBUF_RING_INC = 2;
266 static final int IO_URING_OP_SUPPORTED = 1;
267
268 static final int CQE_SIZE = 16;
269 static final int CQE32_SIZE = 32;
270
271 static String opToStr(byte op) {
272 switch (op) {
273 case IORING_OP_NOP: return "NOP";
274 case IORING_OP_READV: return "READV";
275 case IORING_OP_WRITEV: return "WRITEV";
276 case IORING_OP_FSYNC: return "FSYNC";
277 case IORING_OP_READ_FIXED: return "READ_FIXED";
278 case IORING_OP_WRITE_FIXED: return "WRITE_FIXED";
279 case IORING_OP_POLL_ADD: return "POLL_ADD";
280 case IORING_OP_POLL_REMOVE: return "POLL_REMOVE";
281 case IORING_OP_SYNC_FILE_RANGE: return "SYNC_FILE_RANGE";
282 case IORING_OP_SENDMSG: return "SENDMSG";
283 case IORING_OP_RECVMSG: return "RECVMSG";
284 case IORING_OP_TIMEOUT: return "TIMEOUT";
285 case IORING_OP_TIMEOUT_REMOVE: return "TIMEOUT_REMOVE";
286 case IORING_OP_ACCEPT: return "ACCEPT";
287 case IORING_OP_ASYNC_CANCEL: return "ASYNC_CANCEL";
288 case IORING_OP_LINK_TIMEOUT: return "LINK_TIMEOUT";
289 case IORING_OP_CONNECT: return "CONNECT";
290 case IORING_OP_FALLOCATE: return "FALLOCATE";
291 case IORING_OP_OPENAT: return "OPENAT";
292 case IORING_OP_CLOSE: return "CLOSE";
293 case IORING_OP_FILES_UPDATE: return "FILES_UPDATE";
294 case IORING_OP_STATX: return "STATX";
295 case IORING_OP_READ: return "READ";
296 case IORING_OP_WRITE: return "WRITE";
297 case IORING_OP_FADVISE: return "FADVISE";
298 case IORING_OP_MADVISE: return "MADVISE";
299 case IORING_OP_SEND: return "SEND";
300 case IORING_OP_RECV: return "RECV";
301 case IORING_OP_OPENAT2: return "OPENAT2";
302 case IORING_OP_EPOLL_CTL: return "EPOLL_CTL";
303 case IORING_OP_SPLICE: return "SPLICE";
304 case IORING_OP_PROVIDE_BUFFERS: return "PROVIDE_BUFFERS";
305 case IORING_OP_REMOVE_BUFFERS: return "REMOVE_BUFFERS";
306 case IORING_OP_TEE: return "TEE";
307 case IORING_OP_SHUTDOWN: return "SHUTDOWN";
308 case IORING_OP_RENAMEAT: return "RENAMEAT";
309 case IORING_OP_UNLINKAT: return "UNLINKAT";
310 case IORING_OP_MKDIRAT: return "MKDIRAT";
311 case IORING_OP_SYMLINKAT: return "SYMLINKAT";
312 case IORING_OP_LINKAT: return "LINKAT";
313 case IORING_OP_SEND_ZC: return "SEND_ZC";
314 case IORING_OP_SENDMSG_ZC: return "SENDMSG_ZC";
315 default: return "[OP CODE " + op + ']';
316 }
317 }
318
319 static final int IORING_ENTER_GETEVENTS = NativeStaticallyReferencedJniMethods.ioringEnterGetevents();
320 static final int IORING_ENTER_REGISTERED_RING = 1 << 4;
321 static final int IORING_ENTER_NO_IOWAIT = 1 << 7;
322 static final int IOSQE_ASYNC = NativeStaticallyReferencedJniMethods.iosqeAsync();
323 static final int IOSQE_LINK = NativeStaticallyReferencedJniMethods.iosqeLink();
324 static final int IOSQE_HARDLINK = NativeStaticallyReferencedJniMethods.iosqeHardlink();
325 static final int IOSQE_IO_DRAIN = NativeStaticallyReferencedJniMethods.iosqeDrain();
326 static final int IOSQE_BUFFER_SELECT = NativeStaticallyReferencedJniMethods.iosqeBufferSelect();
327 static final int IOSQE_CQE_SKIP_SUCCESS = 1 << 6;
328 static final int MSG_DONTWAIT = NativeStaticallyReferencedJniMethods.msgDontwait();
329 static final int MSG_FASTOPEN = NativeStaticallyReferencedJniMethods.msgFastopen();
330 static final int MSG_CTRUNC = NativeStaticallyReferencedJniMethods.msgCtrunc();
331 static final int SOL_UDP = NativeStaticallyReferencedJniMethods.solUdp();
332 static final int SOL_SOCKET = NativeStaticallyReferencedJniMethods.solSocket();
333 static final int UDP_SEGMENT = NativeStaticallyReferencedJniMethods.udpSegment();
334 static final int SCM_RIGHTS = NativeStaticallyReferencedJniMethods.scmRights();
335 private static final int TFO_ENABLED_CLIENT_MASK = 0x1;
336 private static final int TFO_ENABLED_SERVER_MASK = 0x2;
337 private static final int TCP_FASTOPEN_MODE = NativeStaticallyReferencedJniMethods.tcpFastopenMode();
338
339
340
341
342 static final boolean IS_SUPPORTING_TCP_FASTOPEN_CLIENT =
343 (TCP_FASTOPEN_MODE & TFO_ENABLED_CLIENT_MASK) == TFO_ENABLED_CLIENT_MASK;
344
345
346
347
348 static final boolean IS_SUPPORTING_TCP_FASTOPEN_SERVER =
349 (TCP_FASTOPEN_MODE & TFO_ENABLED_SERVER_MASK) == TFO_ENABLED_SERVER_MASK;
350
351 private static final int[] REQUIRED_IORING_OPS = {
352 IORING_OP_POLL_ADD,
353 IORING_OP_TIMEOUT,
354 IORING_OP_ACCEPT,
355 IORING_OP_READ,
356 IORING_OP_WRITE,
357 IORING_OP_POLL_REMOVE,
358 IORING_OP_CONNECT,
359 IORING_OP_CLOSE,
360 IORING_OP_WRITEV,
361 IORING_OP_SENDMSG,
362 IORING_OP_RECVMSG,
363 IORING_OP_ASYNC_CANCEL,
364 IORING_OP_RECV,
365 IORING_OP_NOP,
366 IORING_OP_SHUTDOWN,
367 IORING_OP_SEND
368 };
369
370 static int setupFlags(boolean useSingleIssuer) {
371 int flags = Native.IORING_SETUP_R_DISABLED | Native.IORING_SETUP_CLAMP;
372 if (IoUring.isSetupSubmitAllSupported()) {
373 flags |= Native.IORING_SETUP_SUBMIT_ALL;
374 }
375
376 if (useSingleIssuer) {
377
378 if (IoUring.isSetupSingleIssuerSupported()) {
379 flags |= Native.IORING_SETUP_SINGLE_ISSUER;
380 }
381
382 if (IoUring.isSetupDeferTaskrunSupported()) {
383 flags |= Native.IORING_SETUP_DEFER_TASKRUN;
384 flags |= Native.IORING_SETUP_TASKRUN_FLAG;
385 }
386 }
387
388
389 if (IoUring.isIoringSetupNoSqarraySupported()) {
390 flags |= Native.IORING_SETUP_NO_SQARRAY;
391 }
392
393
394 if (IoUring.isSetupCqeMixedSupported()) {
395 flags |= Native.IORING_SETUP_CQE_MIXED;
396 }
397 return flags;
398 }
399
400 static RingBuffer createRingBuffer(int ringSize, int setupFlags) {
401 return createRingBuffer(ringSize, ringSize * 2, setupFlags);
402 }
403
404 static RingBuffer createRingBuffer(int ringSize, int cqeSize, int setupFlags) {
405 ObjectUtil.checkPositive(ringSize, "ringSize");
406 ObjectUtil.checkPositive(cqeSize, "cqeSize");
407 long[] values = ioUringSetup(ringSize, cqeSize, setupFlags);
408 assert values.length == 20;
409 long cqkhead = values[0];
410 long cqktail = values[1];
411 int cqringMask = (int) values[2];
412 int cqringEntries = (int) values[3];
413 long cqkflags = values[4];
414 long cqArrayAddress = values[5];
415 int cqringSize = (int) values[6];
416 long cqringAddress = values[7];
417 int cqringFd = (int) values[8];
418 int cqringCapacity = (int) values[9];
419 int cqeLength = (setupFlags & IORING_SETUP_CQE32) == 0 ? CQE_SIZE : CQE32_SIZE;
420 boolean extraCqeDataNeeded = (setupFlags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED)) != 0;
421 CompletionQueue completionQueue = new CompletionQueue(
422 Buffer.wrapMemoryAddressWithNativeOrder(cqkhead, Integer.BYTES),
423 Buffer.wrapMemoryAddressWithNativeOrder(cqktail, Integer.BYTES),
424 cqringMask,
425 cqringEntries,
426 Buffer.wrapMemoryAddressWithNativeOrder(cqkflags, Integer.BYTES),
427 Buffer.wrapMemoryAddressWithNativeOrder(cqArrayAddress, cqringEntries * cqeLength),
428 cqringSize,
429 cqringAddress,
430 cqringFd,
431 cqringCapacity, cqeLength, extraCqeDataNeeded);
432
433 long sqkhead = values[10];
434 long sqktail = values[11];
435 int sqringMask = (int) values[12];
436 int sqringEntries = (int) values[13];
437 long sqkflags = values[14];
438 long sqArrayAddress = values[15];
439 int sqringSize = (int) values[16];
440 long sqringAddress = values[17];
441 int sqringFd = (int) values[18];
442 SubmissionQueue submissionQueue = new SubmissionQueue(
443 Buffer.wrapMemoryAddressWithNativeOrder(sqkhead, Integer.BYTES),
444 Buffer.wrapMemoryAddressWithNativeOrder(sqktail, Integer.BYTES),
445 sqringMask,
446 sqringEntries,
447 Buffer.wrapMemoryAddressWithNativeOrder(sqkflags, Integer.BYTES),
448 Buffer.wrapMemoryAddressWithNativeOrder(sqArrayAddress, sqringEntries * SubmissionQueue.SQE_SIZE),
449 sqringSize,
450 sqringAddress,
451 sqringFd);
452 return new RingBuffer(submissionQueue, completionQueue, (int) values[19]);
453 }
454
455 static void checkAllIOSupported(IoUringProbe probe) {
456 if (!ioUringProbe(probe, REQUIRED_IORING_OPS)) {
457 throw new UnsupportedOperationException("Not all operations are supported: "
458 + Arrays.toString(REQUIRED_IORING_OPS));
459 }
460 }
461
462 static boolean isRecvMultishotSupported() {
463
464 return Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
465 }
466
467 static boolean isAcceptMultishotSupported(IoUringProbe probe) {
468
469 return ioUringProbe(probe, new int[] { Native.IORING_OP_SOCKET });
470 }
471
472 static boolean isCqeFSockNonEmptySupported(IoUringProbe probe) {
473
474 return ioUringProbe(probe, new int[] { Native.IORING_OP_SOCKET });
475 }
476
477 static boolean isSpliceSupported(IoUringProbe probe) {
478
479 return ioUringProbe(probe, new int[] { Native.IORING_OP_SPLICE });
480 }
481
482 static boolean isPollAddMultiShotSupported(IoUringProbe probe) {
483
484 return isCqeFSockNonEmptySupported(probe);
485 }
486
487 static boolean isSendZcSupported(IoUringProbe probe) {
488
489 return ioUringProbe(probe, new int[] { Native.IORING_OP_SEND_ZC });
490 }
491
492 static boolean isSendmsgZcSupported(IoUringProbe probe) {
493
494 return ioUringProbe(probe, new int[] { Native.IORING_OP_SENDMSG_ZC });
495 }
496
497
498
499
500
501
502 static boolean isRegisterIoWqWorkerSupported(int ringFd) {
503
504 int result = ioUringRegisterIoWqMaxWorkers(ringFd, 0, 0);
505 if (result >= 0) {
506 return true;
507 }
508
509 return false;
510 }
511
512 static boolean isRegisterBufferRingSupported(int ringFd, int flags) {
513 int entries = 2;
514 short bgid = 1;
515 long result = ioUringRegisterBufRing(ringFd, entries, bgid, flags);
516 if (result >= 0) {
517 ioUringUnRegisterBufRing(ringFd, result, entries, bgid);
518 return true;
519 }
520
521 return false;
522 }
523
524 static void checkKernelVersion(String kernelVersion) {
525 boolean enforceKernelVersion = SystemPropertyUtil.getBoolean(
526 "io.netty.transport.iouring.enforceKernelVersion", true);
527 boolean kernelSupported = checkKernelVersion(kernelVersion, 5, 9);
528 if (!kernelSupported) {
529 if (enforceKernelVersion) {
530 throw new UnsupportedOperationException(
531 "you need at least kernel version 5.9, current kernel version: " + kernelVersion);
532 } else {
533 logger.debug("Detected kernel " + kernelVersion + " does not match minimum version of 5.9, " +
534 "trying to use io_uring anyway");
535 }
536 }
537 }
538
539 private static boolean checkKernelVersion(String kernelVersion, int major, int minor) {
540 String[] versionComponents = kernelVersion.split("\\.");
541 if (versionComponents.length < 3) {
542 return false;
543 }
544 int nativeMajor;
545 try {
546 nativeMajor = Integer.parseInt(versionComponents[0]);
547 } catch (NumberFormatException e) {
548 return false;
549 }
550
551 if (nativeMajor < major) {
552 return false;
553 }
554
555 if (nativeMajor > major) {
556 return true;
557 }
558
559 int nativeMinor;
560 try {
561 nativeMinor = Integer.parseInt(versionComponents[1]);
562 } catch (NumberFormatException e) {
563 return false;
564 }
565
566 return nativeMinor >= minor;
567 }
568
569 static final class IoUringProbe {
570 final byte lastOp;
571 final byte opsLen;
572 final IoUringProbeOp[] ops;
573
574 IoUringProbe(int[] values) {
575 int idx = 0;
576 lastOp = (byte) values[idx++];
577 opsLen = (byte) values[idx++];
578 ops = new IoUringProbeOp[opsLen];
579 for (int i = 0; i < opsLen; i++) {
580 ops[i] = new IoUringProbeOp((byte) values[idx++], values[idx++]);
581 }
582 }
583 }
584
585 static class IoUringProbeOp {
586 final byte op;
587 final int flags;
588
589 IoUringProbeOp(byte op, int flags) {
590 this.op = op;
591 this.flags = flags;
592 }
593 }
594
595 static boolean ioUringProbe(IoUringProbe probe, int[] ops) {
596 IoUringProbeOp[] ioUringProbeOps = probe.ops;
597 if (ioUringProbeOps == null) {
598 return false;
599 }
600 for (int op : ops) {
601 if (op > probe.lastOp || (ioUringProbeOps[op].flags & IO_URING_OP_SUPPORTED) == 0) {
602 return false;
603 }
604 }
605 return true;
606 }
607
608 static native boolean ioUringSetupSupportsFlags(int setupFlags);
609 static native boolean isUnixDomainSocketInqSupported();
610 private static native long[] ioUringSetup(int entries, int cqeSize, int setupFlags);
611
612 static IoUringProbe ioUringProbe(int ringfd) {
613 int[] values = ioUringProbe0(ringfd);
614 if (values == null) {
615 return null;
616 }
617 return new IoUringProbe(values);
618 }
619 private static native int[] ioUringProbe0(int ringFd);
620
621 static native int ioUringRegisterIoWqMaxWorkers(int ringFd, int maxBoundedValue, int maxUnboundedValue);
622 static native int ioUringRegisterEnableRings(int ringFd);
623 static native int ioUringRegisterRingFds(int ringFds);
624
625 static native long ioUringRegisterBufRing(int ringFd, int entries, short bufferGroup, int flags);
626 static native int ioUringUnRegisterBufRing(int ringFd, long ioUringBufRingAddr, int entries, short bufferGroupId);
627 static native int ioUringBufRingSize(int entries);
628 static native int ioUringEnter(int ringFd, int toSubmit, int minComplete, int flags);
629
630 static native void eventFdWrite(int fd, long value);
631
632
633
634
635
636 static native int close(int fd);
637
638 static int getFd(DefaultFileRegion fileChannel) {
639 return getFd0(fileChannel);
640 }
641
642 private static native int getFd0(Object fileChannel);
643
644 static FileDescriptor newBlockingEventFd() {
645 return new FileDescriptor(blockingEventFd());
646 }
647
648 static native void ioUringExit(long submissionQueueArrayAddress, int submissionQueueRingEntries,
649 long submissionQueueRingAddress, int submissionQueueRingSize,
650 long completionQueueRingAddress, int completionQueueRingSize,
651 int ringFd, int enterRingFd);
652
653 private static native int blockingEventFd();
654
655
656 static native int createFile(String name);
657
658 private static native int registerUnix();
659
660 static native long cmsghdrData(long hdrAddr);
661
662 static native String kernelVersion();
663
664 private Native() {
665
666 }
667
668
669 private static void loadNativeLibrary() {
670 String name = PlatformDependent.normalizedOs().toLowerCase(Locale.ROOT).trim();
671 if (!name.startsWith("linux")) {
672 throw new IllegalStateException("Only supported on Linux");
673 }
674 String staticLibName = "netty_transport_native_io_uring42";
675 String sharedLibName = staticLibName + '_' + PlatformDependent.normalizedArch();
676 ClassLoader cl = PlatformDependent.getClassLoader(Native.class);
677 try {
678 NativeLibraryLoader.load(sharedLibName, cl);
679 } catch (UnsatisfiedLinkError e1) {
680 try {
681 NativeLibraryLoader.load(staticLibName, cl);
682 logger.info("Failed to load io_uring");
683 } catch (UnsatisfiedLinkError e2) {
684 ThrowableUtil.addSuppressed(e1, e2);
685 throw e1;
686 }
687 }
688 }
689 }