1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.channel.DefaultFileRegion;
19 import io.netty.channel.unix.Buffer;
20 import io.netty.util.internal.ObjectUtil;
21 import io.netty.util.internal.logging.InternalLogger;
22 import io.netty.util.internal.logging.InternalLoggerFactory;
23 import io.netty.channel.unix.FileDescriptor;
24 import io.netty.channel.unix.PeerCredentials;
25 import io.netty.channel.unix.Unix;
26 import io.netty.util.internal.ClassInitializerUtil;
27 import io.netty.util.internal.NativeLibraryLoader;
28 import io.netty.util.internal.PlatformDependent;
29 import io.netty.util.internal.SystemPropertyUtil;
30 import io.netty.util.internal.ThrowableUtil;
31
32 import java.io.File;
33 import java.io.IOException;
34 import java.nio.channels.Selector;
35 import java.nio.file.Path;
36 import java.util.Arrays;
37 import java.util.Locale;
38
39 final class Native {
40 private static final InternalLogger logger = InternalLoggerFactory.getInstance(Native.class);
41 static final int DEFAULT_RING_SIZE = Math.max(64, SystemPropertyUtil.getInt("io.netty.iouring.ringSize", 4096));
42
43 static {
44 Selector selector = null;
45 try {
46
47
48
49
50 selector = Selector.open();
51 } catch (IOException ignore) {
52
53 }
54
55
56
57
58
59
60 ClassInitializerUtil.tryLoadClasses(
61 Native.class,
62
63 PeerCredentials.class, java.io.FileDescriptor.class
64 );
65
66 File tmpDir = PlatformDependent.tmpdir();
67 Path tmpFile = tmpDir.toPath().resolve("netty_io_uring.tmp");
68 try {
69
70
71 Native.createFile(tmpFile.toString());
72 } catch (UnsatisfiedLinkError ignore) {
73
74 loadNativeLibrary();
75 } finally {
76 tmpFile.toFile().delete();
77 try {
78 if (selector != null) {
79 selector.close();
80 }
81 } catch (IOException ignore) {
82
83 }
84 }
85 Unix.registerInternal(Native::registerUnix);
86 }
87
88 static final int SOCK_NONBLOCK = NativeStaticallyReferencedJniMethods.sockNonblock();
89 static final int SOCK_CLOEXEC = NativeStaticallyReferencedJniMethods.sockCloexec();
90 static final short AF_INET = (short) NativeStaticallyReferencedJniMethods.afInet();
91 static final short AF_INET6 = (short) NativeStaticallyReferencedJniMethods.afInet6();
92 static final short AF_UNIX = (short) NativeStaticallyReferencedJniMethods.afUnix();
93 static final int SIZEOF_SOCKADDR_STORAGE = NativeStaticallyReferencedJniMethods.sizeofSockaddrStorage();
94 static final int SIZEOF_SOCKADDR_UN = NativeStaticallyReferencedJniMethods.sizeofSockaddrUn();
95 static final int SOCKADDR_UN_OFFSETOF_SUN_FAMILY =
96 NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunFamily();
97 static final int SOCKADDR_UN_OFFSETOF_SUN_PATH =
98 NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunPath();
99 static final int MAX_SUN_PATH_LEN =
100 NativeStaticallyReferencedJniMethods.maxSunPathLen();
101 static final int SIZEOF_SOCKADDR_IN = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn();
102 static final int SIZEOF_SOCKADDR_IN6 = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn6();
103 static final int SOCKADDR_IN_OFFSETOF_SIN_FAMILY =
104 NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinFamily();
105 static final int SOCKADDR_IN_OFFSETOF_SIN_PORT = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinPort();
106 static final int SOCKADDR_IN_OFFSETOF_SIN_ADDR = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinAddr();
107 static final int IN_ADDRESS_OFFSETOF_S_ADDR = NativeStaticallyReferencedJniMethods.inAddressOffsetofSAddr();
108 static final int SOCKADDR_IN6_OFFSETOF_SIN6_FAMILY =
109 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Family();
110 static final int SOCKADDR_IN6_OFFSETOF_SIN6_PORT =
111 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Port();
112 static final int SOCKADDR_IN6_OFFSETOF_SIN6_FLOWINFO =
113 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Flowinfo();
114 static final int SOCKADDR_IN6_OFFSETOF_SIN6_ADDR =
115 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Addr();
116 static final int SOCKADDR_IN6_OFFSETOF_SIN6_SCOPE_ID =
117 NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6ScopeId();
118 static final int IN6_ADDRESS_OFFSETOF_S6_ADDR = NativeStaticallyReferencedJniMethods.in6AddressOffsetofS6Addr();
119 static final int SIZEOF_SIZE_T = NativeStaticallyReferencedJniMethods.sizeofSizeT();
120 static final int SIZEOF_IOVEC = NativeStaticallyReferencedJniMethods.sizeofIovec();
121 static final int CMSG_SPACE = NativeStaticallyReferencedJniMethods.cmsgSpace();
122 static final int CMSG_SPACE_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgSpaceForFd();
123 static final int CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsgLen();
124 static final int CMSG_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgLenForFd();
125 static final int MSG_CONTROL_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.msgControlLenForFd();
126 static final int CMSG_OFFSETOF_CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLen();
127 static final int CMSG_OFFSETOF_CMSG_LEVEL = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLevel();
128 static final int CMSG_OFFSETOF_CMSG_TYPE = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgType();
129
130 static final int IO_URING_BUFFER_RING_TAIL = NativeStaticallyReferencedJniMethods.ioUringBufferRingOffsetTail();
131
132 static final int IOVEC_OFFSETOF_IOV_BASE = NativeStaticallyReferencedJniMethods.iovecOffsetofIovBase();
133 static final int IOVEC_OFFSETOF_IOV_LEN = NativeStaticallyReferencedJniMethods.iovecOffsetofIovLen();
134 static final int SIZEOF_MSGHDR = NativeStaticallyReferencedJniMethods.sizeofMsghdr();
135 static final int MSGHDR_OFFSETOF_MSG_NAME = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgName();
136 static final int MSGHDR_OFFSETOF_MSG_NAMELEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgNamelen();
137 static final int MSGHDR_OFFSETOF_MSG_IOV = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIov();
138 static final int MSGHDR_OFFSETOF_MSG_IOVLEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIovlen();
139 static final int MSGHDR_OFFSETOF_MSG_CONTROL = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControl();
140 static final int MSGHDR_OFFSETOF_MSG_CONTROLLEN =
141 NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControllen();
142 static final int MSGHDR_OFFSETOF_MSG_FLAGS = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgFlags();
143 static final int POLLIN = NativeStaticallyReferencedJniMethods.pollin();
144 static final int POLLOUT = NativeStaticallyReferencedJniMethods.pollout();
145 static final int POLLRDHUP = NativeStaticallyReferencedJniMethods.pollrdhup();
146 static final int ERRNO_ECANCELED_NEGATIVE = -NativeStaticallyReferencedJniMethods.ecanceled();
147 static final int ERRNO_ETIME_NEGATIVE = -NativeStaticallyReferencedJniMethods.etime();
148 static final int ERRNO_NOBUFS_NEGATIVE = -NativeStaticallyReferencedJniMethods.enobufs();
149
150 static final int PAGE_SIZE = NativeStaticallyReferencedJniMethods.pageSize();
151
152 static final int SIZEOF_IOURING_BUF = NativeStaticallyReferencedJniMethods.sizeofIoUringBuf();
153 static final int IOURING_BUFFER_OFFSETOF_ADDR = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetAddr();
154 static final int IOURING_BUFFER_OFFSETOF_LEN = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetLen();
155 static final int IOURING_BUFFER_OFFSETOF_BID = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetBid();
156
157
158
159
160 static final byte IORING_OP_NOP = 0;
161 static final byte IORING_OP_READV = 1;
162 static final byte IORING_OP_WRITEV = 2;
163 static final byte IORING_OP_FSYNC = 3;
164 static final byte IORING_OP_READ_FIXED = 4;
165 static final byte IORING_OP_WRITE_FIXED = 5;
166 static final byte IORING_OP_POLL_ADD = 6;
167 static final byte IORING_OP_POLL_REMOVE = 7;
168 static final byte IORING_OP_SYNC_FILE_RANGE = 8;
169 static final byte IORING_OP_SENDMSG = 9;
170 static final byte IORING_OP_RECVMSG = 10;
171 static final byte IORING_OP_TIMEOUT = 11;
172 static final byte IORING_OP_TIMEOUT_REMOVE = 12;
173 static final byte IORING_OP_ACCEPT = 13;
174 static final byte IORING_OP_ASYNC_CANCEL = 14;
175 static final byte IORING_OP_LINK_TIMEOUT = 15;
176 static final byte IORING_OP_CONNECT = 16;
177 static final byte IORING_OP_FALLOCATE = 17;
178 static final byte IORING_OP_OPENAT = 18;
179 static final byte IORING_OP_CLOSE = 19;
180 static final byte IORING_OP_FILES_UPDATE = 20;
181 static final byte IORING_OP_STATX = 21;
182 static final byte IORING_OP_READ = 22;
183 static final byte IORING_OP_WRITE = 23;
184 static final byte IORING_OP_FADVISE = 24;
185 static final byte IORING_OP_MADVISE = 25;
186 static final byte IORING_OP_SEND = 26;
187 static final byte IORING_OP_RECV = 27;
188 static final byte IORING_OP_OPENAT2 = 28;
189 static final byte IORING_OP_EPOLL_CTL = 29;
190 static final byte IORING_OP_SPLICE = 30;
191 static final byte IORING_OP_PROVIDE_BUFFERS = 31;
192 static final byte IORING_OP_REMOVE_BUFFERS = 32;
193 static final byte IORING_OP_TEE = 33;
194 static final byte IORING_OP_SHUTDOWN = 34;
195 static final byte IORING_OP_RENAMEAT = 35;
196 static final byte IORING_OP_UNLINKAT = 36;
197 static final byte IORING_OP_MKDIRAT = 37;
198 static final byte IORING_OP_SYMLINKAT = 38;
199 static final byte IORING_OP_LINKAT = 39;
200 static final byte IORING_OP_MSG_RING = 40;
201 static final byte IORING_OP_FSETXATTR = 41;
202 static final byte IORING_OP_SETXATTR = 42;
203 static final byte IORING_OP_FGETXATTR = 43;
204 static final byte IORING_OP_GETXATTR = 44;
205 static final byte IORING_OP_SOCKET = 45;
206 static final byte IORING_OP_URING_CMD = 46;
207 static final byte IORING_OP_SEND_ZC = 47;
208 static final byte IORING_OP_SENDMSG_ZC = 48;
209 static final byte IORING_OP_READ_MULTISHOT = 49;
210 static final byte IORING_OP_WAITID = 50;
211 static final byte IORING_OP_FUTEX_WAIT = 51;
212 static final byte IORING_OP_FUTEX_WAKE = 52;
213 static final byte IORING_OP_FUTEX_WAITV = 53;
214 static final byte IORING_OP_FIXED_FD_INSTALL = 54;
215 static final byte IORING_OP_FTRUNCATE = 55;
216 static final byte IORING_OP_BIND = 56;
217 static final byte IORING_CQE_F_BUFFER = 1 << 0;
218 static final byte IORING_CQE_F_MORE = 1 << 1;
219 static final byte IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
220 static final byte IORING_CQE_F_BUF_MORE = 1 << 4;
221
222 static final int IORING_SETUP_CQSIZE = 1 << 3;
223 static final int IORING_SETUP_CLAMP = 1 << 4;
224
225 static final int IORING_SETUP_R_DISABLED = 1 << 6;
226 static final int IORING_SETUP_SUBMIT_ALL = 1 << 7;
227 static final int IORING_SETUP_SINGLE_ISSUER = 1 << 12;
228 static final int IORING_SETUP_DEFER_TASKRUN = 1 << 13;
229 static final int IORING_CQE_BUFFER_SHIFT = 16;
230
231 static final short IORING_POLL_ADD_MULTI = 1 << 0;
232
233 static final short IORING_RECVSEND_POLL_FIRST = 1 << 0;
234 static final short IORING_RECVSEND_BUNDLE = 1 << 4;
235 static final short IORING_RECV_MULTISHOT = 1 << 1;
236
237 static final short IORING_ACCEPT_MULTISHOT = 1 << 0;
238 static final short IORING_ACCEPT_DONTWAIT = 1 << 1;
239 static final short IORING_ACCEPT_POLL_FIRST = 1 << 2;
240
241 static final int IORING_FEAT_SUBMIT_STABLE = 1 << 2;
242 static final int IORING_FEAT_RECVSEND_BUNDLE = 1 << 14;
243
244 static final int SPLICE_F_MOVE = 1;
245
246 static final int IOU_PBUF_RING_INC = 2;
247 static String opToStr(byte op) {
248 switch (op) {
249 case IORING_OP_NOP: return "NOP";
250 case IORING_OP_READV: return "READV";
251 case IORING_OP_WRITEV: return "WRITEV";
252 case IORING_OP_FSYNC: return "FSYNC";
253 case IORING_OP_READ_FIXED: return "READ_FIXED";
254 case IORING_OP_WRITE_FIXED: return "WRITE_FIXED";
255 case IORING_OP_POLL_ADD: return "POLL_ADD";
256 case IORING_OP_POLL_REMOVE: return "POLL_REMOVE";
257 case IORING_OP_SYNC_FILE_RANGE: return "SYNC_FILE_RANGE";
258 case IORING_OP_SENDMSG: return "SENDMSG";
259 case IORING_OP_RECVMSG: return "RECVMSG";
260 case IORING_OP_TIMEOUT: return "TIMEOUT";
261 case IORING_OP_TIMEOUT_REMOVE: return "TIMEOUT_REMOVE";
262 case IORING_OP_ACCEPT: return "ACCEPT";
263 case IORING_OP_ASYNC_CANCEL: return "ASYNC_CANCEL";
264 case IORING_OP_LINK_TIMEOUT: return "LINK_TIMEOUT";
265 case IORING_OP_CONNECT: return "CONNECT";
266 case IORING_OP_FALLOCATE: return "FALLOCATE";
267 case IORING_OP_OPENAT: return "OPENAT";
268 case IORING_OP_CLOSE: return "CLOSE";
269 case IORING_OP_FILES_UPDATE: return "FILES_UPDATE";
270 case IORING_OP_STATX: return "STATX";
271 case IORING_OP_READ: return "READ";
272 case IORING_OP_WRITE: return "WRITE";
273 case IORING_OP_FADVISE: return "FADVISE";
274 case IORING_OP_MADVISE: return "MADVISE";
275 case IORING_OP_SEND: return "SEND";
276 case IORING_OP_RECV: return "RECV";
277 case IORING_OP_OPENAT2: return "OPENAT2";
278 case IORING_OP_EPOLL_CTL: return "EPOLL_CTL";
279 case IORING_OP_SPLICE: return "SPLICE";
280 case IORING_OP_PROVIDE_BUFFERS: return "PROVIDE_BUFFERS";
281 case IORING_OP_REMOVE_BUFFERS: return "REMOVE_BUFFERS";
282 case IORING_OP_TEE: return "TEE";
283 case IORING_OP_SHUTDOWN: return "SHUTDOWN";
284 case IORING_OP_RENAMEAT: return "RENAMEAT";
285 case IORING_OP_UNLINKAT: return "UNLINKAT";
286 case IORING_OP_MKDIRAT: return "MKDIRAT";
287 case IORING_OP_SYMLINKAT: return "SYMLINKAT";
288 case IORING_OP_LINKAT: return "LINKAT";
289 default: return "[OP CODE " + op + ']';
290 }
291 }
292
293 static final int IORING_ENTER_GETEVENTS = NativeStaticallyReferencedJniMethods.ioringEnterGetevents();
294 static final int IORING_ENTER_REGISTERED_RING = 1 << 4;
295 static final int IOSQE_ASYNC = NativeStaticallyReferencedJniMethods.iosqeAsync();
296 static final int IOSQE_LINK = NativeStaticallyReferencedJniMethods.iosqeLink();
297 static final int IOSQE_IO_DRAIN = NativeStaticallyReferencedJniMethods.iosqeDrain();
298 static final int IOSQE_BUFFER_SELECT = NativeStaticallyReferencedJniMethods.iosqeBufferSelect();
299 static final int IOSQE_CQE_SKIP_SUCCESS = 1 << 6;
300 static final int MSG_DONTWAIT = NativeStaticallyReferencedJniMethods.msgDontwait();
301 static final int MSG_FASTOPEN = NativeStaticallyReferencedJniMethods.msgFastopen();
302 static final int SOL_UDP = NativeStaticallyReferencedJniMethods.solUdp();
303 static final int SOL_SOCKET = NativeStaticallyReferencedJniMethods.solSocket();
304 static final int UDP_SEGMENT = NativeStaticallyReferencedJniMethods.udpSegment();
305 static final int SCM_RIGHTS = NativeStaticallyReferencedJniMethods.scmRights();
306 private static final int TFO_ENABLED_CLIENT_MASK = 0x1;
307 private static final int TFO_ENABLED_SERVER_MASK = 0x2;
308 private static final int TCP_FASTOPEN_MODE = NativeStaticallyReferencedJniMethods.tcpFastopenMode();
309
310
311
312
313 static final boolean IS_SUPPORTING_TCP_FASTOPEN_CLIENT =
314 (TCP_FASTOPEN_MODE & TFO_ENABLED_CLIENT_MASK) == TFO_ENABLED_CLIENT_MASK;
315
316
317
318
319 static final boolean IS_SUPPORTING_TCP_FASTOPEN_SERVER =
320 (TCP_FASTOPEN_MODE & TFO_ENABLED_SERVER_MASK) == TFO_ENABLED_SERVER_MASK;
321
322 private static final int[] REQUIRED_IORING_OPS = {
323 IORING_OP_POLL_ADD,
324 IORING_OP_TIMEOUT,
325 IORING_OP_ACCEPT,
326 IORING_OP_READ,
327 IORING_OP_WRITE,
328 IORING_OP_POLL_REMOVE,
329 IORING_OP_CONNECT,
330 IORING_OP_CLOSE,
331 IORING_OP_WRITEV,
332 IORING_OP_SENDMSG,
333 IORING_OP_RECVMSG,
334 IORING_OP_ASYNC_CANCEL,
335 IORING_OP_RECV,
336 IORING_OP_NOP,
337 IORING_OP_SHUTDOWN,
338 IORING_OP_SEND
339 };
340
341 static int setupFlags() {
342 int flags = Native.IORING_SETUP_R_DISABLED | Native.IORING_SETUP_CLAMP;
343 if (IoUring.isSetupSubmitAllSupported()) {
344 flags |= Native.IORING_SETUP_SUBMIT_ALL;
345 }
346
347
348 if (IoUring.isSetupSingleIssuerSupported()) {
349 flags |= Native.IORING_SETUP_SINGLE_ISSUER;
350 }
351 if (IoUring.isSetupDeferTaskrunSupported()) {
352 flags |= Native.IORING_SETUP_DEFER_TASKRUN;
353 }
354 return flags;
355 }
356
357 static RingBuffer createRingBuffer(int ringSize, int setupFlags) {
358 return createRingBuffer(ringSize, ringSize * 2, setupFlags);
359 }
360
361 static RingBuffer createRingBuffer(int ringSize, int cqeSize, int setupFlags) {
362 ObjectUtil.checkPositive(ringSize, "ringSize");
363 ObjectUtil.checkPositive(cqeSize, "cqeSize");
364 long[] values = ioUringSetup(ringSize, cqeSize, setupFlags);
365 assert values.length == 18;
366 long cqkhead = values[0];
367 long cqktail = values[1];
368 int cqringMask = (int) values[2];
369 int cqringEntries = (int) values[3];
370 long cqArrayAddress = values[4];
371 int cqringSize = (int) values[5];
372 long cqringAddress = values[6];
373 int cqringFd = (int) values[7];
374 int cqringCapacity = (int) values[8];
375 CompletionQueue completionQueue = new CompletionQueue(
376 Buffer.wrapMemoryAddressWithNativeOrder(cqkhead, Integer.BYTES),
377 Buffer.wrapMemoryAddressWithNativeOrder(cqktail, Integer.BYTES),
378 cqringMask,
379 cqringEntries,
380 Buffer.wrapMemoryAddressWithNativeOrder(cqArrayAddress, cqringEntries * CompletionQueue.CQE_SIZE),
381 cqringSize,
382 cqringAddress,
383 cqringFd,
384 cqringCapacity);
385
386 long sqkhead = values[9];
387 long sqktail = values[10];
388 int sqringMask = (int) values[11];
389 int sqringEntries = (int) values[12];
390 long sqArrayAddress = values[13];
391 int sqringSize = (int) values[14];
392 long sqringAddress = values[15];
393 int sqringFd = (int) values[16];
394 SubmissionQueue submissionQueue = new SubmissionQueue(
395 Buffer.wrapMemoryAddressWithNativeOrder(sqkhead, Integer.BYTES),
396 Buffer.wrapMemoryAddressWithNativeOrder(sqktail, Integer.BYTES),
397 sqringMask,
398 sqringEntries,
399 Buffer.wrapMemoryAddressWithNativeOrder(sqArrayAddress, sqringEntries * SubmissionQueue.SQE_SIZE),
400 sqringSize,
401 sqringAddress,
402 sqringFd);
403 return new RingBuffer(submissionQueue, completionQueue, (int) values[17]);
404 }
405
406 static void checkAllIOSupported(int ringFd) {
407 if (!ioUringProbe(ringFd, REQUIRED_IORING_OPS)) {
408 throw new UnsupportedOperationException("Not all operations are supported: "
409 + Arrays.toString(REQUIRED_IORING_OPS));
410 }
411 }
412
413 static boolean isRecvMultishotSupported() {
414
415 return Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
416 }
417
418 static boolean isAcceptMultishotSupported(int ringFd) {
419
420 return ioUringProbe(ringFd, new int[] { Native.IORING_OP_SOCKET });
421 }
422
423 static boolean isCqeFSockNonEmptySupported(int ringFd) {
424
425 return ioUringProbe(ringFd, new int[] { Native.IORING_OP_SOCKET });
426 }
427
428 static boolean isSpliceSupported(int ringFd) {
429
430 return ioUringProbe(ringFd, new int[] { Native.IORING_OP_SPLICE });
431 }
432
433 static boolean isPollAddMultiShotSupported(int ringfd) {
434
435 return isCqeFSockNonEmptySupported(ringfd);
436 }
437
438
439
440
441
442
443 static boolean isRegisterIoWqWorkerSupported(int ringFd) {
444
445 int result = ioUringRegisterIoWqMaxWorkers(ringFd, 0, 0);
446 if (result >= 0) {
447 return true;
448 }
449
450 return false;
451 }
452
453 static boolean isRegisterBufferRingSupported(int ringFd, int flags) {
454 int entries = 2;
455 short bgid = 1;
456 long result = ioUringRegisterBufRing(ringFd, entries, bgid, flags);
457 if (result >= 0) {
458 ioUringUnRegisterBufRing(ringFd, result, entries, bgid);
459 return true;
460 }
461
462 return false;
463 }
464
465 static void checkKernelVersion(String kernelVersion) {
466 boolean enforceKernelVersion = SystemPropertyUtil.getBoolean(
467 "io.netty.transport.iouring.enforceKernelVersion", true);
468 boolean kernelSupported = checkKernelVersion(kernelVersion, 5, 9);
469 if (!kernelSupported) {
470 if (enforceKernelVersion) {
471 throw new UnsupportedOperationException(
472 "you need at least kernel version 5.9, current kernel version: " + kernelVersion);
473 } else {
474 logger.debug("Detected kernel " + kernelVersion + " does not match minimum version of 5.9, " +
475 "trying to use io_uring anyway");
476 }
477 }
478 }
479
480 private static boolean checkKernelVersion(String kernelVersion, int major, int minor) {
481 String[] versionComponents = kernelVersion.split("\\.");
482 if (versionComponents.length < 3) {
483 return false;
484 }
485 int nativeMajor;
486 try {
487 nativeMajor = Integer.parseInt(versionComponents[0]);
488 } catch (NumberFormatException e) {
489 return false;
490 }
491
492 if (nativeMajor < major) {
493 return false;
494 }
495
496 if (nativeMajor > major) {
497 return true;
498 }
499
500 int nativeMinor;
501 try {
502 nativeMinor = Integer.parseInt(versionComponents[1]);
503 } catch (NumberFormatException e) {
504 return false;
505 }
506
507 return nativeMinor >= minor;
508 }
509
510 static native boolean ioUringSetupSupportsFlags(int setupFlags);
511 private static native boolean ioUringProbe(int ringFd, int[] ios);
512 private static native long[] ioUringSetup(int entries, int cqeSize, int setupFlags);
513
514 static native int ioUringRegisterIoWqMaxWorkers(int ringFd, int maxBoundedValue, int maxUnboundedValue);
515 static native int ioUringRegisterEnableRings(int ringFd);
516 static native int ioUringRegisterRingFds(int ringFds);
517
518 static native long ioUringRegisterBufRing(int ringFd, int entries, short bufferGroup, int flags);
519 static native int ioUringUnRegisterBufRing(int ringFd, long ioUringBufRingAddr, int entries, short bufferGroupId);
520 static native int ioUringBufRingSize(int entries);
521 static native int ioUringEnter(int ringFd, int toSubmit, int minComplete, int flags);
522
523 static native void eventFdWrite(int fd, long value);
524
525 static int getFd(DefaultFileRegion fileChannel) {
526 return getFd0(fileChannel);
527 }
528
529 private static native int getFd0(Object fileChannel);
530
531 static FileDescriptor newBlockingEventFd() {
532 return new FileDescriptor(blockingEventFd());
533 }
534
535 static native void ioUringExit(long submissionQueueArrayAddress, int submissionQueueRingEntries,
536 long submissionQueueRingAddress, int submissionQueueRingSize,
537 long completionQueueRingAddress, int completionQueueRingSize,
538 int ringFd, int enterRingFd);
539
540 private static native int blockingEventFd();
541
542
543 static native int createFile(String name);
544
545 private static native int registerUnix();
546
547 static native long cmsghdrData(long hdrAddr);
548
549 static native String kernelVersion();
550
551 private Native() {
552
553 }
554
555
556 private static void loadNativeLibrary() {
557 String name = PlatformDependent.normalizedOs().toLowerCase(Locale.ROOT).trim();
558 if (!name.startsWith("linux")) {
559 throw new IllegalStateException("Only supported on Linux");
560 }
561 String staticLibName = "netty_transport_native_io_uring42";
562 String sharedLibName = staticLibName + '_' + PlatformDependent.normalizedArch();
563 ClassLoader cl = PlatformDependent.getClassLoader(Native.class);
564 try {
565 NativeLibraryLoader.load(sharedLibName, cl);
566 } catch (UnsatisfiedLinkError e1) {
567 try {
568 NativeLibraryLoader.load(staticLibName, cl);
569 logger.info("Failed to load io_uring");
570 } catch (UnsatisfiedLinkError e2) {
571 ThrowableUtil.addSuppressed(e1, e2);
572 throw e1;
573 }
574 }
575 }
576 }