View Javadoc
1   /*
2    * Copyright 2024 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.channel.uring;
17  
18  import io.netty.channel.DefaultFileRegion;
19  import io.netty.channel.unix.Buffer;
20  import io.netty.util.internal.ObjectUtil;
21  import io.netty.util.internal.logging.InternalLogger;
22  import io.netty.util.internal.logging.InternalLoggerFactory;
23  import io.netty.channel.unix.FileDescriptor;
24  import io.netty.channel.unix.PeerCredentials;
25  import io.netty.channel.unix.Unix;
26  import io.netty.util.internal.ClassInitializerUtil;
27  import io.netty.util.internal.NativeLibraryLoader;
28  import io.netty.util.internal.PlatformDependent;
29  import io.netty.util.internal.SystemPropertyUtil;
30  import io.netty.util.internal.ThrowableUtil;
31  
32  import java.io.File;
33  import java.io.IOException;
34  import java.nio.channels.Selector;
35  import java.nio.file.Path;
36  import java.util.Arrays;
37  import java.util.Locale;
38  
39  final class Native {
40      private static final InternalLogger logger = InternalLoggerFactory.getInstance(Native.class);
41  
42      static {
43          Selector selector = null;
44          try {
45              // We call Selector.open() as this will under the hood cause IOUtil to be loaded.
46              // This is a workaround for a possible classloader deadlock that could happen otherwise:
47              //
48              // See https://github.com/netty/netty/issues/10187
49              selector = Selector.open();
50          } catch (IOException ignore) {
51              // Just ignore
52          }
53  
54          // Preload all classes that will be used in the OnLoad(...) function of JNI to eliminate the possiblity of a
55          // class-loader deadlock. This is a workaround for https://github.com/netty/netty/issues/11209.
56  
57          // This needs to match all the classes that are loaded via NETTY_JNI_UTIL_LOAD_CLASS or looked up via
58          // NETTY_JNI_UTIL_FIND_CLASS.
59          ClassInitializerUtil.tryLoadClasses(
60                  Native.class,
61                  // netty_io_uring_linuxsocket
62                  PeerCredentials.class, java.io.FileDescriptor.class
63          );
64  
65          File tmpDir = PlatformDependent.tmpdir();
66          Path tmpFile = tmpDir.toPath().resolve("netty_io_uring.tmp");
67          try {
68              // First, try calling a side-effect free JNI method to see if the library was already
69              // loaded by the application.
70              Native.createFile(tmpFile.toString());
71          } catch (UnsatisfiedLinkError ignore) {
72              // The library was not previously loaded, load it now.
73              loadNativeLibrary();
74          } finally {
75              tmpFile.toFile().delete();
76              try {
77                  if (selector != null) {
78                      selector.close();
79                  }
80              } catch (IOException ignore) {
81                  // Just ignore
82              }
83          }
84          Unix.registerInternal(Native::registerUnix);
85      }
86  
87      static final int SOCK_NONBLOCK = NativeStaticallyReferencedJniMethods.sockNonblock();
88      static final int SOCK_CLOEXEC = NativeStaticallyReferencedJniMethods.sockCloexec();
89      static final short AF_INET = (short) NativeStaticallyReferencedJniMethods.afInet();
90      static final short AF_INET6 = (short) NativeStaticallyReferencedJniMethods.afInet6();
91      static final short AF_UNIX = (short) NativeStaticallyReferencedJniMethods.afUnix();
92      static final int SIZEOF_SOCKADDR_STORAGE = NativeStaticallyReferencedJniMethods.sizeofSockaddrStorage();
93      static final int SIZEOF_SOCKADDR_UN = NativeStaticallyReferencedJniMethods.sizeofSockaddrUn();
94      static final int SOCKADDR_UN_OFFSETOF_SUN_FAMILY =
95              NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunFamily();
96      static final int SOCKADDR_UN_OFFSETOF_SUN_PATH =
97              NativeStaticallyReferencedJniMethods.sockaddrUnOffsetofSunPath();
98      static final int MAX_SUN_PATH_LEN =
99              NativeStaticallyReferencedJniMethods.maxSunPathLen();
100     static final int SIZEOF_SOCKADDR_IN = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn();
101     static final int SIZEOF_SOCKADDR_IN6 = NativeStaticallyReferencedJniMethods.sizeofSockaddrIn6();
102     static final int SOCKADDR_IN_OFFSETOF_SIN_FAMILY =
103             NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinFamily();
104     static final int SOCKADDR_IN_OFFSETOF_SIN_PORT = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinPort();
105     static final int SOCKADDR_IN_OFFSETOF_SIN_ADDR = NativeStaticallyReferencedJniMethods.sockaddrInOffsetofSinAddr();
106     static final int IN_ADDRESS_OFFSETOF_S_ADDR = NativeStaticallyReferencedJniMethods.inAddressOffsetofSAddr();
107     static final int SOCKADDR_IN6_OFFSETOF_SIN6_FAMILY =
108             NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Family();
109     static final int SOCKADDR_IN6_OFFSETOF_SIN6_PORT =
110             NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Port();
111     static final int SOCKADDR_IN6_OFFSETOF_SIN6_FLOWINFO =
112             NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Flowinfo();
113     static final int SOCKADDR_IN6_OFFSETOF_SIN6_ADDR =
114             NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6Addr();
115     static final int SOCKADDR_IN6_OFFSETOF_SIN6_SCOPE_ID =
116             NativeStaticallyReferencedJniMethods.sockaddrIn6OffsetofSin6ScopeId();
117     static final int IN6_ADDRESS_OFFSETOF_S6_ADDR = NativeStaticallyReferencedJniMethods.in6AddressOffsetofS6Addr();
118     static final int SIZEOF_SIZE_T = NativeStaticallyReferencedJniMethods.sizeofSizeT();
119     static final int SIZEOF_IOVEC = NativeStaticallyReferencedJniMethods.sizeofIovec();
120     static final int CMSG_SPACE = NativeStaticallyReferencedJniMethods.cmsgSpace();
121     static final int CMSG_SPACE_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgSpaceForFd();
122     static final int CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsgLen();
123     static final int CMSG_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.cmsgLenForFd();
124     static final int MSG_CONTROL_LEN_FOR_FD = NativeStaticallyReferencedJniMethods.msgControlLenForFd();
125     static final int CMSG_OFFSETOF_CMSG_LEN = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLen();
126     static final int CMSG_OFFSETOF_CMSG_LEVEL = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgLevel();
127     static final int CMSG_OFFSETOF_CMSG_TYPE = NativeStaticallyReferencedJniMethods.cmsghdrOffsetofCmsgType();
128 
129     static final int IO_URING_BUFFER_RING_TAIL = NativeStaticallyReferencedJniMethods.ioUringBufferRingOffsetTail();
130 
131     static final int IOVEC_OFFSETOF_IOV_BASE = NativeStaticallyReferencedJniMethods.iovecOffsetofIovBase();
132     static final int IOVEC_OFFSETOF_IOV_LEN = NativeStaticallyReferencedJniMethods.iovecOffsetofIovLen();
133     static final int SIZEOF_MSGHDR = NativeStaticallyReferencedJniMethods.sizeofMsghdr();
134     static final int MSGHDR_OFFSETOF_MSG_NAME = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgName();
135     static final int MSGHDR_OFFSETOF_MSG_NAMELEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgNamelen();
136     static final int MSGHDR_OFFSETOF_MSG_IOV = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIov();
137     static final int MSGHDR_OFFSETOF_MSG_IOVLEN = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgIovlen();
138     static final int MSGHDR_OFFSETOF_MSG_CONTROL = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControl();
139     static final int MSGHDR_OFFSETOF_MSG_CONTROLLEN =
140             NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgControllen();
141     static final int MSGHDR_OFFSETOF_MSG_FLAGS = NativeStaticallyReferencedJniMethods.msghdrOffsetofMsgFlags();
142     static final int POLLIN = NativeStaticallyReferencedJniMethods.pollin();
143     static final int POLLOUT = NativeStaticallyReferencedJniMethods.pollout();
144     static final int POLLRDHUP = NativeStaticallyReferencedJniMethods.pollrdhup();
145     static final int ERRNO_ECANCELED_NEGATIVE = -NativeStaticallyReferencedJniMethods.ecanceled();
146     static final int ERRNO_ETIME_NEGATIVE = -NativeStaticallyReferencedJniMethods.etime();
147     static final int ERRNO_NOBUFS_NEGATIVE = -NativeStaticallyReferencedJniMethods.enobufs();
148 
149     static final int PAGE_SIZE = NativeStaticallyReferencedJniMethods.pageSize();
150 
151     static final int SIZEOF_IOURING_BUF = NativeStaticallyReferencedJniMethods.sizeofIoUringBuf();
152     static final int IOURING_BUFFER_OFFSETOF_ADDR = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetAddr();
153     static final int IOURING_BUFFER_OFFSETOF_LEN = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetLen();
154     static final int IOURING_BUFFER_OFFSETOF_BID = NativeStaticallyReferencedJniMethods.ioUringBufferOffsetBid();
155 
156     // These constants must be defined to have the same numeric value as their corresponding
157     // ordinal in the enum defined in the io_uring.h header file.
158     // DO NOT CHANGE THESE VALUES!
159     static final byte IORING_OP_NOP = 0; // Specified by IORING_OP_NOP in io_uring.h
160     static final byte IORING_OP_READV = 1; // Specified by IORING_OP_READV in io_uring.h
161     static final byte IORING_OP_WRITEV = 2; // Specified by IORING_OP_WRITEV in io_uring.h
162     static final byte IORING_OP_FSYNC = 3; // Specified by IORING_OP_FSYNC in io_uring.h
163     static final byte IORING_OP_READ_FIXED = 4; // Specified by IORING_OP_READ_FIXED in io_uring.h
164     static final byte IORING_OP_WRITE_FIXED = 5; // Specified by IORING_OP_WRITE_FIXED in io_uring.h
165     static final byte IORING_OP_POLL_ADD = 6; // Specified by IORING_OP_POLL_ADD in io_uring.h
166     static final byte IORING_OP_POLL_REMOVE = 7; // Specified by IORING_OP_POLL_REMOVE in io_uring.h
167     static final byte IORING_OP_SYNC_FILE_RANGE = 8; // Specified by IORING_OP_SYNC_FILE_RANGE in io_uring.h
168     static final byte IORING_OP_SENDMSG = 9; // Specified by IORING_OP_SENDMSG in io_uring.h
169     static final byte IORING_OP_RECVMSG = 10; // Specified by IORING_OP_RECVMSG in io_uring.h
170     static final byte IORING_OP_TIMEOUT = 11; // Specified by IORING_OP_TIMEOUT in io_uring.h
171     static final byte IORING_OP_TIMEOUT_REMOVE = 12; // Specified by IORING_OP_TIMEOUT_REMOVE in io_uring.h
172     static final byte IORING_OP_ACCEPT = 13; // Specified by IORING_OP_ACCEPT in io_uring.h
173     static final byte IORING_OP_ASYNC_CANCEL = 14; // Specified by IORING_OP_ASYNC_CANCEL in io_uring.h
174     static final byte IORING_OP_LINK_TIMEOUT = 15; // Specified by IORING_OP_LINK_TIMEOUT in io_uring.h
175     static final byte IORING_OP_CONNECT = 16; // Specified by IORING_OP_CONNECT in io_uring.h
176     static final byte IORING_OP_FALLOCATE = 17; // Specified by IORING_OP_FALLOCATE in io_uring.h
177     static final byte IORING_OP_OPENAT = 18; // Specified by IORING_OP_OPENAT in io_uring.h
178     static final byte IORING_OP_CLOSE = 19; // Specified by IORING_OP_CLOSE in io_uring.h
179     static final byte IORING_OP_FILES_UPDATE = 20; // Specified by IORING_OP_FILES_UPDATE in io_uring.h
180     static final byte IORING_OP_STATX = 21; // Specified by IORING_OP_STATX in io_uring.h
181     static final byte IORING_OP_READ = 22; // Specified by IORING_OP_READ in io_uring.h
182     static final byte IORING_OP_WRITE = 23; // Specified by IORING_OP_WRITE in io_uring.h
183     static final byte IORING_OP_FADVISE = 24; // Specified by IORING_OP_FADVISE in io_uring.h
184     static final byte IORING_OP_MADVISE = 25; // Specified by IORING_OP_MADVISE in io_uring.h
185     static final byte IORING_OP_SEND = 26; // Specified by IORING_OP_SEND in io_uring.h
186     static final byte IORING_OP_RECV = 27; // Specified by IORING_OP_RECV in io_uring.h
187     static final byte IORING_OP_OPENAT2 = 28; // Specified by IORING_OP_OPENAT2 in io_uring.h
188     static final byte IORING_OP_EPOLL_CTL = 29; // Specified by IORING_OP_EPOLL_CTL in io_uring.h
189     static final byte IORING_OP_SPLICE = 30; // Specified by IORING_OP_SPLICE in io_uring.h
190     static final byte IORING_OP_PROVIDE_BUFFERS = 31; // Specified by IORING_OP_PROVIDE_BUFFERS in io_uring.h
191     static final byte IORING_OP_REMOVE_BUFFERS = 32; // Specified by IORING_OP_REMOVE_BUFFERS in io_uring.h
192     static final byte IORING_OP_TEE = 33; // Specified by IORING_OP_TEE in io_uring.h
193     static final byte IORING_OP_SHUTDOWN = 34; // Specified by IORING_OP_SHUTDOWN in io_uring.h
194     static final byte IORING_OP_RENAMEAT = 35; // Specified by IORING_OP_RENAMEAT in io_uring.h
195     static final byte IORING_OP_UNLINKAT = 36; // Specified by IORING_OP_UNLINKAT in io_uring.h
196     static final byte IORING_OP_MKDIRAT = 37; // Specified by IORING_OP_MKDIRAT in io_uring.h
197     static final byte IORING_OP_SYMLINKAT = 38; // Specified by IORING_OP_SYMLINKAT in io_uring.h
198     static final byte IORING_OP_LINKAT = 39; // Specified by IORING_OP_LINKAT in io_uring.h
199     static final byte IORING_OP_MSG_RING = 40;
200     static final byte IORING_OP_FSETXATTR = 41;
201     static final byte IORING_OP_SETXATTR = 42;
202     static final byte IORING_OP_FGETXATTR = 43;
203     static final byte IORING_OP_GETXATTR = 44;
204     static final byte IORING_OP_SOCKET = 45;
205     static final byte IORING_OP_URING_CMD = 46;
206     static final byte IORING_OP_SEND_ZC = 47;
207     static final byte IORING_OP_SENDMSG_ZC = 48;
208     static final byte IORING_OP_READ_MULTISHOT = 49;
209     static final byte IORING_OP_WAITID = 50;
210     static final byte IORING_OP_FUTEX_WAIT = 51;
211     static final byte IORING_OP_FUTEX_WAKE = 52;
212     static final byte IORING_OP_FUTEX_WAITV = 53;
213     static final byte IORING_OP_FIXED_FD_INSTALL = 54;
214     static final byte IORING_OP_FTRUNCATE = 55;
215     static final byte IORING_OP_BIND = 56;
216     static final byte IORING_CQE_F_BUFFER = 1 << 0;
217     static final byte IORING_CQE_F_MORE = 1 << 1;
218     static final byte IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
219     static final byte IORING_CQE_F_BUF_MORE = 1 << 4;
220 
221     static final int IORING_SETUP_CQSIZE = 1 << 3;
222     static final int IORING_SETUP_CLAMP = 1 << 4;
223 
224     static final int IORING_SETUP_R_DISABLED = 1 << 6;
225     static final int IORING_SETUP_SUBMIT_ALL = 1 << 7;
226     static final int IORING_SETUP_SINGLE_ISSUER = 1 << 12;
227     static final int IORING_SETUP_DEFER_TASKRUN = 1 << 13;
228     static final int IORING_SETUP_NO_SQARRAY = 1 << 16;
229     static final int IORING_CQE_BUFFER_SHIFT = 16;
230 
231     static final short IORING_POLL_ADD_MULTI = 1 << 0;
232 
233     static final short IORING_RECVSEND_POLL_FIRST = 1 << 0;
234     static final short IORING_RECVSEND_BUNDLE = 1 << 4;
235     static final short IORING_RECV_MULTISHOT = 1 << 1;
236 
237     static final short IORING_ACCEPT_MULTISHOT = 1 << 0;
238     static final short IORING_ACCEPT_DONTWAIT = 1 << 1;
239     static final short IORING_ACCEPT_POLL_FIRST = 1 << 2;
240 
241     static final int IORING_FEAT_SUBMIT_STABLE = 1 << 2;
242     static final int IORING_FEAT_RECVSEND_BUNDLE = 1 << 14;
243 
244     static final int SPLICE_F_MOVE = 1;
245 
246     static final int IOU_PBUF_RING_INC = 2;
247 
248     static final int IO_URING_OP_SUPPORTED = 1;
249     static String opToStr(byte op) {
250         switch (op) {
251             case IORING_OP_NOP: return "NOP";
252             case IORING_OP_READV: return "READV";
253             case IORING_OP_WRITEV: return "WRITEV";
254             case IORING_OP_FSYNC: return "FSYNC";
255             case IORING_OP_READ_FIXED: return "READ_FIXED";
256             case IORING_OP_WRITE_FIXED: return "WRITE_FIXED";
257             case IORING_OP_POLL_ADD: return "POLL_ADD";
258             case IORING_OP_POLL_REMOVE: return "POLL_REMOVE";
259             case IORING_OP_SYNC_FILE_RANGE: return "SYNC_FILE_RANGE";
260             case IORING_OP_SENDMSG: return "SENDMSG";
261             case IORING_OP_RECVMSG: return "RECVMSG";
262             case IORING_OP_TIMEOUT: return "TIMEOUT";
263             case IORING_OP_TIMEOUT_REMOVE: return "TIMEOUT_REMOVE";
264             case IORING_OP_ACCEPT: return "ACCEPT";
265             case IORING_OP_ASYNC_CANCEL: return "ASYNC_CANCEL";
266             case IORING_OP_LINK_TIMEOUT: return "LINK_TIMEOUT";
267             case IORING_OP_CONNECT: return "CONNECT";
268             case IORING_OP_FALLOCATE: return "FALLOCATE";
269             case IORING_OP_OPENAT: return "OPENAT";
270             case IORING_OP_CLOSE: return "CLOSE";
271             case IORING_OP_FILES_UPDATE: return "FILES_UPDATE";
272             case IORING_OP_STATX: return "STATX";
273             case IORING_OP_READ: return "READ";
274             case IORING_OP_WRITE: return "WRITE";
275             case IORING_OP_FADVISE: return "FADVISE";
276             case IORING_OP_MADVISE: return "MADVISE";
277             case IORING_OP_SEND: return "SEND";
278             case IORING_OP_RECV: return "RECV";
279             case IORING_OP_OPENAT2: return "OPENAT2";
280             case IORING_OP_EPOLL_CTL: return "EPOLL_CTL";
281             case IORING_OP_SPLICE: return "SPLICE";
282             case IORING_OP_PROVIDE_BUFFERS: return "PROVIDE_BUFFERS";
283             case IORING_OP_REMOVE_BUFFERS: return "REMOVE_BUFFERS";
284             case IORING_OP_TEE: return "TEE";
285             case IORING_OP_SHUTDOWN: return "SHUTDOWN";
286             case IORING_OP_RENAMEAT: return "RENAMEAT";
287             case IORING_OP_UNLINKAT: return "UNLINKAT";
288             case IORING_OP_MKDIRAT: return "MKDIRAT";
289             case IORING_OP_SYMLINKAT: return "SYMLINKAT";
290             case IORING_OP_LINKAT: return "LINKAT";
291             default: return "[OP CODE " + op + ']';
292         }
293     }
294 
295     static final int IORING_ENTER_GETEVENTS = NativeStaticallyReferencedJniMethods.ioringEnterGetevents();
296     static final int IORING_ENTER_REGISTERED_RING = 1 << 4;
297     static final int IOSQE_ASYNC = NativeStaticallyReferencedJniMethods.iosqeAsync();
298     static final int IOSQE_LINK = NativeStaticallyReferencedJniMethods.iosqeLink();
299     static final int IOSQE_IO_DRAIN = NativeStaticallyReferencedJniMethods.iosqeDrain();
300     static final int IOSQE_BUFFER_SELECT = NativeStaticallyReferencedJniMethods.iosqeBufferSelect();
301     static final int IOSQE_CQE_SKIP_SUCCESS = 1 << 6;
302     static final int MSG_DONTWAIT = NativeStaticallyReferencedJniMethods.msgDontwait();
303     static final int MSG_FASTOPEN = NativeStaticallyReferencedJniMethods.msgFastopen();
304     static final int SOL_UDP = NativeStaticallyReferencedJniMethods.solUdp();
305     static final int SOL_SOCKET = NativeStaticallyReferencedJniMethods.solSocket();
306     static final int UDP_SEGMENT = NativeStaticallyReferencedJniMethods.udpSegment();
307     static final int SCM_RIGHTS = NativeStaticallyReferencedJniMethods.scmRights();
308     private static final int TFO_ENABLED_CLIENT_MASK = 0x1;
309     private static final int TFO_ENABLED_SERVER_MASK = 0x2;
310     private static final int TCP_FASTOPEN_MODE = NativeStaticallyReferencedJniMethods.tcpFastopenMode();
311     /**
312      * <a href ="https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt">tcp_fastopen</a> client mode enabled
313      * state.
314      */
315     static final boolean IS_SUPPORTING_TCP_FASTOPEN_CLIENT =
316             (TCP_FASTOPEN_MODE & TFO_ENABLED_CLIENT_MASK) == TFO_ENABLED_CLIENT_MASK;
317     /**
318      * <a href ="https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt">tcp_fastopen</a> server mode enabled
319      * state.
320      */
321     static final boolean IS_SUPPORTING_TCP_FASTOPEN_SERVER =
322             (TCP_FASTOPEN_MODE & TFO_ENABLED_SERVER_MASK) == TFO_ENABLED_SERVER_MASK;
323 
324     private static final int[] REQUIRED_IORING_OPS = {
325             IORING_OP_POLL_ADD,
326             IORING_OP_TIMEOUT,
327             IORING_OP_ACCEPT,
328             IORING_OP_READ,
329             IORING_OP_WRITE,
330             IORING_OP_POLL_REMOVE,
331             IORING_OP_CONNECT,
332             IORING_OP_CLOSE,
333             IORING_OP_WRITEV,
334             IORING_OP_SENDMSG,
335             IORING_OP_RECVMSG,
336             IORING_OP_ASYNC_CANCEL,
337             IORING_OP_RECV,
338             IORING_OP_NOP,
339             IORING_OP_SHUTDOWN,
340             IORING_OP_SEND
341     };
342 
343     static int setupFlags() {
344         int flags = Native.IORING_SETUP_R_DISABLED | Native.IORING_SETUP_CLAMP;
345         if (IoUring.isSetupSubmitAllSupported()) {
346             flags |= Native.IORING_SETUP_SUBMIT_ALL;
347         }
348 
349         // See https://github.com/axboe/liburing/wiki/io_uring-and-networking-in-2023#task-work
350         if (IoUring.isSetupSingleIssuerSupported()) {
351             flags |= Native.IORING_SETUP_SINGLE_ISSUER;
352         }
353         if (IoUring.isSetupDeferTaskrunSupported()) {
354             flags |= Native.IORING_SETUP_DEFER_TASKRUN;
355         }
356         // liburing uses IORING_SETUP_NO_SQARRAY by default these days, we should do the same by default if possible.
357         // See https://github.com/axboe/liburing/releases/tag/liburing-2.6
358         if (IoUring.isIoringSetupNoSqarraySupported()) {
359             flags  |= Native.IORING_SETUP_NO_SQARRAY;
360         }
361         return flags;
362     }
363 
364     static RingBuffer createRingBuffer(int ringSize, int setupFlags) {
365         return createRingBuffer(ringSize, ringSize * 2, setupFlags);
366     }
367 
368     static RingBuffer createRingBuffer(int ringSize, int cqeSize, int setupFlags) {
369         ObjectUtil.checkPositive(ringSize, "ringSize");
370         ObjectUtil.checkPositive(cqeSize, "cqeSize");
371         long[] values = ioUringSetup(ringSize, cqeSize, setupFlags);
372         assert values.length == 18;
373         long cqkhead = values[0];
374         long cqktail = values[1];
375         int cqringMask = (int) values[2];
376         int cqringEntries = (int) values[3];
377         long cqArrayAddress = values[4];
378         int cqringSize = (int) values[5];
379         long cqringAddress = values[6];
380         int cqringFd = (int) values[7];
381         int cqringCapacity = (int) values[8];
382         CompletionQueue completionQueue = new CompletionQueue(
383                 Buffer.wrapMemoryAddressWithNativeOrder(cqkhead, Integer.BYTES),
384                 Buffer.wrapMemoryAddressWithNativeOrder(cqktail, Integer.BYTES),
385                 cqringMask,
386                 cqringEntries,
387                 Buffer.wrapMemoryAddressWithNativeOrder(cqArrayAddress, cqringEntries * CompletionQueue.CQE_SIZE),
388                 cqringSize,
389                 cqringAddress,
390                 cqringFd,
391                 cqringCapacity);
392 
393         long sqkhead = values[9];
394         long sqktail = values[10];
395         int sqringMask = (int) values[11];
396         int sqringEntries = (int) values[12];
397         long sqArrayAddress = values[13];
398         int sqringSize = (int) values[14];
399         long sqringAddress = values[15];
400         int sqringFd = (int) values[16];
401         SubmissionQueue submissionQueue = new SubmissionQueue(
402                 Buffer.wrapMemoryAddressWithNativeOrder(sqkhead, Integer.BYTES),
403                 Buffer.wrapMemoryAddressWithNativeOrder(sqktail, Integer.BYTES),
404                 sqringMask,
405                 sqringEntries,
406                 Buffer.wrapMemoryAddressWithNativeOrder(sqArrayAddress, sqringEntries * SubmissionQueue.SQE_SIZE),
407                 sqringSize,
408                 sqringAddress,
409                 sqringFd);
410         return new RingBuffer(submissionQueue, completionQueue, (int) values[17]);
411     }
412 
413     static void checkAllIOSupported(IoUringProbe probe) {
414         if (!ioUringProbe(probe, REQUIRED_IORING_OPS)) {
415             throw new UnsupportedOperationException("Not all operations are supported: "
416                     + Arrays.toString(REQUIRED_IORING_OPS));
417         }
418     }
419 
420     static boolean isRecvMultishotSupported() {
421         // Added in the same release as IORING_SETUP_SINGLE_ISSUER.
422         return Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
423     }
424 
425     static boolean isAcceptMultishotSupported(IoUringProbe probe) {
426         // IORING_OP_SOCKET was added in the same release (5.19);
427         return ioUringProbe(probe, new int[] { Native.IORING_OP_SOCKET });
428     }
429 
430     static boolean isCqeFSockNonEmptySupported(IoUringProbe probe) {
431         // IORING_OP_SOCKET was added in the same release (5.19);
432         return ioUringProbe(probe, new int[] { Native.IORING_OP_SOCKET });
433     }
434 
435     static boolean isSpliceSupported(IoUringProbe probe) {
436         // IORING_OP_SPLICE Available since 5.7
437         return ioUringProbe(probe, new int[] { Native.IORING_OP_SPLICE });
438     }
439 
440     static boolean isPollAddMultiShotSupported(IoUringProbe probe) {
441         // Was added in the same release and we also need this feature to correctly handle edge-triggered mode.
442         return isCqeFSockNonEmptySupported(probe);
443     }
444 
445     /**
446      * check current kernel version whether support io_uring_register_io_wq_worker
447      * Available since 5.15.
448      * @return true if support io_uring_register_io_wq_worker
449      */
450     static boolean isRegisterIoWqWorkerSupported(int ringFd) {
451         // See https://github.com/torvalds/linux/blob/v5.5/fs/io_uring.c#L5488C10-L5488C16
452         int result = ioUringRegisterIoWqMaxWorkers(ringFd, 0, 0);
453         if (result >= 0) {
454             return true;
455         }
456         // This is not supported and so will return -EINVAL
457         return false;
458     }
459 
460     static boolean isRegisterBufferRingSupported(int ringFd, int flags) {
461         int entries = 2;
462         short bgid = 1;
463         long result = ioUringRegisterBufRing(ringFd, entries, bgid, flags);
464         if (result >= 0) {
465             ioUringUnRegisterBufRing(ringFd, result, entries, bgid);
466             return true;
467         }
468         // This is not supported and so will return -EINVAL
469         return false;
470     }
471 
472     static void checkKernelVersion(String kernelVersion) {
473         boolean enforceKernelVersion = SystemPropertyUtil.getBoolean(
474                 "io.netty.transport.iouring.enforceKernelVersion", true);
475         boolean kernelSupported = checkKernelVersion(kernelVersion, 5, 9);
476         if (!kernelSupported) {
477             if (enforceKernelVersion) {
478                 throw new UnsupportedOperationException(
479                         "you need at least kernel version 5.9, current kernel version: " + kernelVersion);
480             } else {
481                 logger.debug("Detected kernel " + kernelVersion + " does not match minimum version of 5.9, " +
482                         "trying to use io_uring anyway");
483             }
484         }
485     }
486 
487     private static boolean checkKernelVersion(String kernelVersion, int major, int minor) {
488         String[] versionComponents = kernelVersion.split("\\.");
489         if (versionComponents.length < 3) {
490             return false;
491         }
492         int nativeMajor;
493         try {
494             nativeMajor = Integer.parseInt(versionComponents[0]);
495         } catch (NumberFormatException e) {
496             return false;
497         }
498 
499         if (nativeMajor < major) {
500             return false;
501         }
502 
503         if (nativeMajor > major) {
504             return true;
505         }
506 
507         int nativeMinor;
508         try {
509             nativeMinor = Integer.parseInt(versionComponents[1]);
510         } catch (NumberFormatException e) {
511             return false;
512         }
513 
514         return nativeMinor >= minor;
515     }
516 
517     static final class IoUringProbe {
518         final byte lastOp;
519         final byte opsLen;
520         final IoUringProbeOp[] ops;
521 
522         IoUringProbe(int[] values) {
523             int idx = 0;
524             lastOp = (byte) values[idx++];
525             opsLen = (byte) values[idx++];
526             ops  = new IoUringProbeOp[opsLen];
527             for (int i = 0; i < opsLen; i++) {
528                 ops[i] = new IoUringProbeOp((byte) values[idx++], values[idx++]);
529             }
530         }
531     }
532 
533     static class IoUringProbeOp {
534         final byte op;
535         final int flags;
536 
537         IoUringProbeOp(byte op, int flags) {
538             this.op = op;
539             this.flags = flags;
540         }
541     }
542 
543     static boolean ioUringProbe(IoUringProbe probe, int[] ops) {
544         IoUringProbeOp[] ioUringProbeOps = probe.ops;
545         if (ioUringProbeOps == null) {
546             return false;
547         }
548         for (int op : ops) {
549             if (op > probe.lastOp || (ioUringProbeOps[op].flags & IO_URING_OP_SUPPORTED) == 0) {
550                 return false;
551             }
552         }
553         return true;
554     }
555 
556     static native boolean ioUringSetupSupportsFlags(int setupFlags);;
557     private static native long[] ioUringSetup(int entries, int cqeSize, int setupFlags);
558 
559     static IoUringProbe ioUringProbe(int ringfd) {
560         int[] values = ioUringProbe0(ringfd);
561         if (values == null) {
562             return null;
563         }
564         return new IoUringProbe(values);
565     }
566     private static native int[] ioUringProbe0(int ringFd);
567 
568     static native int ioUringRegisterIoWqMaxWorkers(int ringFd, int maxBoundedValue, int maxUnboundedValue);
569     static native int ioUringRegisterEnableRings(int ringFd);
570     static native int ioUringRegisterRingFds(int ringFds);
571 
572     static native long ioUringRegisterBufRing(int ringFd, int entries, short bufferGroup, int flags);
573     static native int ioUringUnRegisterBufRing(int ringFd, long ioUringBufRingAddr, int entries, short bufferGroupId);
574     static native int ioUringBufRingSize(int entries);
575     static native int ioUringEnter(int ringFd, int toSubmit, int minComplete, int flags);
576 
577     static native void eventFdWrite(int fd, long value);
578 
579     static int getFd(DefaultFileRegion fileChannel) {
580         return getFd0(fileChannel);
581     }
582 
583     private static native int getFd0(Object fileChannel);
584 
585     static FileDescriptor newBlockingEventFd() {
586         return new FileDescriptor(blockingEventFd());
587     }
588 
589     static native void ioUringExit(long submissionQueueArrayAddress, int submissionQueueRingEntries,
590                                           long submissionQueueRingAddress, int submissionQueueRingSize,
591                                           long completionQueueRingAddress, int completionQueueRingSize,
592                                           int ringFd, int enterRingFd);
593 
594     private static native int blockingEventFd();
595 
596     // for testing only!
597     static native int createFile(String name);
598 
599     private static native int registerUnix();
600 
601     static native long cmsghdrData(long hdrAddr);
602 
603     static native String kernelVersion();
604 
605     private Native() {
606         // utility
607     }
608 
609     // From io_uring native library
610     private static void loadNativeLibrary() {
611         String name = PlatformDependent.normalizedOs().toLowerCase(Locale.ROOT).trim();
612         if (!name.startsWith("linux")) {
613             throw new IllegalStateException("Only supported on Linux");
614         }
615         String staticLibName = "netty_transport_native_io_uring42";
616         String sharedLibName = staticLibName + '_' + PlatformDependent.normalizedArch();
617         ClassLoader cl = PlatformDependent.getClassLoader(Native.class);
618         try {
619             NativeLibraryLoader.load(sharedLibName, cl);
620         } catch (UnsatisfiedLinkError e1) {
621             try {
622                 NativeLibraryLoader.load(staticLibName, cl);
623                 logger.info("Failed to load io_uring");
624             } catch (UnsatisfiedLinkError e2) {
625                 ThrowableUtil.addSuppressed(e1, e2);
626                 throw e1;
627             }
628         }
629     }
630 }