1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.buffer.ByteBuf;
19 import io.netty.channel.ChannelOption;
20 import io.netty.channel.unix.Buffer;
21 import io.netty.channel.unix.Limits;
22 import io.netty.util.internal.PlatformDependent;
23 import io.netty.util.internal.SystemPropertyUtil;
24 import io.netty.util.internal.logging.InternalLogger;
25 import io.netty.util.internal.logging.InternalLoggerFactory;
26
27 import java.nio.ByteBuffer;
28
29 public final class IoUring {
30
31 private static final Throwable UNAVAILABILITY_CAUSE;
32 private static final boolean IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
33 private static final boolean IORING_SPLICE_SUPPORTED;
34 private static final boolean IORING_SEND_ZC_SUPPORTED;
35 private static final boolean IORING_SENDMSG_ZC_SUPPORTED;
36 private static final boolean IORING_ACCEPT_NO_WAIT_SUPPORTED;
37 private static final boolean IORING_ACCEPT_MULTISHOT_SUPPORTED;
38 private static final boolean IORING_RECV_MULTISHOT_SUPPORTED;
39 private static final boolean IORING_RECVSEND_BUNDLE_SUPPORTED;
40 private static final boolean IORING_POLL_ADD_MULTISHOT_SUPPORTED;
41 private static final boolean IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
42 private static final boolean IORING_SETUP_SUBMIT_ALL_SUPPORTED;
43 private static final boolean IORING_SETUP_CQ_SIZE_SUPPORTED;
44 private static final boolean IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
45 private static final boolean IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
46 private static final boolean IORING_SETUP_NO_SQARRAY_SUPPORTED;
47 private static final boolean IORING_REGISTER_BUFFER_RING_SUPPORTED;
48 private static final boolean IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
49 private static final boolean IORING_ACCEPT_MULTISHOT_ENABLED;
50 private static final boolean IORING_RECV_MULTISHOT_ENABLED;
51 private static final boolean IORING_RECVSEND_BUNDLE_ENABLED;
52 private static final boolean IORING_POLL_ADD_MULTISHOT_ENABLED;
53 static final int NUM_ELEMENTS_IOVEC;
54 static final int DEFAULT_RING_SIZE;
55 static final int DEFAULT_CQ_SIZE;
56 static final int DISABLE_SETUP_CQ_SIZE = -1;
57
58 private static final InternalLogger logger;
59
60 static {
61 logger = InternalLoggerFactory.getInstance(IoUring.class);
62 Throwable cause = null;
63 boolean socketNonEmptySupported = false;
64 boolean spliceSupported = false;
65 boolean sendZcSupported = false;
66 boolean sendmsgZcSupported = false;
67 boolean acceptSupportNoWait = false;
68 boolean acceptMultishotSupported = false;
69 boolean recvsendBundleSupported = false;
70 boolean recvMultishotSupported = false;
71 boolean pollAddMultishotSupported = false;
72 boolean registerIowqWorkersSupported = false;
73 boolean submitAllSupported = false;
74 boolean setUpCqSizeSupported = false;
75 boolean singleIssuerSupported = false;
76 boolean deferTaskrunSupported = false;
77 boolean noSqarraySupported = false;
78 boolean registerBufferRingSupported = false;
79 boolean registerBufferRingIncSupported = false;
80 int numElementsIoVec = 10;
81
82 String kernelVersion = "[unknown]";
83 try {
84 if (SystemPropertyUtil.getBoolean("io.netty.transport.noNative", false)) {
85 cause = new UnsupportedOperationException(
86 "Native transport was explicit disabled with -Dio.netty.transport.noNative=true");
87 } else {
88 kernelVersion = Native.kernelVersion();
89 Native.checkKernelVersion(kernelVersion);
90 if (PlatformDependent.javaVersion() >= 9) {
91 RingBuffer ringBuffer = null;
92 try {
93 ringBuffer = Native.createRingBuffer(1, 0);
94 if ((ringBuffer.features() & Native.IORING_FEAT_SUBMIT_STABLE) == 0) {
95
96 throw new UnsupportedOperationException("IORING_FEAT_SUBMIT_STABLE not supported!");
97 }
98
99
100 numElementsIoVec = SystemPropertyUtil.getInt(
101 "io.netty.iouring.numElementsIoVec", 10 * Limits.IOV_MAX);
102 Native.IoUringProbe ioUringProbe = Native.ioUringProbe(ringBuffer.fd());
103 Native.checkAllIOSupported(ioUringProbe);
104 socketNonEmptySupported = Native.isCqeFSockNonEmptySupported(ioUringProbe);
105 spliceSupported = Native.isSpliceSupported(ioUringProbe);
106 recvsendBundleSupported = (ringBuffer.features() & Native.IORING_FEAT_RECVSEND_BUNDLE) != 0;
107 sendZcSupported = Native.isSendZcSupported(ioUringProbe);
108 sendmsgZcSupported = Native.isSendmsgZcSupported(ioUringProbe);
109
110 acceptSupportNoWait = recvsendBundleSupported;
111
112 acceptMultishotSupported = Native.isAcceptMultishotSupported(ioUringProbe);
113 recvMultishotSupported = Native.isRecvMultishotSupported();
114 pollAddMultishotSupported = Native.isPollAddMultiShotSupported(ioUringProbe);
115 registerIowqWorkersSupported = Native.isRegisterIoWqWorkerSupported(ringBuffer.fd());
116 submitAllSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SUBMIT_ALL);
117 setUpCqSizeSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_CQSIZE);
118 singleIssuerSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
119
120
121 deferTaskrunSupported = Native.ioUringSetupSupportsFlags(
122 Native.IORING_SETUP_SINGLE_ISSUER | Native.IORING_SETUP_DEFER_TASKRUN);
123 noSqarraySupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_NO_SQARRAY);
124 registerBufferRingSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(), 0);
125 registerBufferRingIncSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(),
126 Native.IOU_PBUF_RING_INC);
127 } finally {
128 if (ringBuffer != null) {
129 try {
130 ringBuffer.close();
131 } catch (Exception ignore) {
132
133 }
134 }
135 }
136 } else {
137 cause = new UnsupportedOperationException("Java 9+ is required");
138 }
139 }
140 } catch (Throwable t) {
141 cause = t;
142 }
143 if (cause != null) {
144 if (logger.isTraceEnabled()) {
145 logger.debug("IoUring support is not available using kernel {}", kernelVersion, cause);
146 } else if (logger.isDebugEnabled()) {
147 logger.debug("IoUring support is not available using kernel {}: {}", kernelVersion, cause.getMessage());
148 }
149 } else {
150 if (logger.isDebugEnabled()) {
151 logger.debug("IoUring support is available using kernel {} (" +
152 "CQE_F_SOCK_NONEMPTY_SUPPORTED={}, " +
153 "SPLICE_SUPPORTED={}, " +
154 "ACCEPT_NO_WAIT_SUPPORTED={}, " +
155 "ACCEPT_MULTISHOT_SUPPORTED={}, " +
156 "POLL_ADD_MULTISHOT_SUPPORTED={} " +
157 "RECV_MULTISHOT_SUPPORTED={}, " +
158 "IORING_RECVSEND_BUNDLE_SUPPORTED={}, " +
159 "REGISTER_IOWQ_MAX_WORKERS_SUPPORTED={}, " +
160 "SETUP_SUBMIT_ALL_SUPPORTED={}, " +
161 "SETUP_SINGLE_ISSUER_SUPPORTED={}, " +
162 "SETUP_DEFER_TASKRUN_SUPPORTED={}, " +
163 "REGISTER_BUFFER_RING_SUPPORTED={}, " +
164 "REGISTER_BUFFER_RING_INC_SUPPORTED={}," +
165 "SEND_ZC_SUPPORTED={},",
166 "SENDMSG_ZC_SUPPORTED={}" +
167 ")", kernelVersion, socketNonEmptySupported, spliceSupported, acceptSupportNoWait,
168 acceptMultishotSupported, pollAddMultishotSupported, recvMultishotSupported,
169 recvsendBundleSupported, registerIowqWorkersSupported, submitAllSupported,
170 singleIssuerSupported, deferTaskrunSupported,
171 registerBufferRingSupported, registerBufferRingIncSupported,
172 sendZcSupported, sendmsgZcSupported
173 );
174 }
175 }
176 UNAVAILABILITY_CAUSE = cause;
177 IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED = socketNonEmptySupported;
178 IORING_SPLICE_SUPPORTED = spliceSupported;
179 IORING_SEND_ZC_SUPPORTED = sendZcSupported;
180 IORING_SENDMSG_ZC_SUPPORTED = sendmsgZcSupported;
181 IORING_ACCEPT_NO_WAIT_SUPPORTED = acceptSupportNoWait;
182 IORING_ACCEPT_MULTISHOT_SUPPORTED = acceptMultishotSupported;
183 IORING_RECV_MULTISHOT_SUPPORTED = recvMultishotSupported;
184 IORING_RECVSEND_BUNDLE_SUPPORTED = recvsendBundleSupported;
185 IORING_POLL_ADD_MULTISHOT_SUPPORTED = pollAddMultishotSupported;
186 IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED = registerIowqWorkersSupported;
187 IORING_SETUP_SUBMIT_ALL_SUPPORTED = submitAllSupported;
188 IORING_SETUP_CQ_SIZE_SUPPORTED = setUpCqSizeSupported;
189 IORING_SETUP_SINGLE_ISSUER_SUPPORTED = singleIssuerSupported;
190 IORING_SETUP_DEFER_TASKRUN_SUPPORTED = deferTaskrunSupported;
191 IORING_SETUP_NO_SQARRAY_SUPPORTED = noSqarraySupported;
192 IORING_REGISTER_BUFFER_RING_SUPPORTED = registerBufferRingSupported;
193 IORING_REGISTER_BUFFER_RING_INC_SUPPORTED = registerBufferRingIncSupported;
194
195 IORING_ACCEPT_MULTISHOT_ENABLED = IORING_ACCEPT_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
196 "io.netty.iouring.acceptMultiShotEnabled", true);
197 IORING_RECV_MULTISHOT_ENABLED = IORING_RECV_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
198 "io.netty.iouring.recvMultiShotEnabled", true);
199
200
201
202 IORING_RECVSEND_BUNDLE_ENABLED = IORING_RECVSEND_BUNDLE_SUPPORTED && SystemPropertyUtil.getBoolean(
203 "io.netty.iouring.recvsendBundleEnabled", false);
204 IORING_POLL_ADD_MULTISHOT_ENABLED = IORING_POLL_ADD_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
205 "io.netty.iouring.pollAddMultishotEnabled", true);
206 NUM_ELEMENTS_IOVEC = numElementsIoVec;
207
208 DEFAULT_RING_SIZE = Math.max(16, SystemPropertyUtil.getInt("io.netty.iouring.ringSize", 128));
209
210 if (IORING_SETUP_CQ_SIZE_SUPPORTED) {
211 DEFAULT_CQ_SIZE = Math.max(DEFAULT_RING_SIZE,
212 SystemPropertyUtil.getInt("io.netty.iouring.cqSize", 4096));
213 } else {
214 DEFAULT_CQ_SIZE = DISABLE_SETUP_CQ_SIZE;
215 }
216 }
217
218 public static boolean isAvailable() {
219 return UNAVAILABILITY_CAUSE == null;
220 }
221
222
223
224
225
226
227
228 public static boolean isTcpFastOpenClientSideAvailable() {
229 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_CLIENT;
230 }
231
232
233
234
235
236
237
238 public static boolean isTcpFastOpenServerSideAvailable() {
239 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_SERVER;
240 }
241
242 static boolean isCqeFSockNonEmptySupported() {
243 return IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
244 }
245
246
247
248
249
250
251 public static boolean isSpliceSupported() {
252 return IORING_SPLICE_SUPPORTED;
253 }
254
255
256
257
258
259
260 static boolean isSendZcSupported() {
261 return IORING_SEND_ZC_SUPPORTED;
262 }
263
264
265
266
267
268
269 static boolean isSendmsgZcSupported() {
270 return IORING_SENDMSG_ZC_SUPPORTED;
271 }
272
273 static boolean isAcceptNoWaitSupported() {
274 return IORING_ACCEPT_NO_WAIT_SUPPORTED;
275 }
276
277 static boolean isAcceptMultishotSupported() {
278 return IORING_ACCEPT_MULTISHOT_SUPPORTED;
279 }
280
281 static boolean isRecvMultishotSupported() {
282 return IORING_RECV_MULTISHOT_SUPPORTED;
283 }
284
285 static boolean isRecvsendBundleSupported() {
286 return IORING_RECVSEND_BUNDLE_SUPPORTED;
287 }
288
289 static boolean isPollAddMultishotSupported() {
290 return IORING_POLL_ADD_MULTISHOT_SUPPORTED;
291 }
292
293 static boolean isRegisterIowqMaxWorkersSupported() {
294 return IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
295 }
296
297 static boolean isSetupCqeSizeSupported() {
298 return IORING_SETUP_CQ_SIZE_SUPPORTED;
299 }
300
301 static boolean isSetupSubmitAllSupported() {
302 return IORING_SETUP_SUBMIT_ALL_SUPPORTED;
303 }
304
305 static boolean isSetupSingleIssuerSupported() {
306 return IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
307 }
308
309 static boolean isSetupDeferTaskrunSupported() {
310 return IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
311 }
312
313 static boolean isIoringSetupNoSqarraySupported() {
314 return IORING_SETUP_NO_SQARRAY_SUPPORTED;
315 }
316
317
318
319
320
321 public static boolean isRegisterBufferRingSupported() {
322 return IORING_REGISTER_BUFFER_RING_SUPPORTED;
323 }
324
325
326
327
328
329
330 public static boolean isRegisterBufferRingIncSupported() {
331 return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
332 }
333
334
335
336
337
338
339 public static boolean isAcceptMultishotEnabled() {
340 return IORING_ACCEPT_MULTISHOT_ENABLED;
341 }
342
343
344
345
346
347
348 public static boolean isRecvMultishotEnabled() {
349 return IORING_RECV_MULTISHOT_ENABLED;
350 }
351
352
353
354
355
356
357 public static boolean isRecvsendBundleEnabled() {
358 return IORING_RECVSEND_BUNDLE_ENABLED;
359 }
360
361
362
363
364
365
366 public static boolean isPollAddMultishotEnabled() {
367 return IORING_POLL_ADD_MULTISHOT_ENABLED;
368 }
369
370 public static void ensureAvailability() {
371 if (UNAVAILABILITY_CAUSE != null) {
372 throw (Error) new UnsatisfiedLinkError(
373 "failed to load the required native library").initCause(UNAVAILABILITY_CAUSE);
374 }
375 }
376
377 static long memoryAddress(ByteBuf buffer) {
378 if (buffer.hasMemoryAddress()) {
379 return buffer.memoryAddress();
380 }
381
382
383
384 ByteBuffer byteBuffer = buffer.internalNioBuffer(0, buffer.capacity());
385 return Buffer.memoryAddress(byteBuffer) + byteBuffer.position();
386 }
387
388 public static Throwable unavailabilityCause() {
389 return UNAVAILABILITY_CAUSE;
390 }
391
392 private IoUring() {
393 }
394 }