1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.buffer.ByteBuf;
19 import io.netty.channel.ChannelOption;
20 import io.netty.channel.unix.Buffer;
21 import io.netty.channel.unix.Limits;
22 import io.netty.util.internal.MathUtil;
23 import io.netty.util.internal.PlatformDependent;
24 import io.netty.util.internal.SystemPropertyUtil;
25 import io.netty.util.internal.logging.InternalLogger;
26 import io.netty.util.internal.logging.InternalLoggerFactory;
27
28 import java.nio.ByteBuffer;
29
30 public final class IoUring {
31
32 private static final Throwable UNAVAILABILITY_CAUSE;
33 private static final boolean IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
34 private static final boolean UNIX_DOMAIN_SOCKET_INQ_SUPPORTED;
35 private static final boolean IORING_SPLICE_SUPPORTED;
36 private static final boolean IORING_SEND_ZC_SUPPORTED;
37 private static final boolean IORING_SENDMSG_ZC_SUPPORTED;
38 private static final boolean IORING_ACCEPT_NO_WAIT_SUPPORTED;
39 private static final boolean IORING_ACCEPT_MULTISHOT_SUPPORTED;
40 private static final boolean IORING_RECV_MULTISHOT_SUPPORTED;
41 private static final boolean IORING_RECVSEND_BUNDLE_SUPPORTED;
42 private static final boolean IORING_POLL_ADD_MULTISHOT_SUPPORTED;
43 private static final boolean IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
44 private static final boolean IORING_SETUP_SUBMIT_ALL_SUPPORTED;
45 private static final boolean IORING_SETUP_CQE_MIXED_SUPPORTED;
46 private static final boolean IORING_SETUP_CQ_SIZE_SUPPORTED;
47 private static final boolean IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
48 private static final boolean IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
49 private static final boolean IORING_SETUP_NO_SQARRAY_SUPPORTED;
50 private static final boolean IORING_REGISTER_BUFFER_RING_SUPPORTED;
51 private static final boolean IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
52 private static final boolean IORING_ENTER_NO_IOWAIT_SUPPORTED;
53 private static final boolean IORING_ACCEPT_MULTISHOT_ENABLED;
54 private static final boolean IORING_RECV_MULTISHOT_ENABLED;
55 private static final boolean IORING_RECVSEND_BUNDLE_ENABLED;
56 private static final boolean IORING_POLL_ADD_MULTISHOT_ENABLED;
57 private static final boolean IORING_ENTER_NO_IOWAIT_ENABLED;
58 static final int NUM_ELEMENTS_IOVEC;
59 static final int DEFAULT_RING_SIZE;
60 static final int DEFAULT_CQ_SIZE;
61 static final int DEFAULT_PENDING_OPS_INITIAL_CAPACITY;
62 static final int DISABLE_SETUP_CQ_SIZE = -1;
63
64 private static final InternalLogger logger;
65
66 static {
67 logger = InternalLoggerFactory.getInstance(IoUring.class);
68 Throwable cause = null;
69 boolean socketNonEmptySupported = false;
70 boolean unixDomainSocketInqSupported = false;
71 boolean spliceSupported = false;
72 boolean sendZcSupported = false;
73 boolean sendmsgZcSupported = false;
74 boolean acceptSupportNoWait = false;
75 boolean acceptMultishotSupported = false;
76 boolean recvsendBundleSupported = false;
77 boolean recvMultishotSupported = false;
78 boolean pollAddMultishotSupported = false;
79 boolean registerIowqWorkersSupported = false;
80 boolean submitAllSupported = false;
81 boolean cqeMixedSupported = false;
82 boolean setUpCqSizeSupported = false;
83 boolean singleIssuerSupported = false;
84 boolean deferTaskrunSupported = false;
85 boolean noSqarraySupported = false;
86 boolean registerBufferRingSupported = false;
87 boolean registerBufferRingIncSupported = false;
88 boolean enterNoIoWaitSupported = false;
89 int numElementsIoVec = 10;
90 int pendingOpsInitialCapacity;
91
92 String kernelVersion = "[unknown]";
93 try {
94 if (SystemPropertyUtil.getBoolean("io.netty.transport.noNative", false)) {
95 cause = new UnsupportedOperationException(
96 "Native transport was explicit disabled with -Dio.netty.transport.noNative=true");
97 } else {
98 kernelVersion = Native.kernelVersion();
99 Native.checkKernelVersion(kernelVersion);
100 if (PlatformDependent.javaVersion() >= 9) {
101 RingBuffer ringBuffer = null;
102 try {
103 ringBuffer = Native.createRingBuffer(1, 0);
104 if ((ringBuffer.features() & Native.IORING_FEAT_SUBMIT_STABLE) == 0) {
105
106 throw new UnsupportedOperationException("IORING_FEAT_SUBMIT_STABLE not supported!");
107 }
108
109
110 numElementsIoVec = SystemPropertyUtil.getInt(
111 "io.netty.iouring.numElementsIoVec", 10 * Limits.IOV_MAX);
112 Native.IoUringProbe ioUringProbe = Native.ioUringProbe(ringBuffer.fd());
113 Native.checkAllIOSupported(ioUringProbe);
114 socketNonEmptySupported = Native.isCqeFSockNonEmptySupported(ioUringProbe);
115 unixDomainSocketInqSupported = Native.isUnixDomainSocketInqSupported();
116 spliceSupported = Native.isSpliceSupported(ioUringProbe);
117 recvsendBundleSupported = (ringBuffer.features() & Native.IORING_FEAT_RECVSEND_BUNDLE) != 0;
118 enterNoIoWaitSupported = (ringBuffer.features() & Native.IORING_FEAT_NO_IOWAIT) != 0;
119 sendZcSupported = Native.isSendZcSupported(ioUringProbe);
120 sendmsgZcSupported = Native.isSendmsgZcSupported(ioUringProbe);
121
122 acceptSupportNoWait = recvsendBundleSupported;
123
124 acceptMultishotSupported = Native.isAcceptMultishotSupported(ioUringProbe);
125 recvMultishotSupported = Native.isRecvMultishotSupported();
126 pollAddMultishotSupported = Native.isPollAddMultiShotSupported(ioUringProbe);
127 registerIowqWorkersSupported = Native.isRegisterIoWqWorkerSupported(ringBuffer.fd());
128 submitAllSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SUBMIT_ALL);
129 cqeMixedSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_CQE_MIXED);
130 setUpCqSizeSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_CQSIZE);
131 singleIssuerSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
132
133
134 deferTaskrunSupported = Native.ioUringSetupSupportsFlags(
135 Native.IORING_SETUP_SINGLE_ISSUER | Native.IORING_SETUP_DEFER_TASKRUN);
136 noSqarraySupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_NO_SQARRAY);
137 registerBufferRingSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(), 0);
138 registerBufferRingIncSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(),
139 Native.IOU_PBUF_RING_INC);
140 } finally {
141 if (ringBuffer != null) {
142 try {
143 ringBuffer.close();
144 } catch (Exception ignore) {
145
146 }
147 }
148 }
149 } else {
150 cause = new UnsupportedOperationException("Java 9+ is required");
151 }
152 }
153 } catch (Throwable t) {
154 cause = t;
155 }
156
157 UNAVAILABILITY_CAUSE = cause;
158 IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED = socketNonEmptySupported;
159 UNIX_DOMAIN_SOCKET_INQ_SUPPORTED = unixDomainSocketInqSupported;
160 IORING_SPLICE_SUPPORTED = spliceSupported;
161 IORING_SEND_ZC_SUPPORTED = sendZcSupported;
162 IORING_SENDMSG_ZC_SUPPORTED = sendmsgZcSupported;
163 IORING_ACCEPT_NO_WAIT_SUPPORTED = acceptSupportNoWait;
164 IORING_ACCEPT_MULTISHOT_SUPPORTED = acceptMultishotSupported;
165 IORING_RECV_MULTISHOT_SUPPORTED = recvMultishotSupported;
166 IORING_RECVSEND_BUNDLE_SUPPORTED = recvsendBundleSupported;
167 IORING_POLL_ADD_MULTISHOT_SUPPORTED = pollAddMultishotSupported;
168 IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED = registerIowqWorkersSupported;
169 IORING_SETUP_SUBMIT_ALL_SUPPORTED = submitAllSupported;
170 IORING_SETUP_CQE_MIXED_SUPPORTED = cqeMixedSupported;
171 IORING_SETUP_CQ_SIZE_SUPPORTED = setUpCqSizeSupported;
172 IORING_SETUP_SINGLE_ISSUER_SUPPORTED = singleIssuerSupported;
173 IORING_SETUP_DEFER_TASKRUN_SUPPORTED = deferTaskrunSupported;
174 IORING_SETUP_NO_SQARRAY_SUPPORTED = noSqarraySupported;
175 IORING_REGISTER_BUFFER_RING_SUPPORTED = registerBufferRingSupported;
176 IORING_REGISTER_BUFFER_RING_INC_SUPPORTED = registerBufferRingIncSupported;
177 IORING_ENTER_NO_IOWAIT_SUPPORTED = enterNoIoWaitSupported;
178
179 IORING_ACCEPT_MULTISHOT_ENABLED = IORING_ACCEPT_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
180 "io.netty.iouring.acceptMultiShotEnabled", true);
181 IORING_RECV_MULTISHOT_ENABLED = IORING_RECV_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
182 "io.netty.iouring.recvMultiShotEnabled", true);
183
184
185
186 IORING_RECVSEND_BUNDLE_ENABLED = IORING_RECVSEND_BUNDLE_SUPPORTED && SystemPropertyUtil.getBoolean(
187 "io.netty.iouring.recvsendBundleEnabled", false);
188 IORING_POLL_ADD_MULTISHOT_ENABLED = IORING_POLL_ADD_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
189 "io.netty.iouring.pollAddMultishotEnabled", true);
190 IORING_ENTER_NO_IOWAIT_ENABLED = IORING_ENTER_NO_IOWAIT_SUPPORTED && SystemPropertyUtil.getBoolean(
191 "io.netty.iouring.enterNoIoWaitEnabled", false);
192 NUM_ELEMENTS_IOVEC = numElementsIoVec;
193
194 DEFAULT_RING_SIZE = Math.max(16, SystemPropertyUtil.getInt("io.netty.iouring.ringSize", 128));
195 pendingOpsInitialCapacity = SystemPropertyUtil.getInt(
196 "io.netty.iouring.pendingOpsInitialCapacity", DEFAULT_RING_SIZE);
197 if (pendingOpsInitialCapacity <= 0) {
198 int configuredCapacity = pendingOpsInitialCapacity;
199 pendingOpsInitialCapacity = MathUtil.safeFindNextPositivePowerOfTwo(DEFAULT_RING_SIZE);
200 logger.warn("Invalid value {} for -Dio.netty.iouring.pendingOpsInitialCapacity; using {} instead.",
201 configuredCapacity, pendingOpsInitialCapacity);
202 } else if (Integer.bitCount(pendingOpsInitialCapacity) != 1) {
203 int configuredCapacity = pendingOpsInitialCapacity;
204 pendingOpsInitialCapacity = MathUtil.safeFindNextPositivePowerOfTwo(pendingOpsInitialCapacity);
205 logger.warn("Rounding -Dio.netty.iouring.pendingOpsInitialCapacity from {} up to {}.",
206 configuredCapacity, pendingOpsInitialCapacity);
207 }
208 DEFAULT_PENDING_OPS_INITIAL_CAPACITY = pendingOpsInitialCapacity;
209 if (IORING_SETUP_CQ_SIZE_SUPPORTED) {
210 DEFAULT_CQ_SIZE = Math.max(DEFAULT_RING_SIZE,
211 SystemPropertyUtil.getInt("io.netty.iouring.cqSize", 4096));
212 } else {
213 DEFAULT_CQ_SIZE = DISABLE_SETUP_CQ_SIZE;
214 }
215
216 if (cause != null) {
217 if (logger.isTraceEnabled()) {
218 logger.debug("IoUring support is not available using kernel {}", kernelVersion, cause);
219 } else if (logger.isDebugEnabled()) {
220 logger.debug("IoUring support is not available using kernel {}: {}", kernelVersion, cause.getMessage());
221 }
222 } else {
223 if (logger.isDebugEnabled()) {
224 logger.debug("IoUring support is available using kernel {}: {}", kernelVersion, supportedFeatures());
225 }
226 }
227 }
228
229 public static boolean isAvailable() {
230 return UNAVAILABILITY_CAUSE == null;
231 }
232
233
234
235
236
237
238
239 public static boolean isTcpFastOpenClientSideAvailable() {
240 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_CLIENT;
241 }
242
243
244
245
246
247
248
249 public static boolean isTcpFastOpenServerSideAvailable() {
250 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_SERVER;
251 }
252
253 static boolean isCqeFSockNonEmptySupported() {
254 return IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
255 }
256
257 static boolean isUnixDomainSocketInqSupported() {
258 return UNIX_DOMAIN_SOCKET_INQ_SUPPORTED;
259 }
260
261
262
263
264
265
266 public static boolean isSpliceSupported() {
267 return IORING_SPLICE_SUPPORTED;
268 }
269
270
271
272
273
274
275 static boolean isSendZcSupported() {
276 return IORING_SEND_ZC_SUPPORTED;
277 }
278
279
280
281
282
283
284 static boolean isSendmsgZcSupported() {
285 return IORING_SENDMSG_ZC_SUPPORTED;
286 }
287
288 static boolean isAcceptNoWaitSupported() {
289 return IORING_ACCEPT_NO_WAIT_SUPPORTED;
290 }
291
292 static boolean isAcceptMultishotSupported() {
293 return IORING_ACCEPT_MULTISHOT_SUPPORTED;
294 }
295
296 static boolean isRecvMultishotSupported() {
297 return IORING_RECV_MULTISHOT_SUPPORTED;
298 }
299
300 static boolean isRecvsendBundleSupported() {
301 return IORING_RECVSEND_BUNDLE_SUPPORTED;
302 }
303
304 static boolean isPollAddMultishotSupported() {
305 return IORING_POLL_ADD_MULTISHOT_SUPPORTED;
306 }
307
308 static boolean isRegisterIowqMaxWorkersSupported() {
309 return IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
310 }
311
312 static boolean isSetupCqeSizeSupported() {
313 return IORING_SETUP_CQ_SIZE_SUPPORTED;
314 }
315
316 static boolean isSetupSubmitAllSupported() {
317 return IORING_SETUP_SUBMIT_ALL_SUPPORTED;
318 }
319
320 static boolean isSetupCqeMixedSupported() {
321 return IORING_SETUP_CQE_MIXED_SUPPORTED;
322 }
323
324 static boolean isSetupSingleIssuerSupported() {
325 return IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
326 }
327
328 static boolean isSetupDeferTaskrunSupported() {
329 return IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
330 }
331
332 static boolean isIoringSetupNoSqarraySupported() {
333 return IORING_SETUP_NO_SQARRAY_SUPPORTED;
334 }
335
336
337
338
339
340 public static boolean isRegisterBufferRingSupported() {
341 return IORING_REGISTER_BUFFER_RING_SUPPORTED;
342 }
343
344
345
346
347
348
349 public static boolean isRegisterBufferRingIncSupported() {
350 return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
351 }
352
353 static boolean isIoringEnterNoIoWaitSupported() {
354 return IORING_ENTER_NO_IOWAIT_SUPPORTED;
355 }
356
357
358
359
360
361
362
363
364 public static boolean isIoringEnterNoIoWaitEnabled() {
365 return IORING_ENTER_NO_IOWAIT_ENABLED;
366 }
367
368
369
370
371
372
373 public static boolean isAcceptMultishotEnabled() {
374 return IORING_ACCEPT_MULTISHOT_ENABLED;
375 }
376
377
378
379
380
381
382 public static boolean isRecvMultishotEnabled() {
383 return IORING_RECV_MULTISHOT_ENABLED;
384 }
385
386
387
388
389
390
391 public static boolean isRecvsendBundleEnabled() {
392 return IORING_RECVSEND_BUNDLE_ENABLED;
393 }
394
395
396
397
398
399
400 public static boolean isPollAddMultishotEnabled() {
401 return IORING_POLL_ADD_MULTISHOT_ENABLED;
402 }
403
404 public static void ensureAvailability() {
405 if (UNAVAILABILITY_CAUSE != null) {
406 throw (Error) new UnsatisfiedLinkError(
407 "failed to load the required native library").initCause(UNAVAILABILITY_CAUSE);
408 }
409 }
410
411 static long memoryAddress(ByteBuf buffer) {
412 if (buffer.hasMemoryAddress()) {
413 return buffer.memoryAddress();
414 }
415
416
417
418 ByteBuffer byteBuffer = buffer.internalNioBuffer(0, buffer.capacity());
419 return Buffer.memoryAddress(byteBuffer) + byteBuffer.position();
420 }
421
422 public static Throwable unavailabilityCause() {
423 return UNAVAILABILITY_CAUSE;
424 }
425
426 private static String supportedFeatures() {
427 if (!isAvailable()) {
428 return "";
429 }
430 return "CQE_F_SOCK_NONEMPTY_SUPPORTED=" + IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED
431 + ", UNIX_DOMAIN_SOCKET_INQ_SUPPORTED=" + UNIX_DOMAIN_SOCKET_INQ_SUPPORTED
432 + ", SPLICE_SUPPORTED=" + IORING_SPLICE_SUPPORTED
433 + ", ACCEPT_NO_WAIT_SUPPORTED=" + IORING_ACCEPT_NO_WAIT_SUPPORTED
434 + ", ACCEPT_MULTISHOT_SUPPORTED=" + IORING_ACCEPT_MULTISHOT_SUPPORTED
435 + ", POLL_ADD_MULTISHOT_SUPPORTED=" + IORING_POLL_ADD_MULTISHOT_SUPPORTED
436 + ", RECV_MULTISHOT_SUPPORTED=" + IORING_RECV_MULTISHOT_SUPPORTED
437 + ", IORING_RECVSEND_BUNDLE_SUPPORTED=" + IORING_RECVSEND_BUNDLE_SUPPORTED
438 + ", REGISTER_IOWQ_MAX_WORKERS_SUPPORTED=" + IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED
439 + ", SETUP_SUBMIT_ALL_SUPPORTED=" + IORING_SETUP_SUBMIT_ALL_SUPPORTED
440 + ", SETUP_CQE_MIXED_SUPPORTED=" + IORING_SETUP_CQE_MIXED_SUPPORTED
441 + ", SETUP_CQ_SIZE_SUPPORTED=" + IORING_SETUP_CQ_SIZE_SUPPORTED
442 + ", SETUP_SINGLE_ISSUER_SUPPORTED=" + IORING_SETUP_SINGLE_ISSUER_SUPPORTED
443 + ", SETUP_DEFER_TASKRUN_SUPPORTED=" + IORING_SETUP_DEFER_TASKRUN_SUPPORTED
444 + ", SETUP_NO_SQARRAY_SUPPORTED=" + IORING_SETUP_NO_SQARRAY_SUPPORTED
445 + ", REGISTER_BUFFER_RING_SUPPORTED=" + IORING_REGISTER_BUFFER_RING_SUPPORTED
446 + ", REGISTER_BUFFER_RING_INC_SUPPORTED=" + IORING_REGISTER_BUFFER_RING_INC_SUPPORTED
447 + ", SEND_ZC_SUPPORTED=" + IORING_SEND_ZC_SUPPORTED
448 + ", SENDMSG_ZC_SUPPORTED=" + IORING_SENDMSG_ZC_SUPPORTED
449 + ", ENTER_NO_IOWAIT_SUPPORTED=" + IORING_ENTER_NO_IOWAIT_SUPPORTED;
450 }
451
452
453
454
455
456 public static String featureString() {
457 if (!isAvailable()) {
458 Throwable t = unavailabilityCause();
459 return "IoUring unavailable: " + (t == null ? "unknown cause" : t.toString());
460 }
461 return "IoUring features: " + supportedFeatures();
462 }
463
464 @Override
465 public String toString() {
466 return featureString();
467 }
468
469 private IoUring() {
470 }
471 }