1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.buffer.ByteBuf;
19 import io.netty.channel.ChannelOption;
20 import io.netty.channel.unix.Buffer;
21 import io.netty.channel.unix.Limits;
22 import io.netty.util.internal.MathUtil;
23 import io.netty.util.internal.PlatformDependent;
24 import io.netty.util.internal.SystemPropertyUtil;
25 import io.netty.util.internal.logging.InternalLogger;
26 import io.netty.util.internal.logging.InternalLoggerFactory;
27
28 import java.nio.ByteBuffer;
29
30 public final class IoUring {
31
32 private static final Throwable UNAVAILABILITY_CAUSE;
33 private static final boolean IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
34 private static final boolean IORING_SPLICE_SUPPORTED;
35 private static final boolean IORING_SEND_ZC_SUPPORTED;
36 private static final boolean IORING_SENDMSG_ZC_SUPPORTED;
37 private static final boolean IORING_ACCEPT_NO_WAIT_SUPPORTED;
38 private static final boolean IORING_ACCEPT_MULTISHOT_SUPPORTED;
39 private static final boolean IORING_RECV_MULTISHOT_SUPPORTED;
40 private static final boolean IORING_RECVSEND_BUNDLE_SUPPORTED;
41 private static final boolean IORING_POLL_ADD_MULTISHOT_SUPPORTED;
42 private static final boolean IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
43 private static final boolean IORING_SETUP_SUBMIT_ALL_SUPPORTED;
44 private static final boolean IORING_SETUP_CQE_MIXED_SUPPORTED;
45 private static final boolean IORING_SETUP_CQ_SIZE_SUPPORTED;
46 private static final boolean IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
47 private static final boolean IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
48 private static final boolean IORING_SETUP_NO_SQARRAY_SUPPORTED;
49 private static final boolean IORING_REGISTER_BUFFER_RING_SUPPORTED;
50 private static final boolean IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
51 private static final boolean IORING_ENTER_NO_IOWAIT_SUPPORTED;
52 private static final boolean IORING_ACCEPT_MULTISHOT_ENABLED;
53 private static final boolean IORING_RECV_MULTISHOT_ENABLED;
54 private static final boolean IORING_RECVSEND_BUNDLE_ENABLED;
55 private static final boolean IORING_POLL_ADD_MULTISHOT_ENABLED;
56 private static final boolean IORING_ENTER_NO_IOWAIT_ENABLED;
57 static final int NUM_ELEMENTS_IOVEC;
58 static final int DEFAULT_RING_SIZE;
59 static final int DEFAULT_CQ_SIZE;
60 static final int DEFAULT_PENDING_OPS_INITIAL_CAPACITY;
61 static final int DISABLE_SETUP_CQ_SIZE = -1;
62
63 private static final InternalLogger logger;
64
65 static {
66 logger = InternalLoggerFactory.getInstance(IoUring.class);
67 Throwable cause = null;
68 boolean socketNonEmptySupported = false;
69 boolean spliceSupported = false;
70 boolean sendZcSupported = false;
71 boolean sendmsgZcSupported = false;
72 boolean acceptSupportNoWait = false;
73 boolean acceptMultishotSupported = false;
74 boolean recvsendBundleSupported = false;
75 boolean recvMultishotSupported = false;
76 boolean pollAddMultishotSupported = false;
77 boolean registerIowqWorkersSupported = false;
78 boolean submitAllSupported = false;
79 boolean cqeMixedSupported = false;
80 boolean setUpCqSizeSupported = false;
81 boolean singleIssuerSupported = false;
82 boolean deferTaskrunSupported = false;
83 boolean noSqarraySupported = false;
84 boolean registerBufferRingSupported = false;
85 boolean registerBufferRingIncSupported = false;
86 boolean enterNoIoWaitSupported = false;
87 int numElementsIoVec = 10;
88 int pendingOpsInitialCapacity;
89
90 String kernelVersion = "[unknown]";
91 try {
92 if (SystemPropertyUtil.getBoolean("io.netty.transport.noNative", false)) {
93 cause = new UnsupportedOperationException(
94 "Native transport was explicit disabled with -Dio.netty.transport.noNative=true");
95 } else {
96 kernelVersion = Native.kernelVersion();
97 Native.checkKernelVersion(kernelVersion);
98 if (PlatformDependent.javaVersion() >= 9) {
99 RingBuffer ringBuffer = null;
100 try {
101 ringBuffer = Native.createRingBuffer(1, 0);
102 if ((ringBuffer.features() & Native.IORING_FEAT_SUBMIT_STABLE) == 0) {
103
104 throw new UnsupportedOperationException("IORING_FEAT_SUBMIT_STABLE not supported!");
105 }
106
107
108 numElementsIoVec = SystemPropertyUtil.getInt(
109 "io.netty.iouring.numElementsIoVec", 10 * Limits.IOV_MAX);
110 Native.IoUringProbe ioUringProbe = Native.ioUringProbe(ringBuffer.fd());
111 Native.checkAllIOSupported(ioUringProbe);
112 socketNonEmptySupported = Native.isCqeFSockNonEmptySupported(ioUringProbe);
113 spliceSupported = Native.isSpliceSupported(ioUringProbe);
114 recvsendBundleSupported = (ringBuffer.features() & Native.IORING_FEAT_RECVSEND_BUNDLE) != 0;
115 enterNoIoWaitSupported = (ringBuffer.features() & Native.IORING_FEAT_NO_IOWAIT) != 0;
116 sendZcSupported = Native.isSendZcSupported(ioUringProbe);
117 sendmsgZcSupported = Native.isSendmsgZcSupported(ioUringProbe);
118
119 acceptSupportNoWait = recvsendBundleSupported;
120
121 acceptMultishotSupported = Native.isAcceptMultishotSupported(ioUringProbe);
122 recvMultishotSupported = Native.isRecvMultishotSupported();
123 pollAddMultishotSupported = Native.isPollAddMultiShotSupported(ioUringProbe);
124 registerIowqWorkersSupported = Native.isRegisterIoWqWorkerSupported(ringBuffer.fd());
125 submitAllSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SUBMIT_ALL);
126 cqeMixedSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_CQE_MIXED);
127 setUpCqSizeSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_CQSIZE);
128 singleIssuerSupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_SINGLE_ISSUER);
129
130
131 deferTaskrunSupported = Native.ioUringSetupSupportsFlags(
132 Native.IORING_SETUP_SINGLE_ISSUER | Native.IORING_SETUP_DEFER_TASKRUN);
133 noSqarraySupported = Native.ioUringSetupSupportsFlags(Native.IORING_SETUP_NO_SQARRAY);
134 registerBufferRingSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(), 0);
135 registerBufferRingIncSupported = Native.isRegisterBufferRingSupported(ringBuffer.fd(),
136 Native.IOU_PBUF_RING_INC);
137 } finally {
138 if (ringBuffer != null) {
139 try {
140 ringBuffer.close();
141 } catch (Exception ignore) {
142
143 }
144 }
145 }
146 } else {
147 cause = new UnsupportedOperationException("Java 9+ is required");
148 }
149 }
150 } catch (Throwable t) {
151 cause = t;
152 }
153
154 UNAVAILABILITY_CAUSE = cause;
155 IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED = socketNonEmptySupported;
156 IORING_SPLICE_SUPPORTED = spliceSupported;
157 IORING_SEND_ZC_SUPPORTED = sendZcSupported;
158 IORING_SENDMSG_ZC_SUPPORTED = sendmsgZcSupported;
159 IORING_ACCEPT_NO_WAIT_SUPPORTED = acceptSupportNoWait;
160 IORING_ACCEPT_MULTISHOT_SUPPORTED = acceptMultishotSupported;
161 IORING_RECV_MULTISHOT_SUPPORTED = recvMultishotSupported;
162 IORING_RECVSEND_BUNDLE_SUPPORTED = recvsendBundleSupported;
163 IORING_POLL_ADD_MULTISHOT_SUPPORTED = pollAddMultishotSupported;
164 IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED = registerIowqWorkersSupported;
165 IORING_SETUP_SUBMIT_ALL_SUPPORTED = submitAllSupported;
166 IORING_SETUP_CQE_MIXED_SUPPORTED = cqeMixedSupported;
167 IORING_SETUP_CQ_SIZE_SUPPORTED = setUpCqSizeSupported;
168 IORING_SETUP_SINGLE_ISSUER_SUPPORTED = singleIssuerSupported;
169 IORING_SETUP_DEFER_TASKRUN_SUPPORTED = deferTaskrunSupported;
170 IORING_SETUP_NO_SQARRAY_SUPPORTED = noSqarraySupported;
171 IORING_REGISTER_BUFFER_RING_SUPPORTED = registerBufferRingSupported;
172 IORING_REGISTER_BUFFER_RING_INC_SUPPORTED = registerBufferRingIncSupported;
173 IORING_ENTER_NO_IOWAIT_SUPPORTED = enterNoIoWaitSupported;
174
175 IORING_ACCEPT_MULTISHOT_ENABLED = IORING_ACCEPT_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
176 "io.netty.iouring.acceptMultiShotEnabled", true);
177 IORING_RECV_MULTISHOT_ENABLED = IORING_RECV_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
178 "io.netty.iouring.recvMultiShotEnabled", true);
179
180
181
182 IORING_RECVSEND_BUNDLE_ENABLED = IORING_RECVSEND_BUNDLE_SUPPORTED && SystemPropertyUtil.getBoolean(
183 "io.netty.iouring.recvsendBundleEnabled", false);
184 IORING_POLL_ADD_MULTISHOT_ENABLED = IORING_POLL_ADD_MULTISHOT_SUPPORTED && SystemPropertyUtil.getBoolean(
185 "io.netty.iouring.pollAddMultishotEnabled", true);
186 IORING_ENTER_NO_IOWAIT_ENABLED = IORING_ENTER_NO_IOWAIT_SUPPORTED && SystemPropertyUtil.getBoolean(
187 "io.netty.iouring.enterNoIoWaitEnabled", false);
188 NUM_ELEMENTS_IOVEC = numElementsIoVec;
189
190 DEFAULT_RING_SIZE = Math.max(16, SystemPropertyUtil.getInt("io.netty.iouring.ringSize", 128));
191 pendingOpsInitialCapacity = SystemPropertyUtil.getInt(
192 "io.netty.iouring.pendingOpsInitialCapacity", DEFAULT_RING_SIZE);
193 if (pendingOpsInitialCapacity <= 0) {
194 int configuredCapacity = pendingOpsInitialCapacity;
195 pendingOpsInitialCapacity = MathUtil.safeFindNextPositivePowerOfTwo(DEFAULT_RING_SIZE);
196 logger.warn("Invalid value {} for -Dio.netty.iouring.pendingOpsInitialCapacity; using {} instead.",
197 configuredCapacity, pendingOpsInitialCapacity);
198 } else if (Integer.bitCount(pendingOpsInitialCapacity) != 1) {
199 int configuredCapacity = pendingOpsInitialCapacity;
200 pendingOpsInitialCapacity = MathUtil.safeFindNextPositivePowerOfTwo(pendingOpsInitialCapacity);
201 logger.warn("Rounding -Dio.netty.iouring.pendingOpsInitialCapacity from {} up to {}.",
202 configuredCapacity, pendingOpsInitialCapacity);
203 }
204 DEFAULT_PENDING_OPS_INITIAL_CAPACITY = pendingOpsInitialCapacity;
205 if (IORING_SETUP_CQ_SIZE_SUPPORTED) {
206 DEFAULT_CQ_SIZE = Math.max(DEFAULT_RING_SIZE,
207 SystemPropertyUtil.getInt("io.netty.iouring.cqSize", 4096));
208 } else {
209 DEFAULT_CQ_SIZE = DISABLE_SETUP_CQ_SIZE;
210 }
211
212 if (cause != null) {
213 if (logger.isTraceEnabled()) {
214 logger.debug("IoUring support is not available using kernel {}", kernelVersion, cause);
215 } else if (logger.isDebugEnabled()) {
216 logger.debug("IoUring support is not available using kernel {}: {}", kernelVersion, cause.getMessage());
217 }
218 } else {
219 if (logger.isDebugEnabled()) {
220 logger.debug("IoUring support is available using kernel {}: {}", kernelVersion, supportedFeatures());
221 }
222 }
223 }
224
225 public static boolean isAvailable() {
226 return UNAVAILABILITY_CAUSE == null;
227 }
228
229
230
231
232
233
234
235 public static boolean isTcpFastOpenClientSideAvailable() {
236 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_CLIENT;
237 }
238
239
240
241
242
243
244
245 public static boolean isTcpFastOpenServerSideAvailable() {
246 return isAvailable() && Native.IS_SUPPORTING_TCP_FASTOPEN_SERVER;
247 }
248
249 static boolean isCqeFSockNonEmptySupported() {
250 return IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED;
251 }
252
253
254
255
256
257
258 public static boolean isSpliceSupported() {
259 return IORING_SPLICE_SUPPORTED;
260 }
261
262
263
264
265
266
267 static boolean isSendZcSupported() {
268 return IORING_SEND_ZC_SUPPORTED;
269 }
270
271
272
273
274
275
276 static boolean isSendmsgZcSupported() {
277 return IORING_SENDMSG_ZC_SUPPORTED;
278 }
279
280 static boolean isAcceptNoWaitSupported() {
281 return IORING_ACCEPT_NO_WAIT_SUPPORTED;
282 }
283
284 static boolean isAcceptMultishotSupported() {
285 return IORING_ACCEPT_MULTISHOT_SUPPORTED;
286 }
287
288 static boolean isRecvMultishotSupported() {
289 return IORING_RECV_MULTISHOT_SUPPORTED;
290 }
291
292 static boolean isRecvsendBundleSupported() {
293 return IORING_RECVSEND_BUNDLE_SUPPORTED;
294 }
295
296 static boolean isPollAddMultishotSupported() {
297 return IORING_POLL_ADD_MULTISHOT_SUPPORTED;
298 }
299
300 static boolean isRegisterIowqMaxWorkersSupported() {
301 return IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED;
302 }
303
304 static boolean isSetupCqeSizeSupported() {
305 return IORING_SETUP_CQ_SIZE_SUPPORTED;
306 }
307
308 static boolean isSetupSubmitAllSupported() {
309 return IORING_SETUP_SUBMIT_ALL_SUPPORTED;
310 }
311
312 static boolean isSetupCqeMixedSupported() {
313 return IORING_SETUP_CQE_MIXED_SUPPORTED;
314 }
315
316 static boolean isSetupSingleIssuerSupported() {
317 return IORING_SETUP_SINGLE_ISSUER_SUPPORTED;
318 }
319
320 static boolean isSetupDeferTaskrunSupported() {
321 return IORING_SETUP_DEFER_TASKRUN_SUPPORTED;
322 }
323
324 static boolean isIoringSetupNoSqarraySupported() {
325 return IORING_SETUP_NO_SQARRAY_SUPPORTED;
326 }
327
328
329
330
331
332 public static boolean isRegisterBufferRingSupported() {
333 return IORING_REGISTER_BUFFER_RING_SUPPORTED;
334 }
335
336
337
338
339
340
341 public static boolean isRegisterBufferRingIncSupported() {
342 return IORING_REGISTER_BUFFER_RING_INC_SUPPORTED;
343 }
344
345 static boolean isIoringEnterNoIoWaitSupported() {
346 return IORING_ENTER_NO_IOWAIT_SUPPORTED;
347 }
348
349
350
351
352
353
354
355
356 public static boolean isIoringEnterNoIoWaitEnabled() {
357 return IORING_ENTER_NO_IOWAIT_ENABLED;
358 }
359
360
361
362
363
364
365 public static boolean isAcceptMultishotEnabled() {
366 return IORING_ACCEPT_MULTISHOT_ENABLED;
367 }
368
369
370
371
372
373
374 public static boolean isRecvMultishotEnabled() {
375 return IORING_RECV_MULTISHOT_ENABLED;
376 }
377
378
379
380
381
382
383 public static boolean isRecvsendBundleEnabled() {
384 return IORING_RECVSEND_BUNDLE_ENABLED;
385 }
386
387
388
389
390
391
392 public static boolean isPollAddMultishotEnabled() {
393 return IORING_POLL_ADD_MULTISHOT_ENABLED;
394 }
395
396 public static void ensureAvailability() {
397 if (UNAVAILABILITY_CAUSE != null) {
398 throw (Error) new UnsatisfiedLinkError(
399 "failed to load the required native library").initCause(UNAVAILABILITY_CAUSE);
400 }
401 }
402
403 static long memoryAddress(ByteBuf buffer) {
404 if (buffer.hasMemoryAddress()) {
405 return buffer.memoryAddress();
406 }
407
408
409
410 ByteBuffer byteBuffer = buffer.internalNioBuffer(0, buffer.capacity());
411 return Buffer.memoryAddress(byteBuffer) + byteBuffer.position();
412 }
413
414 public static Throwable unavailabilityCause() {
415 return UNAVAILABILITY_CAUSE;
416 }
417
418 private static String supportedFeatures() {
419 if (!isAvailable()) {
420 return "";
421 }
422 return "CQE_F_SOCK_NONEMPTY_SUPPORTED=" + IORING_CQE_F_SOCK_NONEMPTY_SUPPORTED
423 + ", SPLICE_SUPPORTED=" + IORING_SPLICE_SUPPORTED
424 + ", ACCEPT_NO_WAIT_SUPPORTED=" + IORING_ACCEPT_NO_WAIT_SUPPORTED
425 + ", ACCEPT_MULTISHOT_SUPPORTED=" + IORING_ACCEPT_MULTISHOT_SUPPORTED
426 + ", POLL_ADD_MULTISHOT_SUPPORTED=" + IORING_POLL_ADD_MULTISHOT_SUPPORTED
427 + ", RECV_MULTISHOT_SUPPORTED=" + IORING_RECV_MULTISHOT_SUPPORTED
428 + ", IORING_RECVSEND_BUNDLE_SUPPORTED=" + IORING_RECVSEND_BUNDLE_SUPPORTED
429 + ", REGISTER_IOWQ_MAX_WORKERS_SUPPORTED=" + IORING_REGISTER_IOWQ_MAX_WORKERS_SUPPORTED
430 + ", SETUP_SUBMIT_ALL_SUPPORTED=" + IORING_SETUP_SUBMIT_ALL_SUPPORTED
431 + ", SETUP_CQE_MIXED_SUPPORTED=" + IORING_SETUP_CQE_MIXED_SUPPORTED
432 + ", SETUP_CQ_SIZE_SUPPORTED=" + IORING_SETUP_CQ_SIZE_SUPPORTED
433 + ", SETUP_SINGLE_ISSUER_SUPPORTED=" + IORING_SETUP_SINGLE_ISSUER_SUPPORTED
434 + ", SETUP_DEFER_TASKRUN_SUPPORTED=" + IORING_SETUP_DEFER_TASKRUN_SUPPORTED
435 + ", SETUP_NO_SQARRAY_SUPPORTED=" + IORING_SETUP_NO_SQARRAY_SUPPORTED
436 + ", REGISTER_BUFFER_RING_SUPPORTED=" + IORING_REGISTER_BUFFER_RING_SUPPORTED
437 + ", REGISTER_BUFFER_RING_INC_SUPPORTED=" + IORING_REGISTER_BUFFER_RING_INC_SUPPORTED
438 + ", SEND_ZC_SUPPORTED=" + IORING_SEND_ZC_SUPPORTED
439 + ", SENDMSG_ZC_SUPPORTED=" + IORING_SENDMSG_ZC_SUPPORTED
440 + ", ENTER_NO_IOWAIT_SUPPORTED=" + IORING_ENTER_NO_IOWAIT_SUPPORTED;
441 }
442
443
444
445
446
447 public static String featureString() {
448 if (!isAvailable()) {
449 Throwable t = unavailabilityCause();
450 return "IoUring unavailable: " + (t == null ? "unknown cause" : t.toString());
451 }
452 return "IoUring features: " + supportedFeatures();
453 }
454
455 @Override
456 public String toString() {
457 return featureString();
458 }
459
460 private IoUring() {
461 }
462 }