1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.buffer.ByteBuf;
19 import io.netty.buffer.ByteBufAllocator;
20 import io.netty.channel.Channel;
21 import io.netty.channel.ChannelException;
22 import io.netty.channel.ChannelOutboundBuffer;
23 import io.netty.channel.socket.ServerSocketChannel;
24 import io.netty.channel.socket.SocketChannel;
25 import io.netty.channel.socket.SocketChannelConfig;
26 import io.netty.channel.unix.IovArray;
27 import io.netty.util.ReferenceCounted;
28 import io.netty.util.internal.SystemPropertyUtil;
29
30 import java.io.IOException;
31 import java.net.InetSocketAddress;
32 import java.net.SocketAddress;
33 import java.nio.ByteBuffer;
34 import java.util.ArrayList;
35 import java.util.Collections;
36 import java.util.List;
37
38 import static io.netty.channel.unix.Errors.ioResult;
39
40 public final class IoUringSocketChannel extends AbstractIoUringStreamChannel implements SocketChannel {
41 private final IoUringSocketChannelConfig config;
42
43 public IoUringSocketChannel() {
44 super(null, LinuxSocket.newSocketStream(), false);
45 this.config = new IoUringSocketChannelConfig(this);
46 }
47
48 IoUringSocketChannel(Channel parent, LinuxSocket fd) {
49 super(parent, fd, true);
50 this.config = new IoUringSocketChannelConfig(this);
51 }
52
53 IoUringSocketChannel(Channel parent, LinuxSocket fd, SocketAddress remote) {
54 super(parent, fd, remote);
55 this.config = new IoUringSocketChannelConfig(this);
56 }
57
58
59
60
61
62 public IoUringTcpInfo tcpInfo() {
63 return tcpInfo(new IoUringTcpInfo());
64 }
65
66
67
68
69
70 public IoUringTcpInfo tcpInfo(IoUringTcpInfo info) {
71 try {
72 socket.getTcpInfo(info);
73 return info;
74 } catch (IOException e) {
75 throw new ChannelException(e);
76 }
77 }
78
79 @Override
80 public ServerSocketChannel parent() {
81 return (ServerSocketChannel) super.parent();
82 }
83
84 @Override
85 public SocketChannelConfig config() {
86 return config;
87 }
88
89 @Override
90 public InetSocketAddress remoteAddress() {
91 return (InetSocketAddress) super.remoteAddress();
92 }
93
94 @Override
95 public InetSocketAddress localAddress() {
96 return (InetSocketAddress) super.localAddress();
97 }
98
99 @Override
100 protected AbstractUringUnsafe newUnsafe() {
101 return new IoUringSocketUnsafe();
102 }
103
104 private final class IoUringSocketUnsafe extends IoUringStreamUnsafe {
105
106
107
108 private PendingZeroCopyWrites pendingZeroCopyWrites;
109
110 private ZeroCopyBatch zeroCopyBatch;
111
112 @Override
113 protected int scheduleWriteSingle(Object msg) {
114 assert writeId == 0;
115
116 if (IoUring.isSendZcSupported() && msg instanceof ByteBuf) {
117 ByteBuf buf = (ByteBuf) msg;
118 int length = buf.readableBytes();
119 if (((IoUringSocketChannelConfig) config()).shouldWriteZeroCopy(length)) {
120 long address = IoUring.memoryAddress(buf) + buf.readerIndex();
121 long opsId = nextZeroCopyUserData();
122 IoUringIoOps ops = IoUringIoOps.newSendZc(fd().intValue(), address, length, 0, opsId, 0);
123 byte opCode = ops.opcode();
124 writeId = submitWrite(ops);
125 writeOpCode = opCode;
126 if (writeId == 0) {
127 pendingZeroCopyWrites.recycle(opsId);
128 return 0;
129 }
130 return 1;
131 }
132
133 }
134 return super.scheduleWriteSingle(msg);
135 }
136
137 @Override
138 protected int scheduleWriteMultiple(ChannelOutboundBuffer in) {
139 assert writeId == 0;
140
141 IoUringSocketChannelConfig ioUringSocketChannelConfig = (IoUringSocketChannelConfig) config();
142 int threshold = ioUringSocketChannelConfig.getWriteZeroCopyThreshold();
143 if (!IoUring.isSendmsgZcSupported() || threshold == IoUringSocketChannelConfig.DISABLE_WRITE_ZERO_COPY) {
144 return super.scheduleWriteMultiple(in);
145 }
146 IoUringIoHandler handler = registration().attachment();
147
148 IovArray iovArray = handler.iovArray();
149 int offset = iovArray.count();
150 if (zeroCopyBatch == null) {
151 zeroCopyBatch = new ZeroCopyBatch();
152 }
153 try {
154
155 in.forEachFlushedMessage(zeroCopyBatch.scan(threshold, iovArray));
156 } catch (Exception e) {
157
158 return scheduleWriteSingle(in.current());
159 }
160
161 if (zeroCopyBatch.scannedLarge() && zeroCopyBatch.allocateMerged(alloc())) {
162 try {
163 zeroCopyBatch.collect(in, iovArray, offset);
164 } catch (Exception e) {
165
166 zeroCopyBatch.release();
167 return scheduleWriteSingle(in.current());
168 }
169 long iovArrayAddress = iovArray.memoryAddress(offset);
170 int iovArrayLength = iovArray.count() - offset;
171
172 MsgHdrMemory hdr = writeMsgHdr();
173 hdr.set(iovArrayAddress, iovArrayLength);
174 long opsId = nextZeroCopyUserData();
175 IoUringIoOps ops = IoUringIoOps.newSendmsgZc(
176 fd().intValue(), (byte) 0, 0, hdr.address(), opsId);
177 byte opCode = ops.opcode();
178 writeId = submitWrite(ops);
179 writeOpCode = opCode;
180 if (writeId == 0) {
181 pendingZeroCopyWrites.recycle(opsId);
182 zeroCopyBatch.release();
183 return 0;
184 }
185 return 1;
186 }
187
188 IoUringIoOps ops = IoUringIoOps.newWritev(fd().intValue(), (byte) 0, 0,
189 iovArray.memoryAddress(offset), iovArray.count() - offset, nextOpsId());
190 byte opCode = ops.opcode();
191 writeId = submitWrite(ops);
192 writeOpCode = opCode;
193 return writeId == 0 ? 0 : 1;
194 }
195
196 private long nextZeroCopyUserData() {
197 PendingZeroCopyWrites pendingWrites = pendingZeroCopyWrites;
198 if (pendingWrites == null) {
199 pendingWrites = new PendingZeroCopyWrites();
200 pendingZeroCopyWrites = pendingWrites;
201 }
202 return pendingWrites.nextUserData();
203 }
204
205 @Override
206 List<ReferenceCounted> retainWriteBuffers(ChannelOutboundBuffer buffer) {
207 if (currentWrite.opcode() == Native.IORING_OP_SENDMSG_ZC) {
208 return zeroCopyBatch.retain();
209 }
210 return super.retainWriteBuffers(buffer);
211 }
212
213 @Override
214 boolean writeComplete0(byte op, int res, int flags, long data, int outstanding) {
215 if (op == Native.IORING_OP_SEND_ZC || op == Native.IORING_OP_SENDMSG_ZC) {
216 return handleWriteCompleteZeroCopy(op, res, flags, data);
217 }
218 return super.writeComplete0(op, res, flags, data, outstanding);
219 }
220
221 private void takeSendZCBuffers(long data) {
222 PendingZeroCopyWrites.PendingWrite write = pendingZeroCopyWrites.register(data);
223 if (retainedWriteBuffers == null) {
224 if (currentWrite.opcode() == Native.IORING_OP_SENDMSG_ZC) {
225 zeroCopyBatch.transferTo(write);
226 } else {
227 write.add(((ByteBuf) outboundBuffer().current()).retain());
228 }
229 } else {
230
231 for (int i = 0; i < retainedWriteBuffers.size(); i++) {
232 write.add(retainedWriteBuffers.get(i));
233 }
234 retainedWriteBuffers = Collections.emptyList();
235 }
236 }
237
238 private boolean handleWriteCompleteZeroCopy(byte op, int res, int flags, long data) {
239 if ((flags & Native.IORING_CQE_F_NOTIF) != 0) {
240 pendingZeroCopyWrites.release(data);
241 return true;
242 }
243 writeId = 0;
244 writeOpCode = 0;
245 if ((flags & Native.IORING_CQE_F_MORE) != 0) {
246
247 takeSendZCBuffers(data);
248 } else {
249 pendingZeroCopyWrites.recycle(data);
250 if (op == Native.IORING_OP_SENDMSG_ZC) {
251 zeroCopyBatch.release();
252 }
253 }
254 ChannelOutboundBuffer channelOutboundBuffer = outboundBuffer();
255 if (channelOutboundBuffer == null) {
256 return true;
257 }
258 if (res >= 0) {
259 channelOutboundBuffer.removeBytes(res);
260 return true;
261 }
262 if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
263 return true;
264 }
265 try {
266 return ioResult(op == Native.IORING_OP_SEND_ZC ? "io_uring sendzc" : "io_uring sendmsg_zc", res) != 0;
267 } catch (Throwable cause) {
268 handleWriteError(cause);
269 return true;
270 }
271 }
272 }
273
274
275
276
277
278
279
280
281
282
283
284
285 private static final class ZeroCopyBatch implements ChannelOutboundBuffer.MessageProcessor {
286
287 private static final int MERGED_MAX_SIZE = Math.min(4 * 1024 * 1024,
288 Math.max(1, SystemPropertyUtil.getInt("io.netty.iouring.zeroCopyMergedMaxSize", 64 * 1024)));
289
290
291 private final List<ByteBuf> largeBuffers = new ArrayList<>(4);
292
293 private ByteBuf merged;
294
295 private int runStart;
296 private IovArray iovArray;
297 private int threshold;
298 private long smallBytes;
299 private boolean hasLarge;
300 private boolean scanning;
301
302
303
304
305
306
307 ChannelOutboundBuffer.MessageProcessor scan(int threshold, IovArray iovArray) {
308 this.iovArray = iovArray;
309 this.threshold = threshold;
310 smallBytes = 0;
311 hasLarge = false;
312 scanning = true;
313 return this;
314 }
315
316
317
318
319
320 boolean scannedLarge() {
321 boolean large = hasLarge;
322 hasLarge = false;
323 scanning = false;
324 return large;
325 }
326
327
328
329
330
331
332 boolean allocateMerged(ByteBufAllocator allocator) {
333 if (smallBytes != 0) {
334
335 int capacity = (int) Math.min(smallBytes, MERGED_MAX_SIZE);
336 try {
337 merged = allocator.directBuffer(capacity, capacity);
338 } catch (Exception | OutOfMemoryError e) {
339 return false;
340 }
341 runStart = 0;
342 }
343 return true;
344 }
345
346
347
348
349 void collect(ChannelOutboundBuffer in, IovArray iovArray, int offset) throws Exception {
350 iovArray.truncate(offset);
351 in.forEachFlushedMessage(this);
352 addMergedSmallBuffers();
353 }
354
355 @Override
356 public boolean processMessage(Object msg) throws Exception {
357 if (scanning) {
358 int count = iovArray.count();
359 boolean added = iovArray.processMessage(msg);
360
361 for (int i = count; i < iovArray.count(); i++) {
362 long length = iovArray.length(i);
363 if (length < threshold) {
364 smallBytes += length;
365 } else if (smallBytes <= MERGED_MAX_SIZE) {
366 hasLarge = true;
367 }
368 }
369 return added;
370 }
371 if (!(msg instanceof ByteBuf)) {
372 return false;
373 }
374 ByteBuf buf = (ByteBuf) msg;
375 int index = buf.readerIndex();
376 int readableBytes = buf.readableBytes();
377
378 if (readableBytes < threshold || buf.nioBufferCount() == 1) {
379 return add(buf, index, readableBytes);
380 }
381 for (ByteBuffer nioBuffer : buf.nioBuffers(index, readableBytes)) {
382 int length = nioBuffer.remaining();
383 if (!add(buf, index, length)) {
384 return false;
385 }
386 index += length;
387 }
388 return true;
389 }
390
391 private boolean add(ByteBuf buf, int index, int length) {
392 if (length < threshold) {
393 if (length == 0) {
394 return true;
395 }
396 if (merged == null) {
397 return false;
398 }
399
400 int copied = Math.min(length, merged.writableBytes());
401 merged.writeBytes(buf, index, copied);
402 return copied == length;
403 }
404 if (!addMergedSmallBuffers()) {
405 return false;
406 }
407 int count = iovArray.count();
408 boolean added = iovArray.add(buf, index, length);
409
410 if (iovArray.count() != count
411 && (largeBuffers.isEmpty() || largeBuffers.get(largeBuffers.size() - 1) != buf)) {
412
413
414 largeBuffers.add(buf);
415 }
416 return added;
417 }
418
419 private boolean addMergedSmallBuffers() {
420 if (merged == null || merged.writerIndex() == runStart) {
421 return true;
422 }
423 if (!iovArray.add(merged, runStart, merged.writerIndex() - runStart)) {
424 return false;
425 }
426 runStart = merged.writerIndex();
427 return true;
428 }
429
430 void transferTo(PendingZeroCopyWrites.PendingWrite write) {
431 for (int i = 0; i < largeBuffers.size(); i++) {
432 write.add(largeBuffers.get(i).retain());
433 }
434 if (merged != null) {
435 write.add(merged);
436 }
437 largeBuffers.clear();
438 merged = null;
439 }
440
441
442
443
444
445 List<ReferenceCounted> retain() {
446 List<ReferenceCounted> buffers = new ArrayList<>(largeBuffers.size() + 1);
447 for (int i = 0; i < largeBuffers.size(); i++) {
448 buffers.add(largeBuffers.get(i).retain());
449 }
450 if (merged != null) {
451 buffers.add(merged);
452 }
453 largeBuffers.clear();
454 merged = null;
455 return buffers;
456 }
457
458
459
460
461 void release() {
462 if (merged != null) {
463 merged.release();
464 merged = null;
465 }
466 largeBuffers.clear();
467 }
468 }
469 }