View Javadoc
1   /*
2    * Copyright 2024 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.channel.uring;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.channel.Channel;
21  import io.netty.channel.ChannelException;
22  import io.netty.channel.ChannelOutboundBuffer;
23  import io.netty.channel.socket.ServerSocketChannel;
24  import io.netty.channel.socket.SocketChannel;
25  import io.netty.channel.socket.SocketChannelConfig;
26  import io.netty.channel.unix.IovArray;
27  import io.netty.util.ReferenceCounted;
28  import io.netty.util.internal.SystemPropertyUtil;
29  
30  import java.io.IOException;
31  import java.net.InetSocketAddress;
32  import java.net.SocketAddress;
33  import java.nio.ByteBuffer;
34  import java.util.ArrayList;
35  import java.util.Collections;
36  import java.util.List;
37  
38  import static io.netty.channel.unix.Errors.ioResult;
39  
40  public final class IoUringSocketChannel extends AbstractIoUringStreamChannel implements SocketChannel {
41      private final IoUringSocketChannelConfig config;
42  
43      public IoUringSocketChannel() {
44         super(null, LinuxSocket.newSocketStream(), false);
45         this.config = new IoUringSocketChannelConfig(this);
46      }
47  
48      IoUringSocketChannel(Channel parent, LinuxSocket fd) {
49          super(parent, fd, true);
50          this.config = new IoUringSocketChannelConfig(this);
51      }
52  
53      IoUringSocketChannel(Channel parent, LinuxSocket fd, SocketAddress remote) {
54          super(parent, fd, remote);
55          this.config = new IoUringSocketChannelConfig(this);
56      }
57  
58      /**
59       * Returns the {@code TCP_INFO} for the current socket.
60       * See <a href="https://linux.die.net//man/7/tcp">man 7 tcp</a>.
61       */
62      public IoUringTcpInfo tcpInfo() {
63          return tcpInfo(new IoUringTcpInfo());
64      }
65  
66      /**
67       * Updates and returns the {@code TCP_INFO} for the current socket.
68       * See <a href="https://linux.die.net//man/7/tcp">man 7 tcp</a>.
69       */
70      public IoUringTcpInfo tcpInfo(IoUringTcpInfo info) {
71          try {
72              socket.getTcpInfo(info);
73              return info;
74          } catch (IOException e) {
75              throw new ChannelException(e);
76          }
77      }
78  
79      @Override
80      public ServerSocketChannel parent() {
81          return (ServerSocketChannel) super.parent();
82      }
83  
84      @Override
85      public SocketChannelConfig config() {
86          return config;
87      }
88  
89      @Override
90      public InetSocketAddress remoteAddress() {
91          return (InetSocketAddress) super.remoteAddress();
92      }
93  
94      @Override
95      public InetSocketAddress localAddress() {
96          return (InetSocketAddress) super.localAddress();
97      }
98  
99      @Override
100     protected AbstractUringUnsafe newUnsafe() {
101         return new IoUringSocketUnsafe();
102     }
103 
104     private final class IoUringSocketUnsafe extends IoUringStreamUnsafe {
105         /**
106          * Holds buffers that we can't release yet as the kernel still holds a reference to these.
107          */
108         private PendingZeroCopyWrites pendingZeroCopyWrites;
109 
110         private ZeroCopyBatch zeroCopyBatch;
111 
112         @Override
113         protected int scheduleWriteSingle(Object msg) {
114             assert writeId == 0;
115 
116             if (IoUring.isSendZcSupported() && msg instanceof ByteBuf) {
117                 ByteBuf buf = (ByteBuf) msg;
118                 int length = buf.readableBytes();
119                 if (((IoUringSocketChannelConfig) config()).shouldWriteZeroCopy(length)) {
120                     long address = IoUring.memoryAddress(buf) + buf.readerIndex();
121                     long opsId = nextZeroCopyUserData();
122                     IoUringIoOps ops = IoUringIoOps.newSendZc(fd().intValue(), address, length, 0, opsId, 0);
123                     byte opCode = ops.opcode();
124                     writeId = submitWrite(ops);
125                     writeOpCode = opCode;
126                     if (writeId == 0) {
127                         pendingZeroCopyWrites.recycle(opsId);
128                         return 0;
129                     }
130                     return 1;
131                 }
132                 // Should not use send_zc, just use normal write.
133             }
134             return super.scheduleWriteSingle(msg);
135         }
136 
137         @Override
138         protected int scheduleWriteMultiple(ChannelOutboundBuffer in) {
139             assert writeId == 0;
140 
141             IoUringSocketChannelConfig ioUringSocketChannelConfig = (IoUringSocketChannelConfig) config();
142             int threshold = ioUringSocketChannelConfig.getWriteZeroCopyThreshold();
143             if (!IoUring.isSendmsgZcSupported() || threshold == IoUringSocketChannelConfig.DISABLE_WRITE_ZERO_COPY) {
144                 return super.scheduleWriteMultiple(in);
145             }
146             IoUringIoHandler handler = registration().attachment();
147 
148             IovArray iovArray = handler.iovArray();
149             int offset = iovArray.count();
150             if (zeroCopyBatch == null) {
151                 zeroCopyBatch = new ZeroCopyBatch();
152             }
153             try {
154                 // The iovArray is filled for a normal writev, the batch only watches what is added to it.
155                 in.forEachFlushedMessage(zeroCopyBatch.scan(threshold, iovArray));
156             } catch (Exception e) {
157                 // This should never happen, anyway fallback to single write.
158                 return scheduleWriteSingle(in.current());
159             }
160             // Only use zero copy if collection can reach a large buffer.
161             if (zeroCopyBatch.scannedLarge() && zeroCopyBatch.allocateMerged(alloc())) {
162                 try {
163                     zeroCopyBatch.collect(in, iovArray, offset);
164                 } catch (Exception e) {
165                     // This should never happen, anyway fallback to single write.
166                     zeroCopyBatch.release();
167                     return scheduleWriteSingle(in.current());
168                 }
169                 long iovArrayAddress = iovArray.memoryAddress(offset);
170                 int iovArrayLength = iovArray.count() - offset;
171 
172                 MsgHdrMemory hdr = writeMsgHdr();
173                 hdr.set(iovArrayAddress, iovArrayLength);
174                 long opsId = nextZeroCopyUserData();
175                 IoUringIoOps ops = IoUringIoOps.newSendmsgZc(
176                         fd().intValue(), (byte) 0, 0, hdr.address(), opsId);
177                 byte opCode = ops.opcode();
178                 writeId = submitWrite(ops);
179                 writeOpCode = opCode;
180                 if (writeId == 0) {
181                     pendingZeroCopyWrites.recycle(opsId);
182                     zeroCopyBatch.release();
183                     return 0;
184                 }
185                 return 1;
186             }
187             // Should not use sendmsg_zc, just use normal writev.
188             IoUringIoOps ops = IoUringIoOps.newWritev(fd().intValue(), (byte) 0, 0,
189                     iovArray.memoryAddress(offset), iovArray.count() - offset, nextOpsId());
190             byte opCode = ops.opcode();
191             writeId = submitWrite(ops);
192             writeOpCode = opCode;
193             return writeId == 0 ? 0 : 1;
194         }
195 
196         private long nextZeroCopyUserData() {
197             PendingZeroCopyWrites pendingWrites = pendingZeroCopyWrites;
198             if (pendingWrites == null) {
199                 pendingWrites = new PendingZeroCopyWrites();
200                 pendingZeroCopyWrites = pendingWrites;
201             }
202             return pendingWrites.nextUserData();
203         }
204 
205         @Override
206         List<ReferenceCounted> retainWriteBuffers(ChannelOutboundBuffer buffer) {
207             if (currentWrite.opcode() == Native.IORING_OP_SENDMSG_ZC) {
208                 return zeroCopyBatch.retain();
209             }
210             return super.retainWriteBuffers(buffer);
211         }
212 
213         @Override
214         boolean writeComplete0(byte op, int res, int flags, long data, int outstanding) {
215             if (op == Native.IORING_OP_SEND_ZC || op == Native.IORING_OP_SENDMSG_ZC) {
216                 return handleWriteCompleteZeroCopy(op, res, flags, data);
217             }
218             return super.writeComplete0(op, res, flags, data, outstanding);
219         }
220 
221         private void takeSendZCBuffers(long data) {
222             PendingZeroCopyWrites.PendingWrite write = pendingZeroCopyWrites.register(data);
223             if (retainedWriteBuffers == null) {
224                 if (currentWrite.opcode() == Native.IORING_OP_SENDMSG_ZC) {
225                     zeroCopyBatch.transferTo(write);
226                 } else {
227                     write.add(((ByteBuf) outboundBuffer().current()).retain());
228                 }
229             } else {
230                 // Shutdown may have cleared the outbound buffer before the primary CQE; use the retained buffers.
231                 for (int i = 0; i < retainedWriteBuffers.size(); i++) {
232                     write.add(retainedWriteBuffers.get(i));
233                 }
234                 retainedWriteBuffers = Collections.emptyList();
235             }
236         }
237 
238         private boolean handleWriteCompleteZeroCopy(byte op, int res, int flags, long data) {
239             if ((flags & Native.IORING_CQE_F_NOTIF) != 0) {
240                 pendingZeroCopyWrites.release(data);
241                 return true;
242             }
243             writeId = 0;
244             writeOpCode = 0;
245             if ((flags & Native.IORING_CQE_F_MORE) != 0) {
246                 // Establish notification ownership before removeBytes() can notify a write listener.
247                 takeSendZCBuffers(data);
248             } else {
249                 pendingZeroCopyWrites.recycle(data);
250                 if (op == Native.IORING_OP_SENDMSG_ZC) {
251                     zeroCopyBatch.release();
252                 }
253             }
254             ChannelOutboundBuffer channelOutboundBuffer = outboundBuffer();
255             if (channelOutboundBuffer == null) {
256                 return true;
257             }
258             if (res >= 0) {
259                 channelOutboundBuffer.removeBytes(res);
260                 return true;
261             }
262             if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
263                 return true;
264             }
265             try {
266                 return ioResult(op == Native.IORING_OP_SEND_ZC ? "io_uring sendzc" : "io_uring sendmsg_zc", res) != 0;
267             } catch (Throwable cause) {
268                 handleWriteError(cause);
269                 return true;
270             }
271         }
272     }
273 
274     /**
275      * The buffers of the {@code SENDMSG_ZC} that is in flight.
276      * <p>
277      * A message that reaches the zero-copy threshold is sent from its own memory. Smaller messages are not worth
278      * pinning their pages, so every run of adjacent small messages is copied into one buffer and only takes one
279      * iovec:
280      * <pre>
281      * messages: small small LARGE small LARGE LARGE small small
282      * iovecs:   [  merged  ] LARGE [mer.] LARGE LARGE [  merged  ]
283      * </pre>
284      */
285     private static final class ZeroCopyBatch implements ChannelOutboundBuffer.MessageProcessor {
286         // The pooled allocator pools up to 4MiB, the adaptive allocator only up to 1MiB.
287         private static final int MERGED_MAX_SIZE = Math.min(4 * 1024 * 1024,
288                 Math.max(1, SystemPropertyUtil.getInt("io.netty.iouring.zeroCopyMergedMaxSize", 64 * 1024)));
289 
290         // The messages that are sent from their own memory, these are not retained yet.
291         private final List<ByteBuf> largeBuffers = new ArrayList<>(4);
292         // Holds the bytes of the small messages, the batch owns the reference it was allocated with.
293         private ByteBuf merged;
294         // Start of the run of small messages in merged that was not added to the iovArray yet.
295         private int runStart;
296         private IovArray iovArray;
297         private int threshold;
298         private long smallBytes;
299         private boolean hasLarge;
300         private boolean scanning;
301 
302         /**
303          * Returns a {@link ChannelOutboundBuffer.MessageProcessor} that adds the messages to {@code iovArray} like
304          * the {@code iovArray} itself would do for a writev. This also tells which messages fit and how many bytes
305          * need to be copied, so one allocation is enough.
306          */
307         ChannelOutboundBuffer.MessageProcessor scan(int threshold, IovArray iovArray) {
308             this.iovArray = iovArray;
309             this.threshold = threshold;
310             smallBytes = 0;
311             hasLarge = false;
312             scanning = true;
313             return this;
314         }
315 
316         /**
317          * Returns {@code true} if a scanned iovec reaches the threshold and the preceding small iovecs fit in the
318          * merged buffer. The result is only valid once.
319          */
320         boolean scannedLarge() {
321             boolean large = hasLarge;
322             hasLarge = false;
323             scanning = false;
324             return large;
325         }
326 
327         /**
328          * Allocates the buffer for the scanned small messages. Returns {@code false} if there is no memory for it,
329          * the iovArray still holds the entries of the scan then, so the messages can be written with a writev. This
330          * must not throw as it also runs while handling a completion, where an {@link Error} ends the event loop.
331          */
332         boolean allocateMerged(ByteBufAllocator allocator) {
333             if (smallBytes != 0) {
334                 // What does not fit is written later.
335                 int capacity = (int) Math.min(smallBytes, MERGED_MAX_SIZE);
336                 try {
337                     merged = allocator.directBuffer(capacity, capacity);
338                 } catch (Exception | OutOfMemoryError e) {
339                     return false;
340                 }
341                 runStart = 0;
342             }
343             return true;
344         }
345 
346         /**
347          * Replaces the entries that were added to the iovArray during the scan.
348          */
349         void collect(ChannelOutboundBuffer in, IovArray iovArray, int offset) throws Exception {
350             iovArray.truncate(offset);
351             in.forEachFlushedMessage(this);
352             addMergedSmallBuffers();
353         }
354 
355         @Override
356         public boolean processMessage(Object msg) throws Exception {
357             if (scanning) {
358                 int count = iovArray.count();
359                 boolean added = iovArray.processMessage(msg);
360                 // Look at the entries and not at the message, a message may consist of multiple buffers.
361                 for (int i = count; i < iovArray.count(); i++) {
362                     long length = iovArray.length(i);
363                     if (length < threshold) {
364                         smallBytes += length;
365                     } else if (smallBytes <= MERGED_MAX_SIZE) {
366                         hasLarge = true;
367                     }
368                 }
369                 return added;
370             }
371             if (!(msg instanceof ByteBuf)) {
372                 return false;
373             }
374             ByteBuf buf = (ByteBuf) msg;
375             int index = buf.readerIndex();
376             int readableBytes = buf.readableBytes();
377             // All buffers of a message that is smaller than the threshold are smaller too, so copy it at once.
378             if (readableBytes < threshold || buf.nioBufferCount() == 1) {
379                 return add(buf, index, readableBytes);
380             }
381             for (ByteBuffer nioBuffer : buf.nioBuffers(index, readableBytes)) {
382                 int length = nioBuffer.remaining();
383                 if (!add(buf, index, length)) {
384                     return false;
385                 }
386                 index += length;
387             }
388             return true;
389         }
390 
391         private boolean add(ByteBuf buf, int index, int length) {
392             if (length < threshold) {
393                 if (length == 0) {
394                     return true;
395                 }
396                 if (merged == null) {
397                     return false;
398                 }
399                 // Copy what fits, the rest of the buffer is written later.
400                 int copied = Math.min(length, merged.writableBytes());
401                 merged.writeBytes(buf, index, copied);
402                 return copied == length;
403             }
404             if (!addMergedSmallBuffers()) {
405                 return false;
406             }
407             int count = iovArray.count();
408             boolean added = iovArray.add(buf, index, length);
409             // IovArray.add(...) may return false although the entry was added, so check the count.
410             if (iovArray.count() != count
411                     && (largeBuffers.isEmpty() || largeBuffers.get(largeBuffers.size() - 1) != buf)) {
412                 // A message that consists of CompositeByteBuf has one reference count for
413                 // all of them, and its buffers are added one after another. So it is enough to track it once.
414                 largeBuffers.add(buf);
415             }
416             return added;
417         }
418 
419         private boolean addMergedSmallBuffers() {
420             if (merged == null || merged.writerIndex() == runStart) {
421                 return true;
422             }
423             if (!iovArray.add(merged, runStart, merged.writerIndex() - runStart)) {
424                 return false;
425             }
426             runStart = merged.writerIndex();
427             return true;
428         }
429 
430         void transferTo(PendingZeroCopyWrites.PendingWrite write) {
431             for (int i = 0; i < largeBuffers.size(); i++) {
432                 write.add(largeBuffers.get(i).retain());
433             }
434             if (merged != null) {
435                 write.add(merged);
436             }
437             largeBuffers.clear();
438             merged = null;
439         }
440 
441         /**
442          * Same as {@link #transferTo(PendingZeroCopyWrites.PendingWrite)}, but for the case that the outbound
443          * messages are released before the write completes.
444          */
445         List<ReferenceCounted> retain() {
446             List<ReferenceCounted> buffers = new ArrayList<>(largeBuffers.size() + 1);
447             for (int i = 0; i < largeBuffers.size(); i++) {
448                 buffers.add(largeBuffers.get(i).retain());
449             }
450             if (merged != null) {
451                 buffers.add(merged);
452             }
453             largeBuffers.clear();
454             merged = null;
455             return buffers;
456         }
457 
458         /**
459          * Must be called if the kernel doesn't reference the buffers (anymore).
460          */
461         void release() {
462             if (merged != null) {
463                 merged.release();
464                 merged = null;
465             }
466             largeBuffers.clear();
467         }
468     }
469 }