View Javadoc
1   /*
2    * Copyright 2024 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.channel.uring;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.buffer.ByteBufAllocator;
20  import io.netty.buffer.ByteBufHolder;
21  import io.netty.buffer.ByteBufUtil;
22  import io.netty.buffer.Unpooled;
23  import io.netty.channel.AbstractChannel;
24  import io.netty.channel.Channel;
25  import io.netty.channel.ChannelConfig;
26  import io.netty.channel.ChannelFuture;
27  import io.netty.channel.ChannelFutureListener;
28  import io.netty.channel.ChannelOption;
29  import io.netty.channel.ChannelOutboundBuffer;
30  import io.netty.channel.ChannelPromise;
31  import io.netty.channel.ConnectTimeoutException;
32  import io.netty.channel.EventLoop;
33  import io.netty.channel.IoEvent;
34  import io.netty.channel.IoEventLoop;
35  import io.netty.channel.IoRegistration;
36  import io.netty.channel.RecvByteBufAllocator;
37  import io.netty.channel.ServerChannel;
38  import io.netty.channel.socket.ChannelInputShutdownEvent;
39  import io.netty.channel.socket.ChannelInputShutdownReadComplete;
40  import io.netty.channel.socket.SocketChannelConfig;
41  import io.netty.channel.unix.Buffer;
42  import io.netty.channel.unix.DomainSocketAddress;
43  import io.netty.channel.unix.Errors;
44  import io.netty.channel.unix.FileDescriptor;
45  import io.netty.channel.unix.IovArray;
46  import io.netty.channel.unix.UnixChannel;
47  import io.netty.channel.unix.UnixChannelUtil;
48  import io.netty.util.ReferenceCountUtil;
49  import io.netty.util.concurrent.PromiseNotifier;
50  import io.netty.util.internal.CleanableDirectBuffer;
51  import io.netty.util.internal.StringUtil;
52  import io.netty.util.internal.logging.InternalLogger;
53  import io.netty.util.internal.logging.InternalLoggerFactory;
54  
55  import java.io.IOException;
56  import java.net.InetSocketAddress;
57  import java.net.SocketAddress;
58  import java.nio.ByteBuffer;
59  import java.nio.channels.AlreadyConnectedException;
60  import java.nio.channels.ClosedChannelException;
61  import java.nio.channels.ConnectionPendingException;
62  import java.nio.channels.NotYetConnectedException;
63  import java.nio.channels.UnresolvedAddressException;
64  import java.util.concurrent.ScheduledFuture;
65  import java.util.concurrent.TimeUnit;
66  
67  import static io.netty.channel.unix.Errors.ERRNO_EINPROGRESS_NEGATIVE;
68  import static io.netty.channel.unix.Errors.ERROR_EALREADY_NEGATIVE;
69  import static io.netty.channel.unix.UnixChannelUtil.computeRemoteAddr;
70  import static io.netty.util.internal.ObjectUtil.checkNotNull;
71  import static io.netty.util.internal.StringUtil.className;
72  
73  
74  abstract class AbstractIoUringChannel extends AbstractChannel implements UnixChannel {
75      private static final InternalLogger logger = InternalLoggerFactory.getInstance(AbstractIoUringChannel.class);
76      final LinuxSocket socket;
77      // Owns every in-flight write operation this channel is tracking -- the pooled slot array, the overflow map,
78      // the foreign slot array, and the single stream slot. See WriteOperationTracker for the four namespaces.
79      final WriteOperationTracker writeTracker = new WriteOperationTracker();
80      protected volatile boolean active;
81  
82      // Different masks for outstanding I/O operations.
83      private static final int POLL_IN_SCHEDULED = 1;
84      private static final int POLL_OUT_SCHEDULED = 1 << 2;
85      private static final int POLL_RDHUP_SCHEDULED = 1 << 3;
86      private static final int WRITE_SCHEDULED = 1 << 4;
87      private static final int READ_SCHEDULED = 1 << 5;
88      private static final int CONNECT_SCHEDULED = 1 << 6;
89  
90      private short opsId = Short.MIN_VALUE;
91  
92      private long pollInId;
93      private long pollOutId;
94      private long pollRdhupId;
95      private long connectId;
96  
97      // A byte is enough for now.
98      private byte ioState;
99  
100     // It's possible that multiple read / writes are issued. We need to keep track of these.
101     // Let's limit the amount of pending writes and reads by Short.MAX_VALUE. Maybe Byte.MAX_VALUE would also be good
102     // enough but let's be a bit more flexible for now.
103     private short numOutstandingWrites;
104     // A value of -1 means that multi-shot is used and so reads will be issued as long as the request is not canceled.
105     private short numOutstandingReads;
106 
107     private boolean readPending;
108     private boolean inReadComplete;
109     private boolean socketHasMoreData;
110 
111     private static final class DelayedClose {
112         private final ChannelPromise promise;
113         private final Throwable cause;
114         private final ClosedChannelException closeCause;
115 
116         DelayedClose(ChannelPromise promise, Throwable cause, ClosedChannelException closeCause) {
117             this.promise = promise;
118             this.cause = cause;
119             this.closeCause = closeCause;
120         }
121     }
122     private DelayedClose delayedClose;
123     private boolean inputClosedSeenErrorOnRead;
124 
125     /**
126      * The future of the current connection attempt.  If not null, subsequent connection attempts will fail.
127      */
128     private ChannelPromise connectPromise;
129     private ScheduledFuture<?> connectTimeoutFuture;
130     private SocketAddress requestedRemoteAddress;
131     private CleanableDirectBuffer cleanable;
132     private ByteBuffer remoteAddressMemory;
133     private MsgHdrMemoryArray msgHdrMemoryArray;
134 
135     private IoRegistration registration;
136 
137     private volatile SocketAddress local;
138     private volatile SocketAddress remote;
139 
140     AbstractIoUringChannel(final Channel parent, LinuxSocket socket, boolean active) {
141         super(parent);
142         this.socket = checkNotNull(socket, "fd");
143 
144         if (active) {
145             // Directly cache the remote and local addresses
146             // See https://github.com/netty/netty/issues/2359
147             this.active = true;
148             this.local = socket.localAddress();
149             this.remote = socket.remoteAddress();
150         }
151 
152         logger.trace("Create {} Socket: {}", this instanceof ServerChannel ? "Server" : "Channel", socket.intValue());
153     }
154 
155     AbstractIoUringChannel(Channel parent, LinuxSocket fd, SocketAddress remote) {
156         super(parent);
157         this.socket = checkNotNull(fd, "fd");
158         this.active = true;
159 
160         // Directly cache the remote and local addresses
161         // See https://github.com/netty/netty/issues/2359
162         this.remote = remote;
163         this.local = fd.localAddress();
164     }
165 
166     // Called once a Channel changed from AUTO_READ=true to AUTO_READ=false
167     final void autoReadCleared() {
168         if (!isRegistered()) {
169             return;
170         }
171         IoRegistration registration = this.registration;
172         if (registration == null || !registration.isValid()) {
173             return;
174         }
175         if (eventLoop().inEventLoop()) {
176             clearRead();
177         } else {
178             eventLoop().execute(this::clearRead);
179         }
180     }
181 
182     private void clearRead() {
183         assert eventLoop().inEventLoop();
184         readPending = false;
185         IoRegistration registration = this.registration;
186         if (registration == null || !registration.isValid()) {
187             return;
188         }
189         // Also cancel all outstanding reads as the user did signal there is no more desire to read.
190         cancelOutstandingReads(registration(), numOutstandingReads);
191     }
192 
193     /**
194      * Returns the next id that should be used when submitting {@link IoUringIoOps}.
195      *
196      * @return  opsId
197      */
198     protected final short nextOpsId() {
199         short id = opsId++;
200 
201         // We use 0 for "none".
202         if (id == 0) {
203             id = opsId++;
204         }
205         return id;
206     }
207 
208     public final boolean isOpen() {
209         return socket.isOpen();
210     }
211 
212     @Override
213     public boolean isActive() {
214         return active;
215     }
216 
217     @Override
218     public final FileDescriptor fd() {
219         return socket;
220     }
221 
222     private AbstractUringUnsafe ioUringUnsafe() {
223         return (AbstractUringUnsafe) unsafe();
224     }
225 
226     @Override
227     protected boolean isCompatible(final EventLoop loop) {
228         return loop instanceof IoEventLoop && ((IoEventLoop) loop).isCompatible(AbstractUringUnsafe.class);
229     }
230 
231     protected final ByteBuf newDirectBuffer(ByteBuf buf) {
232         return newDirectBuffer(buf, buf);
233     }
234 
235     protected boolean allowMultiShotPollIn() {
236         return IoUring.isPollAddMultishotEnabled();
237     }
238 
239     protected final ByteBuf newDirectBuffer(Object holder, ByteBuf buf) {
240         final int readableBytes = buf.readableBytes();
241         if (readableBytes == 0) {
242             ReferenceCountUtil.release(holder);
243             return Unpooled.EMPTY_BUFFER;
244         }
245 
246         final ByteBufAllocator alloc = alloc();
247         if (alloc.isDirectBufferPooled()) {
248             return newDirectBuffer0(holder, buf, alloc, readableBytes);
249         }
250 
251         final ByteBuf directBuf = ByteBufUtil.threadLocalDirectBuffer();
252         if (directBuf == null) {
253             return newDirectBuffer0(holder, buf, alloc, readableBytes);
254         }
255 
256         directBuf.writeBytes(buf, buf.readerIndex(), readableBytes);
257         ReferenceCountUtil.safeRelease(holder);
258         return directBuf;
259     }
260 
261     private static ByteBuf newDirectBuffer0(Object holder, ByteBuf buf, ByteBufAllocator alloc, int capacity) {
262         final ByteBuf directBuf = alloc.directBuffer(capacity);
263         directBuf.writeBytes(buf, buf.readerIndex(), capacity);
264         ReferenceCountUtil.safeRelease(holder);
265         return directBuf;
266     }
267 
268     /**
269      * Cancel all outstanding reads
270      *
271      * @param registration          the {@link IoRegistration}.
272      * @param numOutstandingReads   the number of outstanding reads, or {@code -1} if multi-shot was used.
273      */
274     protected abstract void cancelOutstandingReads(IoRegistration registration, int numOutstandingReads);
275 
276     /**
277      * Cancel all outstanding writes
278      *
279      * @param registration          the {@link IoRegistration}.
280      * @param numOutstandingWrites  the number of outstanding writes.
281      */
282     protected abstract void cancelOutstandingWrites(IoRegistration registration, int numOutstandingWrites);
283 
284     @Override
285     protected void doDisconnect() throws Exception {
286     }
287 
288     private void freeRemoteAddressMemory() {
289         if (remoteAddressMemory != null) {
290             cleanable.clean();
291             cleanable = null;
292             remoteAddressMemory = null;
293         }
294     }
295 
296     private void freeMsgHdrArray() {
297         if (msgHdrMemoryArray != null) {
298             msgHdrMemoryArray.release();
299             msgHdrMemoryArray = null;
300         }
301     }
302 
303     @Override
304     protected void doClose() throws Exception {
305         active = false;
306 
307         if (registration != null) {
308             if (socket.markClosed()) {
309                 int fd = fd().intValue();
310                 IoUringIoOps ops = IoUringIoOps.newClose(fd, (byte) 0, nextOpsId());
311                 registration.submit(ops);
312             }
313         } else {
314             // This one was never registered just use a syscall to close.
315             socket.close();
316             ioUringUnsafe().unregistered();
317         }
318     }
319 
320     /**
321      * Retains every in-flight write's references before handing off to {@link #doShutdownOutput0()}, so a write
322      * completion that races the shutdown still finds a live reference to release instead of one the outbound buffer
323      * already dropped.
324      */
325     @Override
326     protected final void doShutdownOutput() throws Exception {
327         writeTracker.retainAll();
328         doShutdownOutput0();
329     }
330 
331     /**
332      * Performs the actual output shutdown. Overridden by subclasses that support it.
333      */
334     protected void doShutdownOutput0() throws Exception {
335         super.doShutdownOutput();
336     }
337 
338     @Override
339     protected final void doBeginRead() {
340         if (inputClosedSeenErrorOnRead) {
341             // We did see an error while reading and so closed the input. Stop reading.
342             return;
343         }
344         if (readPending) {
345             // We already have a read pending.
346             return;
347         }
348         readPending = true;
349         if (inReadComplete || !isActive()) {
350             // We are currently in the readComplete(...) callback which might issue more reads by itself.
351             // If readComplete(...) will not issue more reads itself it will pick up the readPending flag, reset it and
352             // call doBeginReadNow().
353             return;
354         }
355         doBeginReadNow();
356     }
357 
358     private void doBeginReadNow() {
359         if (inputClosedSeenErrorOnRead) {
360             // We did see an error while reading and so closed the input.
361             return;
362         }
363         if (!isPollInFirst() ||
364                 // If the socket was not empty, and we stopped reading we need to ensure we just force the
365                 // read as POLLIN might be edge-triggered (in case of POLL_ADD_MULTI).
366                 socketHasMoreData) {
367             // If the socket is blocking we will directly call scheduleFirstReadIfNeeded() as we can use FASTPOLL.
368             ioUringUnsafe().scheduleFirstReadIfNeeded();
369         } else if ((ioState & POLL_IN_SCHEDULED) == 0) {
370             ioUringUnsafe().schedulePollIn();
371         }
372     }
373 
374     @Override
375     protected void doWrite(ChannelOutboundBuffer in) {
376         scheduleWriteIfNeeded(in, true);
377     }
378 
379     protected void scheduleWriteIfNeeded(ChannelOutboundBuffer in, boolean submitAndRunNow) {
380         if ((ioState & WRITE_SCHEDULED) != 0) {
381             return;
382         }
383         if (scheduleWrite(in) > 0) {
384             ioState |= WRITE_SCHEDULED;
385             if (submitAndRunNow && !isWritable()) {
386                 submitAndRunNow();
387             }
388         }
389     }
390 
391     protected void submitAndRunNow() {
392         // NOOP
393     }
394 
395     private int scheduleWrite(ChannelOutboundBuffer in) {
396         if (delayedClose != null || numOutstandingWrites == Short.MAX_VALUE) {
397             return 0;
398         }
399         if (in == null) {
400             return 0;
401         }
402 
403         int msgCount = in.size();
404         if (msgCount == 0) {
405             return 0;
406         }
407         Object msg = in.current();
408 
409         int scheduled;
410         if (msgCount > 1 && in.current() instanceof ByteBuf) {
411             scheduled = ioUringUnsafe().scheduleWriteMultiple(in);
412         } else if (msg instanceof ByteBuf && ((ByteBuf) msg).nioBufferCount() > 1 ||
413                     (msg instanceof ByteBufHolder && ((ByteBufHolder) msg).content().nioBufferCount() > 1)) {
414             // We also need some special handling for CompositeByteBuf
415             scheduled = ioUringUnsafe().scheduleWriteMultiple(in);
416         } else {
417             scheduled = ioUringUnsafe().scheduleWriteSingle(msg);
418         }
419         // A zero return means the write could not be scheduled: registration.submit(...) failed because the
420         // registration is no longer valid, a FileRegion's open() threw, or transferTo(...) produced no bytes or
421         // threw. numOutstandingWrites is a short, so guard the narrowing before it happens: a future
422         // scheduleWriteSingle/scheduleWriteMultiple override that batches more writes than a short can hold must
423         // not silently wrap around.
424         assert scheduled <= Short.MAX_VALUE;
425         numOutstandingWrites = (short) scheduled;
426         return numOutstandingWrites;
427     }
428 
429     protected final IoRegistration registration() {
430         assert registration != null;
431         return registration;
432     }
433 
434     private void schedulePollOut() {
435         pollOutId = schedulePollAdd(POLL_OUT_SCHEDULED, Native.POLLOUT, false);
436     }
437 
438     final void schedulePollRdHup() {
439         pollRdhupId = schedulePollAdd(POLL_RDHUP_SCHEDULED, Native.POLLRDHUP, false);
440     }
441 
442     protected abstract boolean isStreamSocket();
443 
444     private long schedulePollAdd(int ioMask, int mask, boolean multishot) {
445         assert (ioState & ioMask) == 0;
446         int fd = fd().intValue();
447         IoRegistration registration = registration();
448         IoUringIoOps ops = IoUringIoOps.newPollAdd(
449                 fd, (byte) 0, mask, multishot ? Native.IORING_POLL_ADD_MULTI : 0, nextOpsId());
450         long id = registration.submit(ops);
451         if (id != 0) {
452             ioState |= (byte) ioMask;
453         }
454         return id;
455     }
456 
457     final void resetCachedAddresses() {
458         local = socket.localAddress();
459         remote = socket.remoteAddress();
460     }
461 
462     // Write completions may carry an id that fell back out of the short range, so they get the untruncated
463     // user_data. Connect completions always submit a short id, but connectComplete(...) takes a long to stay
464     // consistent with writeComplete(...), so it is also handed the untruncated value. The remaining completions
465     // (read, poll, cancel) only ever submit short ids, so narrowUserData(...) below asserts and narrows those.
466     private static short narrowUserData(long userData) {
467         assert userData == (short) userData : "user_data does not fit a short: " + userData;
468         return (short) userData;
469     }
470 
471     protected abstract class AbstractUringUnsafe extends AbstractUnsafe implements IoUringIoHandle {
472         private IoUringRecvByteAllocatorHandle allocHandle;
473         private boolean closed;
474         private boolean socketIsEmpty;
475         private ChannelPromise deregisterPromise;
476 
477         /**
478          * Schedule the write of multiple messages in the {@link ChannelOutboundBuffer} and returns the number of
479          * {@link #writeComplete(byte, int, int, long)} calls that are expected because of the scheduled write.
480          */
481         protected abstract int scheduleWriteMultiple(ChannelOutboundBuffer in);
482 
483         /**
484          * Schedule the write of a single message and returns the number of
485          * {@link #writeComplete(byte, int, int, long)} calls that are expected because of the scheduled write.
486          */
487         protected abstract int scheduleWriteSingle(Object msg);
488 
489         @Override
490         public final void handle(IoRegistration registration, IoEvent ioEvent) {
491             IoUringIoEvent event = (IoUringIoEvent) ioEvent;
492             byte op = event.opcode();
493             int res = event.res();
494             int flags = event.flags();
495             long userData = event.userData();
496             switch (op) {
497                 case Native.IORING_OP_RECV:
498                 case Native.IORING_OP_ACCEPT:
499                 case Native.IORING_OP_RECVMSG:
500                 case Native.IORING_OP_READ:
501                     readComplete(op, res, flags, narrowUserData(userData));
502                     break;
503                 case Native.IORING_OP_WRITEV:
504                 case Native.IORING_OP_SEND:
505                 case Native.IORING_OP_SENDMSG:
506                 case Native.IORING_OP_WRITE:
507                 case Native.IORING_OP_SPLICE:
508                 case Native.IORING_OP_SEND_ZC:
509                 case Native.IORING_OP_SENDMSG_ZC:
510                     writeComplete(op, res, flags, userData);
511                     break;
512                 case Native.IORING_OP_POLL_ADD:
513                     pollAddComplete(res, flags, narrowUserData(userData));
514                     break;
515                 case Native.IORING_OP_ASYNC_CANCEL:
516                     cancelComplete0(op, res, flags, narrowUserData(userData));
517                     break;
518                 case Native.IORING_OP_CONNECT:
519                     connectComplete(op, res, flags, userData);
520 
521                     // once the connect was completed we can also free some resources that are not needed anymore.
522                     freeMsgHdrArray();
523                     freeRemoteAddressMemory();
524                     break;
525                 case Native.IORING_OP_CLOSE:
526                     if (res != Native.ERRNO_ECANCELED_NEGATIVE) {
527                         if (delayedClose != null) {
528                             delayedClose.promise.setSuccess();
529                         }
530                         closed = true;
531                     }
532                     break;
533                 default:
534                     break;
535             }
536 
537             // We delay the actual close if there is still a write or read scheduled, let's see if there
538             // was a close that needs to be done now.
539             handleDelayedClosed();
540 
541             if (ioState == 0 && (closed || !isRegistered())) {
542                 // Cancel the registration now.
543                 registration.cancel();
544             }
545         }
546 
547         @Override
548         public void unregistered() {
549             freeMsgHdrArray();
550             freeRemoteAddressMemory();
551             writeTracker.releaseAll();
552 
553             // Check if we need to notify about the deregistration.
554             if (deregisterPromise != null) {
555                 ChannelPromise promise = deregisterPromise;
556                 deregisterPromise = null;
557                 promise.setSuccess();
558             }
559         }
560 
561         private void handleDelayedClosed() {
562             if (delayedClose != null && canCloseNow()) {
563                 closeNow();
564             }
565         }
566 
567         private void pollAddComplete(int res, int flags, short data) {
568             if ((res & Native.POLLOUT) != 0) {
569                 pollOut(res);
570             }
571             if ((res & Native.POLLIN) != 0) {
572                 pollIn(res, flags, data);
573             }
574             if ((res & Native.POLLRDHUP) != 0) {
575                 pollRdHup(res);
576             }
577         }
578 
579         @Override
580         public final void close() throws Exception {
581             close(voidPromise());
582         }
583 
584         @Override
585         protected void close(ChannelPromise promise, Throwable cause, ClosedChannelException closeCause) {
586             if (closeFuture().isDone()) {
587                 // Closed already before.
588                 safeSetSuccess(promise);
589                 return;
590             }
591             if (delayedClose == null) {
592                 // We have a write operation pending that should be completed asap.
593                 // We will do the actual close operation one this write result is returned as otherwise
594                 // we may get into trouble as we may close the fd while we did not process the write yet.
595                 delayedClose = new DelayedClose(promise.isVoid() ? newPromise() : promise, cause, closeCause);
596             } else {
597                 delayedClose.promise.addListener(new PromiseNotifier<>(false, promise));
598                 return;
599             }
600 
601             boolean cancelConnect = false;
602             try {
603                 ChannelPromise connectPromise = AbstractIoUringChannel.this.connectPromise;
604                 if (connectPromise != null) {
605                     // Use tryFailure() instead of setFailure() to avoid the race against cancel().
606                     connectPromise.tryFailure(new ClosedChannelException());
607                     AbstractIoUringChannel.this.connectPromise = null;
608                     cancelConnect = true;
609                 }
610 
611                 cancelConnectTimeoutFuture();
612             } finally {
613                 // It's important we cancel all outstanding connect, write and read operations now so
614                 // we will be able to process a delayed close if needed.
615                 cancelOps(cancelConnect);
616             }
617 
618             if (canCloseNow()) {
619                 // Currently there are is no WRITE and READ scheduled so we can start to teardown the channel.
620                 closeNow();
621             }
622         }
623 
624         private boolean cancelOps(boolean cancelConnect) {
625             if (registration == null || !registration.isValid()) {
626                 return false;
627             }
628             boolean cancelled = false;
629             byte flags = (byte) 0;
630             if ((ioState & POLL_RDHUP_SCHEDULED) != 0 && pollRdhupId != 0) {
631                 long id = registration.submit(
632                         IoUringIoOps.newAsyncCancel(flags, pollRdhupId, Native.IORING_OP_POLL_ADD));
633                 assert id != 0;
634                 pollRdhupId = 0;
635                 cancelled = true;
636             }
637             if ((ioState & POLL_IN_SCHEDULED) != 0 && pollInId != 0) {
638                 long id = registration.submit(
639                         IoUringIoOps.newAsyncCancel(flags, pollInId, Native.IORING_OP_POLL_ADD));
640                 assert id != 0;
641                 pollInId = 0;
642                 cancelled = true;
643             }
644             if ((ioState & POLL_OUT_SCHEDULED) != 0 && pollOutId != 0) {
645                 long id = registration.submit(
646                         IoUringIoOps.newAsyncCancel(flags, pollOutId, Native.IORING_OP_POLL_ADD));
647                 assert id != 0;
648                 pollOutId = 0;
649                 cancelled = true;
650             }
651             if (cancelConnect && connectId != 0) {
652                 // Best effort to cancel the already submitted connect request.
653                 long id = registration.submit(IoUringIoOps.newAsyncCancel(flags, connectId, Native.IORING_OP_CONNECT));
654                 assert id != 0;
655                 connectId = 0;
656                 cancelled = true;
657             }
658             if (numOutstandingReads != 0 || numOutstandingWrites != 0) {
659                 cancelled = true;
660             }
661             cancelOutstandingReads(registration, numOutstandingReads);
662             cancelOutstandingWrites(registration, numOutstandingWrites);
663             return cancelled;
664         }
665 
666         private boolean canCloseNow() {
667             // Currently there are is no WRITE and READ scheduled, we can close the channel now without
668             // problems related to re-ordering of completions.
669             return canCloseNow0() && (ioState & (WRITE_SCHEDULED | READ_SCHEDULED)) == 0;
670         }
671 
672         protected boolean canCloseNow0() {
673             return true;
674         }
675 
676         private void closeNow() {
677             super.close(newPromise(), delayedClose.cause, delayedClose.closeCause);
678         }
679 
680         @Override
681         protected final void flush0() {
682             // Flush immediately only when there's no pending flush.
683             // If there's a pending flush operation, event loop will call forceFlush() later,
684             // and thus there's no need to call it now.
685             if ((ioState & POLL_OUT_SCHEDULED) == 0) {
686                 super.flush0();
687             }
688         }
689 
690         private void fulfillConnectPromise(ChannelPromise promise, Throwable cause) {
691             if (promise == null) {
692                 // Closed via cancellation and the promise has been notified already.
693                 return;
694             }
695 
696             // Use tryFailure() instead of setFailure() to avoid the race against cancel().
697             promise.tryFailure(cause);
698             closeIfClosed();
699         }
700 
701         private void fulfillConnectPromise(ChannelPromise promise, boolean wasActive) {
702             if (promise == null) {
703                 // Closed via cancellation and the promise has been notified already.
704                 return;
705             }
706             active = true;
707 
708             if (local == null) {
709                 local = socket.localAddress();
710             }
711             computeRemote();
712 
713             if (isStreamSocket()) {
714                 // Register POLLRDHUP
715                 schedulePollRdHup();
716             }
717 
718             // Get the state as trySuccess() may trigger an ChannelFutureListener that will close the Channel.
719             // We still need to ensure we call fireChannelActive() in this case.
720             boolean active = isActive();
721 
722             // trySuccess() will return false if a user cancelled the connection attempt.
723             boolean promiseSet = promise.trySuccess();
724 
725             // Regardless if the connection attempt was cancelled, channelActive() event should be triggered,
726             // because what happened is what happened.
727             if (!wasActive && active) {
728                 pipeline().fireChannelActive();
729             }
730 
731             // If a user cancelled the connection attempt, close the channel, which is followed by channelInactive().
732             if (!promiseSet) {
733                 close(voidPromise());
734             }
735         }
736 
737         @Override
738         public final IoUringRecvByteAllocatorHandle recvBufAllocHandle() {
739             if (allocHandle == null) {
740                 allocHandle = new IoUringRecvByteAllocatorHandle(
741                         (RecvByteBufAllocator.ExtendedHandle) super.recvBufAllocHandle());
742             }
743             return allocHandle;
744         }
745 
746         final void shutdownInput(boolean allDataRead) {
747             logger.trace("shutdownInput Fd: {}", fd().intValue());
748             if (!socket.isInputShutdown()) {
749                 if (isAllowHalfClosure(config())) {
750                     try {
751                         socket.shutdown(true, false);
752                     } catch (IOException ignored) {
753                         // We attempted to shutdown and failed, which means the input has already effectively been
754                         // shutdown.
755                         fireEventAndClose(ChannelInputShutdownEvent.INSTANCE);
756                         return;
757                     } catch (NotYetConnectedException ignore) {
758                         // We attempted to shutdown and failed, which means the input has already effectively been
759                         // shutdown.
760                     }
761                     pipeline().fireUserEventTriggered(ChannelInputShutdownEvent.INSTANCE);
762                 } else {
763                     // Handle this same way as if we did read all data so we don't schedule another read.
764                     inputClosedSeenErrorOnRead = true;
765                     close(voidPromise());
766                     return;
767                 }
768             }
769             if (allDataRead && !inputClosedSeenErrorOnRead) {
770                 inputClosedSeenErrorOnRead = true;
771                 pipeline().fireUserEventTriggered(ChannelInputShutdownReadComplete.INSTANCE);
772             }
773         }
774 
775         private void fireEventAndClose(Object evt) {
776             pipeline().fireUserEventTriggered(evt);
777             close(voidPromise());
778         }
779 
780         final void schedulePollIn() {
781             assert (ioState & POLL_IN_SCHEDULED) == 0;
782             if (!isActive() || shouldBreakIoUringInReady(config())) {
783                 return;
784             }
785             pollInId = schedulePollAdd(POLL_IN_SCHEDULED, Native.POLLIN, allowMultiShotPollIn());
786         }
787 
788         protected final boolean isReadMultishot() {
789             return numOutstandingReads == -1;
790         }
791 
792         private void readComplete(byte op, int res, int flags, short data) {
793             assert numOutstandingReads > 0 || numOutstandingReads == -1 : numOutstandingReads;
794 
795             boolean multishot = isReadMultishot();
796             boolean rearm = (flags & Native.IORING_CQE_F_MORE) == 0;
797             boolean pending = readPending;
798             if (multishot) {
799                 if (rearm) {
800                     // Reset READ_SCHEDULED if there is nothing more to handle and so we need to re-arm.
801                     ioState &= ~READ_SCHEDULED;
802                 }
803                 // Reset readPending so we can still keep track if we might need to cancel the multi-shot read or
804                 // not.
805                 readPending = false;
806             } else if (--numOutstandingReads == 0) {
807                 // We received all outstanding completions.
808                 readPending = false;
809                 ioState &= ~READ_SCHEDULED;
810             }
811             inReadComplete = true;
812             try {
813                 socketIsEmpty = socketIsEmpty(flags);
814                 socketHasMoreData = IoUring.isCqeFSockNonEmptySupported() &&
815                         (flags & Native.IORING_CQE_F_SOCK_NONEMPTY) != 0;
816                 readComplete0(op, res, flags, data, numOutstandingReads);
817             } finally {
818                 try {
819                     // Check if we should consider the read loop to be done.
820                     if (recvBufAllocHandle().isReadComplete()) {
821                         // Reset the handle as we are done with the read-loop.
822                         recvBufAllocHandle().reset(config());
823 
824                         // Check if this was a readComplete(...) triggered by a read or multi-shot read.
825                         if (!multishot) {
826                             if (readPending) {
827                                 // This was a "normal" read and the user did signal we should continue reading.
828                                 // Let's schedule the read now.
829                                 doBeginReadNow();
830                             }
831                         } else {
832                             // The readComplete(...) was triggered by a multi-shot read. Because of this the state
833                             // machine is a bit more complicated.
834 
835                             if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
836                                 // The readComplete(...) was triggered because the previous read was cancelled.
837                                 // In this case we we need to check if the user did signal the desire to read again
838                                 // in the meantime. If this is the case we need to schedule the read to ensure
839                                 // we do not stall.
840                                 if (pending) {
841                                     readPending = true;
842                                     doBeginReadNow();
843                                 }
844                             } else if (rearm) {
845                                 // We need to rearm the multishot as otherwise we might miss some data.
846                                 doBeginReadNow();
847                             } else if (!readPending) {
848                                 // Cancel the multi-shot read now as the user did not signal that we want to keep
849                                 // reading while we handle the completion event.
850                                 cancelOutstandingReads(registration, numOutstandingReads);
851                             }
852                         }
853                     } else if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
854                         // The readComplete(...) was triggered because the previous read was cancelled.
855                         // In this case we we need to check if the user did signal the desire to read again
856                         // in the meantime. If this is the case we need to schedule the read to ensure
857                         // we do not stall.
858                         if (pending) {
859                             readPending = true;
860                             doBeginReadNow();
861                         }
862                     } else if (multishot && rearm) {
863                         // We need to rearm the multishot as otherwise we might miss some data.
864                         doBeginReadNow();
865                     }
866                 } finally {
867                     inReadComplete = false;
868                     socketIsEmpty = false;
869                 }
870             }
871         }
872 
873         /**
874          * Called once a read was completed.
875          */
876         protected abstract void readComplete0(byte op, int res, int flags, short data, int outstandingCompletes);
877 
878         /**
879          * Called once POLLRDHUP event is ready to be processed
880          */
881         private void pollRdHup(int res) {
882             ioState &= ~POLL_RDHUP_SCHEDULED;
883             pollRdhupId = 0;
884             if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
885                 return;
886             }
887 
888             // Mark that we received a POLLRDHUP and so need to continue reading until all the input ist drained.
889             recvBufAllocHandle().rdHupReceived();
890 
891             if (isActive()) {
892                 scheduleFirstReadIfNeeded();
893             } else {
894                 // Just to be safe make sure the input marked as closed.
895                 shutdownInput(false);
896             }
897         }
898 
899         /**
900          * Called once POLLIN event is ready to be processed
901          */
902         private void pollIn(int res, int flags, short data) {
903             // Check if we need to rearm. This works for both cases, POLL_ADD and POLL_ADD_MULTI.
904             boolean rearm = (flags & Native.IORING_CQE_F_MORE) == 0;
905             if (rearm) {
906                 ioState &= ~POLL_IN_SCHEDULED;
907                 pollInId = 0;
908             }
909             if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
910                 return;
911             }
912             if (!readPending) {
913                 // We received the POLLIN but the user is not interested yet in reading, just mark socketHasMoreData
914                 // as true so we will trigger a read directly once the user calls read()
915                 socketHasMoreData = true;
916                 return;
917             }
918             scheduleFirstReadIfNeeded();
919         }
920 
921         private void scheduleFirstReadIfNeeded() {
922             if ((ioState & READ_SCHEDULED) == 0) {
923                 scheduleFirstRead();
924             }
925         }
926 
927         private void scheduleFirstRead() {
928             // This is a new "read loop" so we need to reset the allocHandle.
929             final ChannelConfig config = config();
930             final IoUringRecvByteAllocatorHandle allocHandle = recvBufAllocHandle();
931             allocHandle.reset(config);
932             scheduleRead(true);
933         }
934 
935         protected final void scheduleRead(boolean first) {
936             // Only schedule another read if the fd is still open.
937             if (delayedClose == null && fd().isOpen() && (ioState & READ_SCHEDULED) == 0) {
938                 numOutstandingReads = (short) scheduleRead0(first, socketIsEmpty);
939                 if (numOutstandingReads > 0 || numOutstandingReads == -1) {
940                     ioState |= READ_SCHEDULED;
941                 }
942             }
943         }
944 
945         /**
946          * Schedule a read and returns the number of {@link #readComplete(byte, int, int, short)}
947          * calls that are expected because of the scheduled read.
948          *
949          * @param first             {@code true} if this is the first read of a read loop.
950          * @param socketIsEmpty     {@code true} if the socket is guaranteed to be empty, {@code false} otherwise.
951          * @return                  the number of {@link #readComplete(byte, int, int, short)} calls expected or
952          *                          {@code -1} if {@link #readComplete(byte, int, int, short)} is called until
953          *                          the read is cancelled (multi-shot).
954          */
955         protected abstract int scheduleRead0(boolean first, boolean socketIsEmpty);
956 
957         /**
958          * Called once POLLOUT event is ready to be processed
959          *
960          * @param res   the result.
961          */
962         private void pollOut(int res) {
963             ioState &= ~POLL_OUT_SCHEDULED;
964             pollOutId = 0;
965             if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
966                 return;
967             }
968             // pending connect
969             if (connectPromise != null) {
970                 // Note this method is invoked by the event loop only if the connection attempt was
971                 // neither cancelled nor timed out.
972 
973                 assert eventLoop().inEventLoop();
974 
975                 boolean connectStillInProgress = false;
976                 try {
977                     boolean wasActive = isActive();
978                     if (!socket.finishConnect()) {
979                         connectStillInProgress = true;
980                         return;
981                     }
982                     fulfillConnectPromise(connectPromise, wasActive);
983                 } catch (Throwable t) {
984                     fulfillConnectPromise(connectPromise, annotateConnectException(t, requestedRemoteAddress));
985                 } finally {
986                     if (!connectStillInProgress) {
987                         // Check for null as the connectTimeoutFuture is only created if a connectTimeoutMillis > 0
988                         // is used
989                         // See https://github.com/netty/netty/issues/1770
990                         cancelConnectTimeoutFuture();
991                         connectPromise = null;
992                     } else {
993                         // The connect was not done yet, register for POLLOUT again
994                         schedulePollOut();
995                     }
996                 }
997             } else if (!socket.isOutputShutdown()) {
998                 // Try writing again
999                 super.flush0();
1000             }
1001         }
1002 
1003         /**
1004          * Called once a write was completed.
1005          *
1006          * @param op    the op code.
1007          * @param res   the result.
1008          * @param flags the flags.
1009          * @param data  the data that was passed when submitting the op.
1010          */
1011         private void writeComplete(byte op, int res, int flags, long data) {
1012             writeTracker.complete(data, op, flags);
1013             if ((ioState & CONNECT_SCHEDULED) != 0) {
1014                 // The writeComplete(...) callback was called because of a sendmsg(...) result that was used for
1015                 // TCP_FASTOPEN_CONNECT.
1016                 freeMsgHdrArray();
1017                 if (res > 0) {
1018                     // Connect complete!
1019                     // The completion may arrive after close() or shutdownOutput() already dropped the
1020                     // outbound buffer, in which case there is nothing left to remove.
1021                     ChannelOutboundBuffer channelOutboundBuffer = outboundBuffer();
1022                     if (channelOutboundBuffer != null) {
1023                         channelOutboundBuffer.removeBytes(res);
1024                     }
1025 
1026                     // Explicit pass in 0 as this is returned by a connect(...) call when it was successful.
1027                     connectComplete(op, 0, flags, data);
1028                 } else if (res == ERRNO_EINPROGRESS_NEGATIVE || res == 0) {
1029                     // This happens when we (as a client) have no pre-existing cookie for doing a fast-open connection.
1030                     // In this case, our TCP connection will be established normally, but no data was transmitted at
1031                     // this time. We'll just transmit the data with normal writes later.
1032                     // Let's submit a normal connect.
1033                     submitConnect((InetSocketAddress) requestedRemoteAddress);
1034                 } else {
1035                     // There was an error, handle it as a normal connect error.
1036                     connectComplete(op, res, flags, data);
1037                 }
1038                 return;
1039             }
1040 
1041             if ((flags & Native.IORING_CQE_F_NOTIF) == 0) {
1042                 assert numOutstandingWrites > 0;
1043                 --numOutstandingWrites;
1044             }
1045 
1046             boolean writtenAll = writeComplete0(op, res, flags, data, numOutstandingWrites);
1047             if (!writtenAll && (ioState & POLL_OUT_SCHEDULED) == 0) {
1048 
1049                 // We were not able to write everything, let's register for POLLOUT
1050                 schedulePollOut();
1051             }
1052 
1053             // We only reset this once we are done with calling removeBytes(...) as otherwise we may trigger a write
1054             // while still removing messages internally in removeBytes(...) which then may corrupt state.
1055             if (numOutstandingWrites == 0) {
1056                 ioState &= ~WRITE_SCHEDULED;
1057 
1058                 // If we could write all and we did not schedule a pollout yet let us try to write again
1059                 if (writtenAll && (ioState & POLL_OUT_SCHEDULED) == 0) {
1060                     scheduleWriteIfNeeded(unsafe().outboundBuffer(), false);
1061                 }
1062             }
1063         }
1064 
1065         /**
1066          * Called once a write was completed.
1067          * @param op            the op code
1068          * @param res           the result.
1069          * @param flags         the flags.
1070          * @param data          the data that was passed when submitting the op.
1071          * @param outstanding   the outstanding write completions.
1072          */
1073         abstract boolean writeComplete0(byte op, int res, int flags, long data, int outstanding);
1074 
1075         /**
1076          * Called once a cancel was completed.
1077          *
1078          * @param op            the op code
1079          * @param res           the result.
1080          * @param flags         the flags.
1081          * @param data          the data that was passed when submitting the op.
1082          */
1083         void cancelComplete0(byte op, int res, int flags, short data) {
1084             // NOOP
1085         }
1086 
1087         /**
1088          * Called once a connect was completed.
1089          * @param op            the op code.
1090          * @param res           the result.
1091          * @param flags         the flags.
1092          * @param data          the data that was passed when submitting the op.
1093          */
1094         void connectComplete(byte op, int res, int flags, long data) {
1095             ioState &= ~CONNECT_SCHEDULED;
1096             freeRemoteAddressMemory();
1097 
1098             if (res == ERRNO_EINPROGRESS_NEGATIVE || res == ERROR_EALREADY_NEGATIVE) {
1099                 // connect not complete yet need to wait for poll_out event
1100                 schedulePollOut();
1101             } else {
1102                 try {
1103                     if (res == 0) {
1104                         fulfillConnectPromise(connectPromise, active);
1105                         if (readPending) {
1106                             doBeginReadNow();
1107                         }
1108                     } else {
1109                         try {
1110                             Errors.throwConnectException("io_uring connect", res);
1111                         } catch (Throwable cause) {
1112                             fulfillConnectPromise(connectPromise, cause);
1113                         }
1114                     }
1115                 } finally {
1116                     // Check for null as the connectTimeoutFuture is only created if a connectTimeoutMillis > 0 is
1117                     // used
1118                     // See https://github.com/netty/netty/issues/1770
1119                     cancelConnectTimeoutFuture();
1120                     connectPromise = null;
1121                 }
1122             }
1123         }
1124 
1125         @Override
1126         public void connect(
1127                 final SocketAddress remoteAddress, final SocketAddress localAddress, final ChannelPromise promise) {
1128             // Don't mark the connect promise as uncancellable as in fact we can cancel it as it is using
1129             // non-blocking io.
1130             if (promise.isDone() || !ensureOpen(promise)) {
1131                 return;
1132             }
1133 
1134             if (delayedClose != null) {
1135                 promise.tryFailure(annotateConnectException(new ClosedChannelException(), remoteAddress));
1136                 return;
1137             }
1138             try {
1139                 if (connectPromise != null) {
1140                     throw new ConnectionPendingException();
1141                 }
1142                 if (localAddress instanceof InetSocketAddress) {
1143                     checkResolvable((InetSocketAddress) localAddress);
1144                 }
1145 
1146                 if (remoteAddress instanceof InetSocketAddress) {
1147                     checkResolvable((InetSocketAddress) remoteAddress);
1148                 }
1149 
1150                 if (remote != null) {
1151                     // Check if already connected before trying to connect. This is needed as connect(...) will not#
1152                     // return -1 and set errno to EISCONN if a previous connect(...) attempt was setting errno to
1153                     // EINPROGRESS and finished later.
1154                     throw new AlreadyConnectedException();
1155                 }
1156 
1157                 if (localAddress != null) {
1158                     socket.bind(localAddress);
1159                 }
1160 
1161                 if (remoteAddress instanceof InetSocketAddress) {
1162                     InetSocketAddress inetSocketAddress = (InetSocketAddress) remoteAddress;
1163                     ByteBuf initialData = null;
1164                     if (IoUring.isTcpFastOpenClientSideAvailable() &&
1165                         config().getOption(ChannelOption.TCP_FASTOPEN_CONNECT) == Boolean.TRUE) {
1166                         ChannelOutboundBuffer outbound = unsafe().outboundBuffer();
1167                         outbound.addFlush();
1168                         Object curr;
1169                         if ((curr = outbound.current()) instanceof ByteBuf) {
1170                             initialData = (ByteBuf) curr;
1171                         }
1172                     }
1173                     if (initialData != null) {
1174                         msgHdrMemoryArray = new MsgHdrMemoryArray((short) 1);
1175                         MsgHdrMemory hdr = msgHdrMemoryArray.hdr(0);
1176                         fillTFOInitData(hdr, inetSocketAddress, initialData);
1177 
1178                         int fd = fd().intValue();
1179                         IoRegistration registration = registration();
1180                         short opsId = writeTracker.nextId();
1181                         if (opsId == 0) {
1182                             freeMsgHdrArray();
1183                             submitConnect(inetSocketAddress);
1184                         } else {
1185                             IoUringIoOps ops = IoUringIoOps.newSendmsg(fd, (byte) 0, Native.MSG_FASTOPEN,
1186                                     hdr.address(), opsId);
1187                             writeTracker.record(opsId, ops.opcode(), initialData);
1188                             connectId = registration.submit(ops);
1189                             if (connectId == 0) {
1190                                 writeTracker.abandon(opsId, ops.opcode());
1191                                 // Directly release the memory if submitting failed.
1192                                 freeMsgHdrArray();
1193                             }
1194                         }
1195                     } else {
1196                         submitConnect(inetSocketAddress);
1197                     }
1198                 } else if (remoteAddress instanceof DomainSocketAddress) {
1199                     DomainSocketAddress unixDomainSocketAddress = (DomainSocketAddress) remoteAddress;
1200                     submitConnect(unixDomainSocketAddress);
1201                 } else {
1202                     throw new Error("Unexpected SocketAddress implementation " + className(remoteAddress));
1203                 }
1204 
1205                 if (connectId != 0) {
1206                     ioState |= CONNECT_SCHEDULED;
1207                 }
1208             } catch (Throwable t) {
1209                 closeIfClosed();
1210                 promise.tryFailure(annotateConnectException(t, remoteAddress));
1211                 return;
1212             }
1213             connectPromise = promise;
1214             requestedRemoteAddress = remoteAddress;
1215             // Schedule connect timeout.
1216             int connectTimeoutMillis = config().getConnectTimeoutMillis();
1217             if (connectTimeoutMillis > 0) {
1218                 connectTimeoutFuture = eventLoop().schedule(new Runnable() {
1219                     @Override
1220                     public void run() {
1221                         ChannelPromise connectPromise = AbstractIoUringChannel.this.connectPromise;
1222                         if (connectPromise != null && !connectPromise.isDone() &&
1223                                 connectPromise.tryFailure(new ConnectTimeoutException(
1224                                         "connection timed out: " + remoteAddress))) {
1225                             close(voidPromise());
1226                         }
1227                     }
1228                 }, connectTimeoutMillis, TimeUnit.MILLISECONDS);
1229             }
1230 
1231             promise.addListener(new ChannelFutureListener() {
1232                 @Override
1233                 public void operationComplete(ChannelFuture future) {
1234                     // If the connect future is cancelled we also cancel the timeout and close the
1235                     // underlying socket.
1236                     if (future.isCancelled()) {
1237                         cancelConnectTimeoutFuture();
1238                         connectPromise = null;
1239                         close(voidPromise());
1240                     }
1241                 }
1242             });
1243         }
1244 
1245         private void fillTFOInitData(MsgHdrMemory hdr, InetSocketAddress inetSocketAddress,
1246                                      ByteBuf initialData) throws Exception {
1247            if (initialData.hasMemoryAddress()) {
1248                hdr.set(socket, inetSocketAddress,
1249                        initialData.memoryAddress() + initialData.readerIndex(),
1250                        initialData.readableBytes(), (short) 0);
1251            } else {
1252                // Use an iovec array for CompositeByteBuf and other buffers without a memory address.
1253                // If the shared IovArray has not enough space, the rest is sent after connect.
1254                IoUringIoHandler handler = registration().attachment();
1255                IovArray iovArray = handler.iovArray();
1256                int iovOffset = iovArray.count();
1257                iovArray.processMessage(initialData);
1258                long iovArrayAddress = iovArray.memoryAddress(iovOffset);
1259                int iovArrayLength = iovArray.count() - iovOffset;
1260                hdr.setWithIovArrayAddress(socket, inetSocketAddress, iovArrayAddress, iovArrayLength, (short) 0);
1261            }
1262         }
1263 
1264         @Override
1265         public final void deregister(ChannelPromise promise) {
1266             if (deregisterPromise != null) {
1267                 // A deregistration is already in progress.
1268                 PromiseNotifier.cascade(deregisterPromise, promise);
1269             } else if (!isRegistered()) {
1270                 promise.setSuccess();
1271             } else {
1272                 // We need to store a reference to the original promise as we should only notify it once we
1273                 // have handles all pending completions.
1274                 deregisterPromise = promise;
1275                 super.deregister(newPromise().addListener(f -> {
1276                     if (!f.isSuccess()) {
1277                         this.deregisterPromise = null;
1278                         promise.setFailure(f.cause());
1279                     }
1280                 }));
1281             }
1282         }
1283     }
1284 
1285     private void submitConnect(InetSocketAddress inetSocketAddress) {
1286         cleanable = Buffer.allocateDirectBufferWithNativeOrder(Native.SIZEOF_SOCKADDR_STORAGE);
1287         remoteAddressMemory = cleanable.buffer();
1288 
1289         SockaddrIn.set(socket.isIpv6(), remoteAddressMemory, inetSocketAddress);
1290 
1291         int fd = fd().intValue();
1292         IoRegistration registration = registration();
1293         IoUringIoOps ops = IoUringIoOps.newConnect(
1294                 fd, (byte) 0, Buffer.memoryAddress(remoteAddressMemory), nextOpsId());
1295         connectId = registration.submit(ops);
1296         if (connectId == 0) {
1297             // Directly release the memory if submitting failed.
1298             freeRemoteAddressMemory();
1299         }
1300     }
1301 
1302     private void submitConnect(DomainSocketAddress unixDomainSocketAddress) {
1303         cleanable = Buffer.allocateDirectBufferWithNativeOrder(Native.SIZEOF_SOCKADDR_UN);
1304         remoteAddressMemory = cleanable.buffer();
1305         int addrLen = SockaddrIn.setUds(remoteAddressMemory, unixDomainSocketAddress);
1306         int fd = fd().intValue();
1307         IoRegistration registration = registration();
1308         long addr = Buffer.memoryAddress(remoteAddressMemory);
1309         IoUringIoOps ops = IoUringIoOps.newConnect(fd, (byte) 0, addr, addrLen, nextOpsId());
1310         connectId = registration.submit(ops);
1311         if (connectId == 0) {
1312             // Directly release the memory if submitting failed.
1313             freeRemoteAddressMemory();
1314         }
1315     }
1316 
1317     @Override
1318     protected Object filterOutboundMessage(Object msg) {
1319         if (msg instanceof ByteBuf) {
1320             ByteBuf buf = (ByteBuf) msg;
1321             return UnixChannelUtil.isBufferCopyNeededForWrite(buf)? newDirectBuffer(buf) : buf;
1322         }
1323         throw new UnsupportedOperationException("unsupported message type: " + StringUtil.simpleClassName(msg));
1324     }
1325 
1326     @Override
1327     protected void doRegister(ChannelPromise promise) {
1328         IoEventLoop eventLoop = (IoEventLoop) eventLoop();
1329         eventLoop.register(ioUringUnsafe()).addListener(f -> {
1330             if (f.isSuccess()) {
1331                 registration = (IoRegistration) f.getNow();
1332                 promise.setSuccess();
1333             } else {
1334                 promise.setFailure(f.cause());
1335             }
1336         });
1337     }
1338 
1339     @Override
1340     protected final void doDeregister() {
1341         // Cancel all previous submitted ops.
1342         if (!ioUringUnsafe().cancelOps(connectPromise != null)) {
1343             // It's possible that we never registered anything and so we did not submit any ASYNC_CANCEL.
1344             // In this case directly call cancel as we will not receive any completion at all.
1345             if (registration != null) {
1346                 registration.cancel();
1347             }
1348         }
1349     }
1350 
1351     @Override
1352     protected void doBind(final SocketAddress local) throws Exception {
1353         if (local instanceof InetSocketAddress) {
1354             checkResolvable((InetSocketAddress) local);
1355         }
1356         socket.bind(local);
1357         this.local = socket.localAddress();
1358     }
1359 
1360     protected static void checkResolvable(InetSocketAddress addr) {
1361         if (addr.isUnresolved()) {
1362             throw new UnresolvedAddressException();
1363         }
1364     }
1365 
1366     @Override
1367     protected final SocketAddress localAddress0() {
1368         return local;
1369     }
1370 
1371     @Override
1372     protected final SocketAddress remoteAddress0() {
1373         return remote;
1374     }
1375 
1376     private static boolean isAllowHalfClosure(ChannelConfig config) {
1377         return config instanceof SocketChannelConfig &&
1378                ((SocketChannelConfig) config).isAllowHalfClosure();
1379     }
1380 
1381     private void cancelConnectTimeoutFuture() {
1382         if (connectTimeoutFuture != null) {
1383             connectTimeoutFuture.cancel(false);
1384             connectTimeoutFuture = null;
1385         }
1386     }
1387 
1388     private void computeRemote() {
1389         if (requestedRemoteAddress instanceof InetSocketAddress) {
1390             remote = computeRemoteAddr((InetSocketAddress) requestedRemoteAddress, socket.remoteAddress());
1391         }
1392     }
1393 
1394     private boolean shouldBreakIoUringInReady(ChannelConfig config) {
1395         return socket.isInputShutdown() && (inputClosedSeenErrorOnRead || !isAllowHalfClosure(config));
1396     }
1397 
1398     /**
1399      * Return if the socket is guaranteed to be empty when the submitted io was executed and the completion event be
1400      * created.
1401      * @param flags     the flags that were part of the completion
1402      * @return          {@code true} if empty.
1403      */
1404     protected abstract boolean socketIsEmpty(int flags);
1405 
1406     abstract boolean isPollInFirst();
1407 }