1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package io.netty.channel.uring;
17
18 import io.netty.buffer.ByteBuf;
19 import io.netty.buffer.ByteBufAllocator;
20 import io.netty.buffer.ByteBufHolder;
21 import io.netty.buffer.ByteBufUtil;
22 import io.netty.buffer.Unpooled;
23 import io.netty.channel.AbstractChannel;
24 import io.netty.channel.Channel;
25 import io.netty.channel.ChannelConfig;
26 import io.netty.channel.ChannelFuture;
27 import io.netty.channel.ChannelFutureListener;
28 import io.netty.channel.ChannelOption;
29 import io.netty.channel.ChannelOutboundBuffer;
30 import io.netty.channel.ChannelPromise;
31 import io.netty.channel.ConnectTimeoutException;
32 import io.netty.channel.EventLoop;
33 import io.netty.channel.IoEvent;
34 import io.netty.channel.IoEventLoop;
35 import io.netty.channel.IoRegistration;
36 import io.netty.channel.RecvByteBufAllocator;
37 import io.netty.channel.ServerChannel;
38 import io.netty.channel.socket.ChannelInputShutdownEvent;
39 import io.netty.channel.socket.ChannelInputShutdownReadComplete;
40 import io.netty.channel.socket.SocketChannelConfig;
41 import io.netty.channel.unix.Buffer;
42 import io.netty.channel.unix.DomainSocketAddress;
43 import io.netty.channel.unix.Errors;
44 import io.netty.channel.unix.FileDescriptor;
45 import io.netty.channel.unix.IovArray;
46 import io.netty.channel.unix.UnixChannel;
47 import io.netty.channel.unix.UnixChannelUtil;
48 import io.netty.util.ReferenceCountUtil;
49 import io.netty.util.concurrent.PromiseNotifier;
50 import io.netty.util.internal.CleanableDirectBuffer;
51 import io.netty.util.internal.StringUtil;
52 import io.netty.util.internal.logging.InternalLogger;
53 import io.netty.util.internal.logging.InternalLoggerFactory;
54
55 import java.io.IOException;
56 import java.net.InetSocketAddress;
57 import java.net.SocketAddress;
58 import java.nio.ByteBuffer;
59 import java.nio.channels.AlreadyConnectedException;
60 import java.nio.channels.ClosedChannelException;
61 import java.nio.channels.ConnectionPendingException;
62 import java.nio.channels.NotYetConnectedException;
63 import java.nio.channels.UnresolvedAddressException;
64 import java.util.concurrent.ScheduledFuture;
65 import java.util.concurrent.TimeUnit;
66
67 import static io.netty.channel.unix.Errors.ERRNO_EINPROGRESS_NEGATIVE;
68 import static io.netty.channel.unix.Errors.ERROR_EALREADY_NEGATIVE;
69 import static io.netty.channel.unix.UnixChannelUtil.computeRemoteAddr;
70 import static io.netty.util.internal.ObjectUtil.checkNotNull;
71 import static io.netty.util.internal.StringUtil.className;
72
73
74 abstract class AbstractIoUringChannel extends AbstractChannel implements UnixChannel {
75 private static final InternalLogger logger = InternalLoggerFactory.getInstance(AbstractIoUringChannel.class);
76 final LinuxSocket socket;
77
78
79 final WriteOperationTracker writeTracker = new WriteOperationTracker();
80 protected volatile boolean active;
81
82
83 private static final int POLL_IN_SCHEDULED = 1;
84 private static final int POLL_OUT_SCHEDULED = 1 << 2;
85 private static final int POLL_RDHUP_SCHEDULED = 1 << 3;
86 private static final int WRITE_SCHEDULED = 1 << 4;
87 private static final int READ_SCHEDULED = 1 << 5;
88 private static final int CONNECT_SCHEDULED = 1 << 6;
89
90 private short opsId = Short.MIN_VALUE;
91
92 private long pollInId;
93 private long pollOutId;
94 private long pollRdhupId;
95 private long connectId;
96
97
98 private byte ioState;
99
100
101
102
103 private short numOutstandingWrites;
104
105 private short numOutstandingReads;
106
107 private boolean readPending;
108 private boolean inReadComplete;
109 private boolean socketHasMoreData;
110
111 private static final class DelayedClose {
112 private final ChannelPromise promise;
113 private final Throwable cause;
114 private final ClosedChannelException closeCause;
115
116 DelayedClose(ChannelPromise promise, Throwable cause, ClosedChannelException closeCause) {
117 this.promise = promise;
118 this.cause = cause;
119 this.closeCause = closeCause;
120 }
121 }
122 private DelayedClose delayedClose;
123 private boolean inputClosedSeenErrorOnRead;
124
125
126
127
128 private ChannelPromise connectPromise;
129 private ScheduledFuture<?> connectTimeoutFuture;
130 private SocketAddress requestedRemoteAddress;
131 private CleanableDirectBuffer cleanable;
132 private ByteBuffer remoteAddressMemory;
133 private MsgHdrMemoryArray msgHdrMemoryArray;
134
135 private IoRegistration registration;
136
137 private volatile SocketAddress local;
138 private volatile SocketAddress remote;
139
140 AbstractIoUringChannel(final Channel parent, LinuxSocket socket, boolean active) {
141 super(parent);
142 this.socket = checkNotNull(socket, "fd");
143
144 if (active) {
145
146
147 this.active = true;
148 this.local = socket.localAddress();
149 this.remote = socket.remoteAddress();
150 }
151
152 logger.trace("Create {} Socket: {}", this instanceof ServerChannel ? "Server" : "Channel", socket.intValue());
153 }
154
155 AbstractIoUringChannel(Channel parent, LinuxSocket fd, SocketAddress remote) {
156 super(parent);
157 this.socket = checkNotNull(fd, "fd");
158 this.active = true;
159
160
161
162 this.remote = remote;
163 this.local = fd.localAddress();
164 }
165
166
167 final void autoReadCleared() {
168 if (!isRegistered()) {
169 return;
170 }
171 IoRegistration registration = this.registration;
172 if (registration == null || !registration.isValid()) {
173 return;
174 }
175 if (eventLoop().inEventLoop()) {
176 clearRead();
177 } else {
178 eventLoop().execute(this::clearRead);
179 }
180 }
181
182 private void clearRead() {
183 assert eventLoop().inEventLoop();
184 readPending = false;
185 IoRegistration registration = this.registration;
186 if (registration == null || !registration.isValid()) {
187 return;
188 }
189
190 cancelOutstandingReads(registration(), numOutstandingReads);
191 }
192
193
194
195
196
197
198 protected final short nextOpsId() {
199 short id = opsId++;
200
201
202 if (id == 0) {
203 id = opsId++;
204 }
205 return id;
206 }
207
208 public final boolean isOpen() {
209 return socket.isOpen();
210 }
211
212 @Override
213 public boolean isActive() {
214 return active;
215 }
216
217 @Override
218 public final FileDescriptor fd() {
219 return socket;
220 }
221
222 private AbstractUringUnsafe ioUringUnsafe() {
223 return (AbstractUringUnsafe) unsafe();
224 }
225
226 @Override
227 protected boolean isCompatible(final EventLoop loop) {
228 return loop instanceof IoEventLoop && ((IoEventLoop) loop).isCompatible(AbstractUringUnsafe.class);
229 }
230
231 protected final ByteBuf newDirectBuffer(ByteBuf buf) {
232 return newDirectBuffer(buf, buf);
233 }
234
235 protected boolean allowMultiShotPollIn() {
236 return IoUring.isPollAddMultishotEnabled();
237 }
238
239 protected final ByteBuf newDirectBuffer(Object holder, ByteBuf buf) {
240 final int readableBytes = buf.readableBytes();
241 if (readableBytes == 0) {
242 ReferenceCountUtil.release(holder);
243 return Unpooled.EMPTY_BUFFER;
244 }
245
246 final ByteBufAllocator alloc = alloc();
247 if (alloc.isDirectBufferPooled()) {
248 return newDirectBuffer0(holder, buf, alloc, readableBytes);
249 }
250
251 final ByteBuf directBuf = ByteBufUtil.threadLocalDirectBuffer();
252 if (directBuf == null) {
253 return newDirectBuffer0(holder, buf, alloc, readableBytes);
254 }
255
256 directBuf.writeBytes(buf, buf.readerIndex(), readableBytes);
257 ReferenceCountUtil.safeRelease(holder);
258 return directBuf;
259 }
260
261 private static ByteBuf newDirectBuffer0(Object holder, ByteBuf buf, ByteBufAllocator alloc, int capacity) {
262 final ByteBuf directBuf = alloc.directBuffer(capacity);
263 directBuf.writeBytes(buf, buf.readerIndex(), capacity);
264 ReferenceCountUtil.safeRelease(holder);
265 return directBuf;
266 }
267
268
269
270
271
272
273
274 protected abstract void cancelOutstandingReads(IoRegistration registration, int numOutstandingReads);
275
276
277
278
279
280
281
282 protected abstract void cancelOutstandingWrites(IoRegistration registration, int numOutstandingWrites);
283
284 @Override
285 protected void doDisconnect() throws Exception {
286 }
287
288 private void freeRemoteAddressMemory() {
289 if (remoteAddressMemory != null) {
290 cleanable.clean();
291 cleanable = null;
292 remoteAddressMemory = null;
293 }
294 }
295
296 private void freeMsgHdrArray() {
297 if (msgHdrMemoryArray != null) {
298 msgHdrMemoryArray.release();
299 msgHdrMemoryArray = null;
300 }
301 }
302
303 @Override
304 protected void doClose() throws Exception {
305 active = false;
306
307 if (registration != null) {
308 if (socket.markClosed()) {
309 int fd = fd().intValue();
310 IoUringIoOps ops = IoUringIoOps.newClose(fd, (byte) 0, nextOpsId());
311 registration.submit(ops);
312 }
313 } else {
314
315 socket.close();
316 ioUringUnsafe().unregistered();
317 }
318 }
319
320
321
322
323
324
325 @Override
326 protected final void doShutdownOutput() throws Exception {
327 writeTracker.retainAll();
328 doShutdownOutput0();
329 }
330
331
332
333
334 protected void doShutdownOutput0() throws Exception {
335 super.doShutdownOutput();
336 }
337
338 @Override
339 protected final void doBeginRead() {
340 if (inputClosedSeenErrorOnRead) {
341
342 return;
343 }
344 if (readPending) {
345
346 return;
347 }
348 readPending = true;
349 if (inReadComplete || !isActive()) {
350
351
352
353 return;
354 }
355 doBeginReadNow();
356 }
357
358 private void doBeginReadNow() {
359 if (inputClosedSeenErrorOnRead) {
360
361 return;
362 }
363 if (!isPollInFirst() ||
364
365
366 socketHasMoreData) {
367
368 ioUringUnsafe().scheduleFirstReadIfNeeded();
369 } else if ((ioState & POLL_IN_SCHEDULED) == 0) {
370 ioUringUnsafe().schedulePollIn();
371 }
372 }
373
374 @Override
375 protected void doWrite(ChannelOutboundBuffer in) {
376 scheduleWriteIfNeeded(in, true);
377 }
378
379 protected void scheduleWriteIfNeeded(ChannelOutboundBuffer in, boolean submitAndRunNow) {
380 if ((ioState & WRITE_SCHEDULED) != 0) {
381 return;
382 }
383 if (scheduleWrite(in) > 0) {
384 ioState |= WRITE_SCHEDULED;
385 if (submitAndRunNow && !isWritable()) {
386 submitAndRunNow();
387 }
388 }
389 }
390
391 protected void submitAndRunNow() {
392
393 }
394
395 private int scheduleWrite(ChannelOutboundBuffer in) {
396 if (delayedClose != null || numOutstandingWrites == Short.MAX_VALUE) {
397 return 0;
398 }
399 if (in == null) {
400 return 0;
401 }
402
403 int msgCount = in.size();
404 if (msgCount == 0) {
405 return 0;
406 }
407 Object msg = in.current();
408
409 int scheduled;
410 if (msgCount > 1 && in.current() instanceof ByteBuf) {
411 scheduled = ioUringUnsafe().scheduleWriteMultiple(in);
412 } else if (msg instanceof ByteBuf && ((ByteBuf) msg).nioBufferCount() > 1 ||
413 (msg instanceof ByteBufHolder && ((ByteBufHolder) msg).content().nioBufferCount() > 1)) {
414
415 scheduled = ioUringUnsafe().scheduleWriteMultiple(in);
416 } else {
417 scheduled = ioUringUnsafe().scheduleWriteSingle(msg);
418 }
419
420
421
422
423
424 assert scheduled <= Short.MAX_VALUE;
425 numOutstandingWrites = (short) scheduled;
426 return numOutstandingWrites;
427 }
428
429 protected final IoRegistration registration() {
430 assert registration != null;
431 return registration;
432 }
433
434 private void schedulePollOut() {
435 pollOutId = schedulePollAdd(POLL_OUT_SCHEDULED, Native.POLLOUT, false);
436 }
437
438 final void schedulePollRdHup() {
439 pollRdhupId = schedulePollAdd(POLL_RDHUP_SCHEDULED, Native.POLLRDHUP, false);
440 }
441
442 protected abstract boolean isStreamSocket();
443
444 private long schedulePollAdd(int ioMask, int mask, boolean multishot) {
445 assert (ioState & ioMask) == 0;
446 int fd = fd().intValue();
447 IoRegistration registration = registration();
448 IoUringIoOps ops = IoUringIoOps.newPollAdd(
449 fd, (byte) 0, mask, multishot ? Native.IORING_POLL_ADD_MULTI : 0, nextOpsId());
450 long id = registration.submit(ops);
451 if (id != 0) {
452 ioState |= (byte) ioMask;
453 }
454 return id;
455 }
456
457 final void resetCachedAddresses() {
458 local = socket.localAddress();
459 remote = socket.remoteAddress();
460 }
461
462
463
464
465
466 private static short narrowUserData(long userData) {
467 assert userData == (short) userData : "user_data does not fit a short: " + userData;
468 return (short) userData;
469 }
470
471 protected abstract class AbstractUringUnsafe extends AbstractUnsafe implements IoUringIoHandle {
472 private IoUringRecvByteAllocatorHandle allocHandle;
473 private boolean closed;
474 private boolean socketIsEmpty;
475 private ChannelPromise deregisterPromise;
476
477
478
479
480
481 protected abstract int scheduleWriteMultiple(ChannelOutboundBuffer in);
482
483
484
485
486
487 protected abstract int scheduleWriteSingle(Object msg);
488
489 @Override
490 public final void handle(IoRegistration registration, IoEvent ioEvent) {
491 IoUringIoEvent event = (IoUringIoEvent) ioEvent;
492 byte op = event.opcode();
493 int res = event.res();
494 int flags = event.flags();
495 long userData = event.userData();
496 switch (op) {
497 case Native.IORING_OP_RECV:
498 case Native.IORING_OP_ACCEPT:
499 case Native.IORING_OP_RECVMSG:
500 case Native.IORING_OP_READ:
501 readComplete(op, res, flags, narrowUserData(userData));
502 break;
503 case Native.IORING_OP_WRITEV:
504 case Native.IORING_OP_SEND:
505 case Native.IORING_OP_SENDMSG:
506 case Native.IORING_OP_WRITE:
507 case Native.IORING_OP_SPLICE:
508 case Native.IORING_OP_SEND_ZC:
509 case Native.IORING_OP_SENDMSG_ZC:
510 writeComplete(op, res, flags, userData);
511 break;
512 case Native.IORING_OP_POLL_ADD:
513 pollAddComplete(res, flags, narrowUserData(userData));
514 break;
515 case Native.IORING_OP_ASYNC_CANCEL:
516 cancelComplete0(op, res, flags, narrowUserData(userData));
517 break;
518 case Native.IORING_OP_CONNECT:
519 connectComplete(op, res, flags, userData);
520
521
522 freeMsgHdrArray();
523 freeRemoteAddressMemory();
524 break;
525 case Native.IORING_OP_CLOSE:
526 if (res != Native.ERRNO_ECANCELED_NEGATIVE) {
527 if (delayedClose != null) {
528 delayedClose.promise.setSuccess();
529 }
530 closed = true;
531 }
532 break;
533 default:
534 break;
535 }
536
537
538
539 handleDelayedClosed();
540
541 if (ioState == 0 && (closed || !isRegistered())) {
542
543 registration.cancel();
544 }
545 }
546
547 @Override
548 public void unregistered() {
549 freeMsgHdrArray();
550 freeRemoteAddressMemory();
551 writeTracker.releaseAll();
552
553
554 if (deregisterPromise != null) {
555 ChannelPromise promise = deregisterPromise;
556 deregisterPromise = null;
557 promise.setSuccess();
558 }
559 }
560
561 private void handleDelayedClosed() {
562 if (delayedClose != null && canCloseNow()) {
563 closeNow();
564 }
565 }
566
567 private void pollAddComplete(int res, int flags, short data) {
568 if ((res & Native.POLLOUT) != 0) {
569 pollOut(res);
570 }
571 if ((res & Native.POLLIN) != 0) {
572 pollIn(res, flags, data);
573 }
574 if ((res & Native.POLLRDHUP) != 0) {
575 pollRdHup(res);
576 }
577 }
578
579 @Override
580 public final void close() throws Exception {
581 close(voidPromise());
582 }
583
584 @Override
585 protected void close(ChannelPromise promise, Throwable cause, ClosedChannelException closeCause) {
586 if (closeFuture().isDone()) {
587
588 safeSetSuccess(promise);
589 return;
590 }
591 if (delayedClose == null) {
592
593
594
595 delayedClose = new DelayedClose(promise.isVoid() ? newPromise() : promise, cause, closeCause);
596 } else {
597 delayedClose.promise.addListener(new PromiseNotifier<>(false, promise));
598 return;
599 }
600
601 boolean cancelConnect = false;
602 try {
603 ChannelPromise connectPromise = AbstractIoUringChannel.this.connectPromise;
604 if (connectPromise != null) {
605
606 connectPromise.tryFailure(new ClosedChannelException());
607 AbstractIoUringChannel.this.connectPromise = null;
608 cancelConnect = true;
609 }
610
611 cancelConnectTimeoutFuture();
612 } finally {
613
614
615 cancelOps(cancelConnect);
616 }
617
618 if (canCloseNow()) {
619
620 closeNow();
621 }
622 }
623
624 private boolean cancelOps(boolean cancelConnect) {
625 if (registration == null || !registration.isValid()) {
626 return false;
627 }
628 boolean cancelled = false;
629 byte flags = (byte) 0;
630 if ((ioState & POLL_RDHUP_SCHEDULED) != 0 && pollRdhupId != 0) {
631 long id = registration.submit(
632 IoUringIoOps.newAsyncCancel(flags, pollRdhupId, Native.IORING_OP_POLL_ADD));
633 assert id != 0;
634 pollRdhupId = 0;
635 cancelled = true;
636 }
637 if ((ioState & POLL_IN_SCHEDULED) != 0 && pollInId != 0) {
638 long id = registration.submit(
639 IoUringIoOps.newAsyncCancel(flags, pollInId, Native.IORING_OP_POLL_ADD));
640 assert id != 0;
641 pollInId = 0;
642 cancelled = true;
643 }
644 if ((ioState & POLL_OUT_SCHEDULED) != 0 && pollOutId != 0) {
645 long id = registration.submit(
646 IoUringIoOps.newAsyncCancel(flags, pollOutId, Native.IORING_OP_POLL_ADD));
647 assert id != 0;
648 pollOutId = 0;
649 cancelled = true;
650 }
651 if (cancelConnect && connectId != 0) {
652
653 long id = registration.submit(IoUringIoOps.newAsyncCancel(flags, connectId, Native.IORING_OP_CONNECT));
654 assert id != 0;
655 connectId = 0;
656 cancelled = true;
657 }
658 if (numOutstandingReads != 0 || numOutstandingWrites != 0) {
659 cancelled = true;
660 }
661 cancelOutstandingReads(registration, numOutstandingReads);
662 cancelOutstandingWrites(registration, numOutstandingWrites);
663 return cancelled;
664 }
665
666 private boolean canCloseNow() {
667
668
669 return canCloseNow0() && (ioState & (WRITE_SCHEDULED | READ_SCHEDULED)) == 0;
670 }
671
672 protected boolean canCloseNow0() {
673 return true;
674 }
675
676 private void closeNow() {
677 super.close(newPromise(), delayedClose.cause, delayedClose.closeCause);
678 }
679
680 @Override
681 protected final void flush0() {
682
683
684
685 if ((ioState & POLL_OUT_SCHEDULED) == 0) {
686 super.flush0();
687 }
688 }
689
690 private void fulfillConnectPromise(ChannelPromise promise, Throwable cause) {
691 if (promise == null) {
692
693 return;
694 }
695
696
697 promise.tryFailure(cause);
698 closeIfClosed();
699 }
700
701 private void fulfillConnectPromise(ChannelPromise promise, boolean wasActive) {
702 if (promise == null) {
703
704 return;
705 }
706 active = true;
707
708 if (local == null) {
709 local = socket.localAddress();
710 }
711 computeRemote();
712
713 if (isStreamSocket()) {
714
715 schedulePollRdHup();
716 }
717
718
719
720 boolean active = isActive();
721
722
723 boolean promiseSet = promise.trySuccess();
724
725
726
727 if (!wasActive && active) {
728 pipeline().fireChannelActive();
729 }
730
731
732 if (!promiseSet) {
733 close(voidPromise());
734 }
735 }
736
737 @Override
738 public final IoUringRecvByteAllocatorHandle recvBufAllocHandle() {
739 if (allocHandle == null) {
740 allocHandle = new IoUringRecvByteAllocatorHandle(
741 (RecvByteBufAllocator.ExtendedHandle) super.recvBufAllocHandle());
742 }
743 return allocHandle;
744 }
745
746 final void shutdownInput(boolean allDataRead) {
747 logger.trace("shutdownInput Fd: {}", fd().intValue());
748 if (!socket.isInputShutdown()) {
749 if (isAllowHalfClosure(config())) {
750 try {
751 socket.shutdown(true, false);
752 } catch (IOException ignored) {
753
754
755 fireEventAndClose(ChannelInputShutdownEvent.INSTANCE);
756 return;
757 } catch (NotYetConnectedException ignore) {
758
759
760 }
761 pipeline().fireUserEventTriggered(ChannelInputShutdownEvent.INSTANCE);
762 } else {
763
764 inputClosedSeenErrorOnRead = true;
765 close(voidPromise());
766 return;
767 }
768 }
769 if (allDataRead && !inputClosedSeenErrorOnRead) {
770 inputClosedSeenErrorOnRead = true;
771 pipeline().fireUserEventTriggered(ChannelInputShutdownReadComplete.INSTANCE);
772 }
773 }
774
775 private void fireEventAndClose(Object evt) {
776 pipeline().fireUserEventTriggered(evt);
777 close(voidPromise());
778 }
779
780 final void schedulePollIn() {
781 assert (ioState & POLL_IN_SCHEDULED) == 0;
782 if (!isActive() || shouldBreakIoUringInReady(config())) {
783 return;
784 }
785 pollInId = schedulePollAdd(POLL_IN_SCHEDULED, Native.POLLIN, allowMultiShotPollIn());
786 }
787
788 protected final boolean isReadMultishot() {
789 return numOutstandingReads == -1;
790 }
791
792 private void readComplete(byte op, int res, int flags, short data) {
793 assert numOutstandingReads > 0 || numOutstandingReads == -1 : numOutstandingReads;
794
795 boolean multishot = isReadMultishot();
796 boolean rearm = (flags & Native.IORING_CQE_F_MORE) == 0;
797 boolean pending = readPending;
798 if (multishot) {
799 if (rearm) {
800
801 ioState &= ~READ_SCHEDULED;
802 }
803
804
805 readPending = false;
806 } else if (--numOutstandingReads == 0) {
807
808 readPending = false;
809 ioState &= ~READ_SCHEDULED;
810 }
811 inReadComplete = true;
812 try {
813 socketIsEmpty = socketIsEmpty(flags);
814 socketHasMoreData = IoUring.isCqeFSockNonEmptySupported() &&
815 (flags & Native.IORING_CQE_F_SOCK_NONEMPTY) != 0;
816 readComplete0(op, res, flags, data, numOutstandingReads);
817 } finally {
818 try {
819
820 if (recvBufAllocHandle().isReadComplete()) {
821
822 recvBufAllocHandle().reset(config());
823
824
825 if (!multishot) {
826 if (readPending) {
827
828
829 doBeginReadNow();
830 }
831 } else {
832
833
834
835 if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
836
837
838
839
840 if (pending) {
841 readPending = true;
842 doBeginReadNow();
843 }
844 } else if (rearm) {
845
846 doBeginReadNow();
847 } else if (!readPending) {
848
849
850 cancelOutstandingReads(registration, numOutstandingReads);
851 }
852 }
853 } else if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
854
855
856
857
858 if (pending) {
859 readPending = true;
860 doBeginReadNow();
861 }
862 } else if (multishot && rearm) {
863
864 doBeginReadNow();
865 }
866 } finally {
867 inReadComplete = false;
868 socketIsEmpty = false;
869 }
870 }
871 }
872
873
874
875
876 protected abstract void readComplete0(byte op, int res, int flags, short data, int outstandingCompletes);
877
878
879
880
881 private void pollRdHup(int res) {
882 ioState &= ~POLL_RDHUP_SCHEDULED;
883 pollRdhupId = 0;
884 if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
885 return;
886 }
887
888
889 recvBufAllocHandle().rdHupReceived();
890
891 if (isActive()) {
892 scheduleFirstReadIfNeeded();
893 } else {
894
895 shutdownInput(false);
896 }
897 }
898
899
900
901
902 private void pollIn(int res, int flags, short data) {
903
904 boolean rearm = (flags & Native.IORING_CQE_F_MORE) == 0;
905 if (rearm) {
906 ioState &= ~POLL_IN_SCHEDULED;
907 pollInId = 0;
908 }
909 if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
910 return;
911 }
912 if (!readPending) {
913
914
915 socketHasMoreData = true;
916 return;
917 }
918 scheduleFirstReadIfNeeded();
919 }
920
921 private void scheduleFirstReadIfNeeded() {
922 if ((ioState & READ_SCHEDULED) == 0) {
923 scheduleFirstRead();
924 }
925 }
926
927 private void scheduleFirstRead() {
928
929 final ChannelConfig config = config();
930 final IoUringRecvByteAllocatorHandle allocHandle = recvBufAllocHandle();
931 allocHandle.reset(config);
932 scheduleRead(true);
933 }
934
935 protected final void scheduleRead(boolean first) {
936
937 if (delayedClose == null && fd().isOpen() && (ioState & READ_SCHEDULED) == 0) {
938 numOutstandingReads = (short) scheduleRead0(first, socketIsEmpty);
939 if (numOutstandingReads > 0 || numOutstandingReads == -1) {
940 ioState |= READ_SCHEDULED;
941 }
942 }
943 }
944
945
946
947
948
949
950
951
952
953
954
955 protected abstract int scheduleRead0(boolean first, boolean socketIsEmpty);
956
957
958
959
960
961
962 private void pollOut(int res) {
963 ioState &= ~POLL_OUT_SCHEDULED;
964 pollOutId = 0;
965 if (res == Native.ERRNO_ECANCELED_NEGATIVE) {
966 return;
967 }
968
969 if (connectPromise != null) {
970
971
972
973 assert eventLoop().inEventLoop();
974
975 boolean connectStillInProgress = false;
976 try {
977 boolean wasActive = isActive();
978 if (!socket.finishConnect()) {
979 connectStillInProgress = true;
980 return;
981 }
982 fulfillConnectPromise(connectPromise, wasActive);
983 } catch (Throwable t) {
984 fulfillConnectPromise(connectPromise, annotateConnectException(t, requestedRemoteAddress));
985 } finally {
986 if (!connectStillInProgress) {
987
988
989
990 cancelConnectTimeoutFuture();
991 connectPromise = null;
992 } else {
993
994 schedulePollOut();
995 }
996 }
997 } else if (!socket.isOutputShutdown()) {
998
999 super.flush0();
1000 }
1001 }
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011 private void writeComplete(byte op, int res, int flags, long data) {
1012 writeTracker.complete(data, op, flags);
1013 if ((ioState & CONNECT_SCHEDULED) != 0) {
1014
1015
1016 freeMsgHdrArray();
1017 if (res > 0) {
1018
1019
1020
1021 ChannelOutboundBuffer channelOutboundBuffer = outboundBuffer();
1022 if (channelOutboundBuffer != null) {
1023 channelOutboundBuffer.removeBytes(res);
1024 }
1025
1026
1027 connectComplete(op, 0, flags, data);
1028 } else if (res == ERRNO_EINPROGRESS_NEGATIVE || res == 0) {
1029
1030
1031
1032
1033 submitConnect((InetSocketAddress) requestedRemoteAddress);
1034 } else {
1035
1036 connectComplete(op, res, flags, data);
1037 }
1038 return;
1039 }
1040
1041 if ((flags & Native.IORING_CQE_F_NOTIF) == 0) {
1042 assert numOutstandingWrites > 0;
1043 --numOutstandingWrites;
1044 }
1045
1046 boolean writtenAll = writeComplete0(op, res, flags, data, numOutstandingWrites);
1047 if (!writtenAll && (ioState & POLL_OUT_SCHEDULED) == 0) {
1048
1049
1050 schedulePollOut();
1051 }
1052
1053
1054
1055 if (numOutstandingWrites == 0) {
1056 ioState &= ~WRITE_SCHEDULED;
1057
1058
1059 if (writtenAll && (ioState & POLL_OUT_SCHEDULED) == 0) {
1060 scheduleWriteIfNeeded(unsafe().outboundBuffer(), false);
1061 }
1062 }
1063 }
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073 abstract boolean writeComplete0(byte op, int res, int flags, long data, int outstanding);
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083 void cancelComplete0(byte op, int res, int flags, short data) {
1084
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094 void connectComplete(byte op, int res, int flags, long data) {
1095 ioState &= ~CONNECT_SCHEDULED;
1096 freeRemoteAddressMemory();
1097
1098 if (res == ERRNO_EINPROGRESS_NEGATIVE || res == ERROR_EALREADY_NEGATIVE) {
1099
1100 schedulePollOut();
1101 } else {
1102 try {
1103 if (res == 0) {
1104 fulfillConnectPromise(connectPromise, active);
1105 if (readPending) {
1106 doBeginReadNow();
1107 }
1108 } else {
1109 try {
1110 Errors.throwConnectException("io_uring connect", res);
1111 } catch (Throwable cause) {
1112 fulfillConnectPromise(connectPromise, cause);
1113 }
1114 }
1115 } finally {
1116
1117
1118
1119 cancelConnectTimeoutFuture();
1120 connectPromise = null;
1121 }
1122 }
1123 }
1124
1125 @Override
1126 public void connect(
1127 final SocketAddress remoteAddress, final SocketAddress localAddress, final ChannelPromise promise) {
1128
1129
1130 if (promise.isDone() || !ensureOpen(promise)) {
1131 return;
1132 }
1133
1134 if (delayedClose != null) {
1135 promise.tryFailure(annotateConnectException(new ClosedChannelException(), remoteAddress));
1136 return;
1137 }
1138 try {
1139 if (connectPromise != null) {
1140 throw new ConnectionPendingException();
1141 }
1142 if (localAddress instanceof InetSocketAddress) {
1143 checkResolvable((InetSocketAddress) localAddress);
1144 }
1145
1146 if (remoteAddress instanceof InetSocketAddress) {
1147 checkResolvable((InetSocketAddress) remoteAddress);
1148 }
1149
1150 if (remote != null) {
1151
1152
1153
1154 throw new AlreadyConnectedException();
1155 }
1156
1157 if (localAddress != null) {
1158 socket.bind(localAddress);
1159 }
1160
1161 if (remoteAddress instanceof InetSocketAddress) {
1162 InetSocketAddress inetSocketAddress = (InetSocketAddress) remoteAddress;
1163 ByteBuf initialData = null;
1164 if (IoUring.isTcpFastOpenClientSideAvailable() &&
1165 config().getOption(ChannelOption.TCP_FASTOPEN_CONNECT) == Boolean.TRUE) {
1166 ChannelOutboundBuffer outbound = unsafe().outboundBuffer();
1167 outbound.addFlush();
1168 Object curr;
1169 if ((curr = outbound.current()) instanceof ByteBuf) {
1170 initialData = (ByteBuf) curr;
1171 }
1172 }
1173 if (initialData != null) {
1174 msgHdrMemoryArray = new MsgHdrMemoryArray((short) 1);
1175 MsgHdrMemory hdr = msgHdrMemoryArray.hdr(0);
1176 fillTFOInitData(hdr, inetSocketAddress, initialData);
1177
1178 int fd = fd().intValue();
1179 IoRegistration registration = registration();
1180 short opsId = writeTracker.nextId();
1181 if (opsId == 0) {
1182 freeMsgHdrArray();
1183 submitConnect(inetSocketAddress);
1184 } else {
1185 IoUringIoOps ops = IoUringIoOps.newSendmsg(fd, (byte) 0, Native.MSG_FASTOPEN,
1186 hdr.address(), opsId);
1187 writeTracker.record(opsId, ops.opcode(), initialData);
1188 connectId = registration.submit(ops);
1189 if (connectId == 0) {
1190 writeTracker.abandon(opsId, ops.opcode());
1191
1192 freeMsgHdrArray();
1193 }
1194 }
1195 } else {
1196 submitConnect(inetSocketAddress);
1197 }
1198 } else if (remoteAddress instanceof DomainSocketAddress) {
1199 DomainSocketAddress unixDomainSocketAddress = (DomainSocketAddress) remoteAddress;
1200 submitConnect(unixDomainSocketAddress);
1201 } else {
1202 throw new Error("Unexpected SocketAddress implementation " + className(remoteAddress));
1203 }
1204
1205 if (connectId != 0) {
1206 ioState |= CONNECT_SCHEDULED;
1207 }
1208 } catch (Throwable t) {
1209 closeIfClosed();
1210 promise.tryFailure(annotateConnectException(t, remoteAddress));
1211 return;
1212 }
1213 connectPromise = promise;
1214 requestedRemoteAddress = remoteAddress;
1215
1216 int connectTimeoutMillis = config().getConnectTimeoutMillis();
1217 if (connectTimeoutMillis > 0) {
1218 connectTimeoutFuture = eventLoop().schedule(new Runnable() {
1219 @Override
1220 public void run() {
1221 ChannelPromise connectPromise = AbstractIoUringChannel.this.connectPromise;
1222 if (connectPromise != null && !connectPromise.isDone() &&
1223 connectPromise.tryFailure(new ConnectTimeoutException(
1224 "connection timed out: " + remoteAddress))) {
1225 close(voidPromise());
1226 }
1227 }
1228 }, connectTimeoutMillis, TimeUnit.MILLISECONDS);
1229 }
1230
1231 promise.addListener(new ChannelFutureListener() {
1232 @Override
1233 public void operationComplete(ChannelFuture future) {
1234
1235
1236 if (future.isCancelled()) {
1237 cancelConnectTimeoutFuture();
1238 connectPromise = null;
1239 close(voidPromise());
1240 }
1241 }
1242 });
1243 }
1244
1245 private void fillTFOInitData(MsgHdrMemory hdr, InetSocketAddress inetSocketAddress,
1246 ByteBuf initialData) throws Exception {
1247 if (initialData.hasMemoryAddress()) {
1248 hdr.set(socket, inetSocketAddress,
1249 initialData.memoryAddress() + initialData.readerIndex(),
1250 initialData.readableBytes(), (short) 0);
1251 } else {
1252
1253
1254 IoUringIoHandler handler = registration().attachment();
1255 IovArray iovArray = handler.iovArray();
1256 int iovOffset = iovArray.count();
1257 iovArray.processMessage(initialData);
1258 long iovArrayAddress = iovArray.memoryAddress(iovOffset);
1259 int iovArrayLength = iovArray.count() - iovOffset;
1260 hdr.setWithIovArrayAddress(socket, inetSocketAddress, iovArrayAddress, iovArrayLength, (short) 0);
1261 }
1262 }
1263
1264 @Override
1265 public final void deregister(ChannelPromise promise) {
1266 if (deregisterPromise != null) {
1267
1268 PromiseNotifier.cascade(deregisterPromise, promise);
1269 } else if (!isRegistered()) {
1270 promise.setSuccess();
1271 } else {
1272
1273
1274 deregisterPromise = promise;
1275 super.deregister(newPromise().addListener(f -> {
1276 if (!f.isSuccess()) {
1277 this.deregisterPromise = null;
1278 promise.setFailure(f.cause());
1279 }
1280 }));
1281 }
1282 }
1283 }
1284
1285 private void submitConnect(InetSocketAddress inetSocketAddress) {
1286 cleanable = Buffer.allocateDirectBufferWithNativeOrder(Native.SIZEOF_SOCKADDR_STORAGE);
1287 remoteAddressMemory = cleanable.buffer();
1288
1289 SockaddrIn.set(socket.isIpv6(), remoteAddressMemory, inetSocketAddress);
1290
1291 int fd = fd().intValue();
1292 IoRegistration registration = registration();
1293 IoUringIoOps ops = IoUringIoOps.newConnect(
1294 fd, (byte) 0, Buffer.memoryAddress(remoteAddressMemory), nextOpsId());
1295 connectId = registration.submit(ops);
1296 if (connectId == 0) {
1297
1298 freeRemoteAddressMemory();
1299 }
1300 }
1301
1302 private void submitConnect(DomainSocketAddress unixDomainSocketAddress) {
1303 cleanable = Buffer.allocateDirectBufferWithNativeOrder(Native.SIZEOF_SOCKADDR_UN);
1304 remoteAddressMemory = cleanable.buffer();
1305 int addrLen = SockaddrIn.setUds(remoteAddressMemory, unixDomainSocketAddress);
1306 int fd = fd().intValue();
1307 IoRegistration registration = registration();
1308 long addr = Buffer.memoryAddress(remoteAddressMemory);
1309 IoUringIoOps ops = IoUringIoOps.newConnect(fd, (byte) 0, addr, addrLen, nextOpsId());
1310 connectId = registration.submit(ops);
1311 if (connectId == 0) {
1312
1313 freeRemoteAddressMemory();
1314 }
1315 }
1316
1317 @Override
1318 protected Object filterOutboundMessage(Object msg) {
1319 if (msg instanceof ByteBuf) {
1320 ByteBuf buf = (ByteBuf) msg;
1321 return UnixChannelUtil.isBufferCopyNeededForWrite(buf)? newDirectBuffer(buf) : buf;
1322 }
1323 throw new UnsupportedOperationException("unsupported message type: " + StringUtil.simpleClassName(msg));
1324 }
1325
1326 @Override
1327 protected void doRegister(ChannelPromise promise) {
1328 IoEventLoop eventLoop = (IoEventLoop) eventLoop();
1329 eventLoop.register(ioUringUnsafe()).addListener(f -> {
1330 if (f.isSuccess()) {
1331 registration = (IoRegistration) f.getNow();
1332 promise.setSuccess();
1333 } else {
1334 promise.setFailure(f.cause());
1335 }
1336 });
1337 }
1338
1339 @Override
1340 protected final void doDeregister() {
1341
1342 if (!ioUringUnsafe().cancelOps(connectPromise != null)) {
1343
1344
1345 if (registration != null) {
1346 registration.cancel();
1347 }
1348 }
1349 }
1350
1351 @Override
1352 protected void doBind(final SocketAddress local) throws Exception {
1353 if (local instanceof InetSocketAddress) {
1354 checkResolvable((InetSocketAddress) local);
1355 }
1356 socket.bind(local);
1357 this.local = socket.localAddress();
1358 }
1359
1360 protected static void checkResolvable(InetSocketAddress addr) {
1361 if (addr.isUnresolved()) {
1362 throw new UnresolvedAddressException();
1363 }
1364 }
1365
1366 @Override
1367 protected final SocketAddress localAddress0() {
1368 return local;
1369 }
1370
1371 @Override
1372 protected final SocketAddress remoteAddress0() {
1373 return remote;
1374 }
1375
1376 private static boolean isAllowHalfClosure(ChannelConfig config) {
1377 return config instanceof SocketChannelConfig &&
1378 ((SocketChannelConfig) config).isAllowHalfClosure();
1379 }
1380
1381 private void cancelConnectTimeoutFuture() {
1382 if (connectTimeoutFuture != null) {
1383 connectTimeoutFuture.cancel(false);
1384 connectTimeoutFuture = null;
1385 }
1386 }
1387
1388 private void computeRemote() {
1389 if (requestedRemoteAddress instanceof InetSocketAddress) {
1390 remote = computeRemoteAddr((InetSocketAddress) requestedRemoteAddress, socket.remoteAddress());
1391 }
1392 }
1393
1394 private boolean shouldBreakIoUringInReady(ChannelConfig config) {
1395 return socket.isInputShutdown() && (inputClosedSeenErrorOnRead || !isAllowHalfClosure(config));
1396 }
1397
1398
1399
1400
1401
1402
1403
1404 protected abstract boolean socketIsEmpty(int flags);
1405
1406 abstract boolean isPollInFirst();
1407 }