View Javadoc
1   /*
2    * Copyright 2026 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package io.netty.channel.uring;
17  
18  import io.netty.buffer.ByteBuf;
19  import io.netty.channel.ChannelOutboundBuffer;
20  import io.netty.util.ReferenceCountUtil;
21  import io.netty.util.ReferenceCounted;
22  import io.netty.util.collection.LongObjectHashMap;
23  
24  import java.util.Arrays;
25  
26  import static io.netty.channel.uring.AbstractIoUringStreamChannel.IovBufferCollector.remainingIovs;
27  
28  /**
29   * Retains buffers until their zero-copy notifications arrive.
30   *
31   * <p>{@code freeIdStack} is a LIFO stack of available IDs indexing {@code pooled}. Submission pops an ID;
32   * a failed submission, a primary CQE without MORE, or the NOTIF CQE returns it. This allows out-of-order
33   * notifications to recycle IDs independently.
34   *
35   * <p>The pool grows up to {@link Short#MAX_VALUE} IDs. If all are in use, for example because notifications
36   * lag behind new writes, further requests receive increasing long IDs above that range. Their retained
37   * buffers are stored in {@code overflow} until NOTIF, and these IDs are never reused. Returned pooled IDs
38   * are preferred even while overflow requests remain pending.
39   */
40  final class PendingZeroCopyWrites {
41      private static final int MAX_POOLED_ID = Short.MAX_VALUE;
42  
43      private PendingWrite[] pooled;
44      private short[] freeIdStack;
45      // Next insertion index in the free ID stack.
46      private int freeStackTop;
47      // Overflow IDs are never reused; exhausting the positive long range is not feasible in practice.
48      private long nextOverflowId = MAX_POOLED_ID + 1L;
49      private LongObjectHashMap<PendingWrite> overflow;
50  
51      PendingZeroCopyWrites() {
52          pooled = new PendingWrite[4];
53          freeIdStack = new short[pooled.length];
54          // Slot 0 is unused.
55          for (int id = pooled.length - 1; id > 0; id--) {
56              freeIdStack[freeStackTop++] = (short) id;
57          }
58      }
59  
60      long nextUserData() {
61          if (freeStackTop > 0) {
62              return freeIdStack[--freeStackTop];
63          }
64          return nextUserDataSlow();
65      }
66  
67      private long nextUserDataSlow() {
68          int oldCapacity = pooled.length;
69          if (oldCapacity == MAX_POOLED_ID + 1) {
70              return nextOverflowId++;
71          }
72          int newCapacity = oldCapacity << 1;
73          pooled = Arrays.copyOf(pooled, newCapacity);
74          freeIdStack = new short[newCapacity];
75          for (int id = newCapacity - 1; id >= oldCapacity; id--) {
76              freeIdStack[freeStackTop++] = (short) id;
77          }
78          return nextUserData();
79      }
80  
81      PendingWrite register(long userData) {
82          if (userData > MAX_POOLED_ID) {
83              return registerOverflow(userData);
84          }
85          int id = (int) userData;
86          PendingWrite write = pooled[id];
87          if (write == null) {
88              pooled[id] = write = new PendingWrite();
89          }
90          return write;
91      }
92  
93      // Only used when all pooled IDs are in flight; registration preserves these IDs on its slow path.
94      private PendingWrite registerOverflow(long userData) {
95          if (overflow == null) {
96              overflow = new LongObjectHashMap<>(2);
97          }
98          PendingWrite write = new PendingWrite();
99          overflow.put(userData, write);
100         return write;
101     }
102 
103     void release(long userData) {
104         PendingWrite write = userData <= MAX_POOLED_ID ?
105                 pooled[(int) userData] : overflow.remove(userData);
106         write.release();
107         recycle(userData);
108     }
109 
110     // Also used for requests without MORE, which retain no buffers.
111     void recycle(long userData) {
112         if (userData <= MAX_POOLED_ID) {
113             freeIdStack[freeStackTop++] = (short) userData;
114         }
115     }
116 
117     static final class PendingWrite implements ChannelOutboundBuffer.MessageProcessor {
118         private ReferenceCounted[] references = new ReferenceCounted[1];
119         private int count;
120         private int remaining;
121 
122         void retain(ChannelOutboundBuffer buffer, int iovCount) {
123             if (iovCount == 0) {
124                 return;
125             }
126             ByteBuf first = (ByteBuf) buffer.current();
127             if (buffer.size() == 1 || iovCount == 1 && first.isReadable()) {
128                 add(first.retain());
129                 return;
130             }
131             remaining = iovCount;
132             try {
133                 buffer.forEachFlushedMessage(this);
134             } catch (Exception e) {
135                 throw new IllegalStateException(e);
136             }
137         }
138 
139         @Override
140         public boolean processMessage(Object msg) {
141             ByteBuf buf = (ByteBuf) msg;
142             if (buf.isReadable()) {
143                 add(buf.retain());
144                 remaining = remainingIovs(buf, remaining);
145             }
146             return remaining != 0;
147         }
148 
149         void add(ReferenceCounted reference) {
150             if (count == references.length) {
151                 references = Arrays.copyOf(references, count << 1);
152             }
153             references[count++] = reference;
154         }
155 
156         private void release() {
157             for (int i = 0; i < count; i++) {
158                 ReferenceCountUtil.safeRelease(references[i]);
159                 references[i] = null;
160             }
161             count = 0;
162         }
163     }
164 }