View Javadoc
1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   https://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  
17  package io.netty.buffer;
18  
19  import io.netty.util.internal.CleanableDirectBuffer;
20  import io.netty.util.internal.PlatformDependent;
21  import io.netty.util.internal.StringUtil;
22  
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.List;
27  import java.util.concurrent.atomic.AtomicInteger;
28  import java.util.concurrent.atomic.AtomicReference;
29  import java.util.concurrent.atomic.LongAdder;
30  import java.util.concurrent.locks.ReentrantLock;
31  
32  import static io.netty.buffer.PoolChunk.isSubpage;
33  import static java.lang.Math.max;
34  
35  abstract class PoolArena<T> implements PoolArenaMetric {
36      private static final boolean HAS_UNSAFE = PlatformDependent.hasUnsafe();
37  
38      enum SizeClass {
39          Small,
40          Normal
41      }
42  
43      final PooledByteBufAllocator parent;
44  
45      final PoolSubpage<T>[] smallSubpagePools;
46  
47      private final PoolChunkList<T> q050;
48      private final PoolChunkList<T> q025;
49      private final PoolChunkList<T> q000;
50      private final PoolChunkList<T> qInit;
51      private final PoolChunkList<T> q075;
52      private final PoolChunkList<T> q100;
53  
54      private final List<PoolChunkListMetric> chunkListMetrics;
55  
56      // Metrics for allocations and deallocations
57      private long allocationsNormal;
58      // We need to use the LongCounter here as this is not guarded via synchronized block.
59      private final LongAdder allocationsSmall = new LongAdder();
60      private final LongAdder allocationsHuge = new LongAdder();
61      private final LongAdder activeBytesHuge = new LongAdder();
62  
63      private long deallocationsSmall;
64      private long deallocationsNormal;
65  
66      private long pooledChunkAllocations;
67      private long pooledChunkDeallocations;
68  
69      // We need to use the LongCounter here as this is not guarded via synchronized block.
70      private final LongAdder deallocationsHuge = new LongAdder();
71  
72      // Number of thread caches backed by this arena.
73      final AtomicInteger numThreadCaches = new AtomicInteger();
74  
75      // TODO: Test if adding padding helps under contention
76      //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7;
77  
78      private final ReentrantLock lock = new ReentrantLock();
79  
80      final SizeClasses sizeClass;
81  
82      protected PoolArena(PooledByteBufAllocator parent, SizeClasses sizeClass) {
83          assert null != sizeClass;
84          this.parent = parent;
85          this.sizeClass = sizeClass;
86          smallSubpagePools = newSubpagePoolArray(sizeClass.nSubpages);
87          for (int i = 0; i < smallSubpagePools.length; i ++) {
88              smallSubpagePools[i] = newSubpagePoolHead(i);
89          }
90  
91          q100 = new PoolChunkList<T>(this, null, 100, Integer.MAX_VALUE, sizeClass.chunkSize);
92          q075 = new PoolChunkList<T>(this, q100, 75, 100, sizeClass.chunkSize);
93          q050 = new PoolChunkList<T>(this, q100, 50, 100, sizeClass.chunkSize);
94          q025 = new PoolChunkList<T>(this, q050, 25, 75, sizeClass.chunkSize);
95          q000 = new PoolChunkList<T>(this, q025, 1, 50, sizeClass.chunkSize);
96          qInit = new PoolChunkList<T>(this, q000, Integer.MIN_VALUE, 25, sizeClass.chunkSize);
97  
98          q100.prevList(q075);
99          q075.prevList(q050);
100         q050.prevList(q025);
101         q025.prevList(q000);
102         q000.prevList(null);
103         qInit.prevList(qInit);
104 
105         List<PoolChunkListMetric> metrics = new ArrayList<PoolChunkListMetric>(6);
106         metrics.add(qInit);
107         metrics.add(q000);
108         metrics.add(q025);
109         metrics.add(q050);
110         metrics.add(q075);
111         metrics.add(q100);
112         chunkListMetrics = Collections.unmodifiableList(metrics);
113     }
114 
115     private PoolSubpage<T> newSubpagePoolHead(int index) {
116         PoolSubpage<T> head = new PoolSubpage<T>(index);
117         head.prev = head;
118         head.next = head;
119         return head;
120     }
121 
122     @SuppressWarnings("unchecked")
123     private PoolSubpage<T>[] newSubpagePoolArray(int size) {
124         return new PoolSubpage[size];
125     }
126 
127     abstract boolean isDirect();
128 
129     PooledByteBuf<T> allocate(PoolThreadCache cache, int reqCapacity, int maxCapacity) {
130         PooledByteBuf<T> buf = newByteBuf(maxCapacity);
131         allocate(cache, buf, reqCapacity);
132         return buf;
133     }
134 
135     private void allocate(PoolThreadCache cache, PooledByteBuf<T> buf, final int reqCapacity) {
136         final int sizeIdx = sizeClass.size2SizeIdx(reqCapacity);
137 
138         if (sizeIdx <= sizeClass.smallMaxSizeIdx) {
139             tcacheAllocateSmall(cache, buf, reqCapacity, sizeIdx);
140         } else if (sizeIdx < sizeClass.nSizes) {
141             tcacheAllocateNormal(cache, buf, reqCapacity, sizeIdx);
142         } else {
143             int normCapacity = sizeClass.directMemoryCacheAlignment > 0
144                     ? sizeClass.normalizeSize(reqCapacity) : reqCapacity;
145             // Huge allocations are never served via the cache so just call allocateHuge
146             allocateHuge(buf, normCapacity);
147         }
148     }
149 
150     private void tcacheAllocateSmall(PoolThreadCache cache, PooledByteBuf<T> buf, final int reqCapacity,
151                                      final int sizeIdx) {
152 
153         if (cache.allocateSmall(this, buf, reqCapacity, sizeIdx)) {
154             // was able to allocate out of the cache so move on
155             return;
156         }
157 
158         /*
159          * Synchronize on the head. This is needed as {@link PoolChunk#allocateSubpage(int)} and
160          * {@link PoolChunk#free(long)} may modify the doubly linked list as well.
161          */
162         final PoolSubpage<T> head = smallSubpagePools[sizeIdx];
163         final boolean needsNormalAllocation;
164         head.lock();
165         try {
166             final PoolSubpage<T> s = head.next;
167             needsNormalAllocation = s == head;
168             if (!needsNormalAllocation) {
169                 assert s.doNotDestroy && s.elemSize == sizeClass.sizeIdx2size(sizeIdx) : "doNotDestroy=" +
170                         s.doNotDestroy + ", elemSize=" + s.elemSize + ", sizeIdx=" + sizeIdx;
171                 long handle = s.allocate();
172                 assert handle >= 0;
173                 s.chunk.initBufWithSubpage(buf, null, handle, reqCapacity, cache, false);
174             }
175         } finally {
176             head.unlock();
177         }
178 
179         if (needsNormalAllocation) {
180             lock();
181             try {
182                 allocateNormal(buf, reqCapacity, sizeIdx, cache);
183             } finally {
184                 unlock();
185             }
186         }
187 
188         incSmallAllocation();
189     }
190 
191     private void tcacheAllocateNormal(PoolThreadCache cache, PooledByteBuf<T> buf, final int reqCapacity,
192                                       final int sizeIdx) {
193         if (cache.allocateNormal(this, buf, reqCapacity, sizeIdx)) {
194             // was able to allocate out of the cache so move on
195             return;
196         }
197         lock();
198         try {
199             allocateNormal(buf, reqCapacity, sizeIdx, cache);
200             ++allocationsNormal;
201         } finally {
202             unlock();
203         }
204     }
205 
206     private void allocateNormal(PooledByteBuf<T> buf, int reqCapacity, int sizeIdx, PoolThreadCache threadCache) {
207         assert lock.isHeldByCurrentThread();
208         if (q050.allocate(buf, reqCapacity, sizeIdx, threadCache) ||
209             q025.allocate(buf, reqCapacity, sizeIdx, threadCache) ||
210             q000.allocate(buf, reqCapacity, sizeIdx, threadCache) ||
211             qInit.allocate(buf, reqCapacity, sizeIdx, threadCache) ||
212             q075.allocate(buf, reqCapacity, sizeIdx, threadCache)) {
213             return;
214         }
215 
216         // Add a new chunk.
217         PoolChunk<T> c = newChunk(sizeClass.pageSize, sizeClass.nPSizes, sizeClass.pageShifts, sizeClass.chunkSize);
218         PooledByteBufAllocator.onAllocateChunk(c, true);
219         boolean success = c.allocate(buf, reqCapacity, sizeIdx, threadCache);
220         assert success;
221         qInit.add(c);
222         ++pooledChunkAllocations;
223     }
224 
225     private void incSmallAllocation() {
226         allocationsSmall.increment();
227     }
228 
229     private void allocateHuge(PooledByteBuf<T> buf, int reqCapacity) {
230         PoolChunk<T> chunk = newUnpooledChunk(reqCapacity);
231         PooledByteBufAllocator.onAllocateChunk(chunk, false);
232         activeBytesHuge.add(chunk.chunkSize());
233         buf.initUnpooled(chunk, reqCapacity);
234         allocationsHuge.increment();
235     }
236 
237     void free(PoolChunk<T> chunk, ByteBuffer nioBuffer, long handle, int normCapacity, PoolThreadCache cache) {
238         chunk.decrementPinnedMemory(normCapacity);
239         if (chunk.unpooled) {
240             int size = chunk.chunkSize();
241             destroyChunk(chunk);
242             activeBytesHuge.add(-size);
243             deallocationsHuge.increment();
244         } else {
245             SizeClass sizeClass = sizeClass(handle);
246             if (cache != null && cache.add(this, chunk, nioBuffer, handle, normCapacity, sizeClass)) {
247                 // cached so not free it.
248                 return;
249             }
250 
251             freeChunk(chunk, handle, normCapacity, sizeClass, nioBuffer, false);
252         }
253     }
254 
255     private static SizeClass sizeClass(long handle) {
256         return isSubpage(handle) ? SizeClass.Small : SizeClass.Normal;
257     }
258 
259     void freeChunk(PoolChunk<T> chunk, long handle, int normCapacity, SizeClass sizeClass, ByteBuffer nioBuffer,
260                    boolean finalizer) {
261         final boolean destroyChunk;
262         lock();
263         try {
264             // We only call this if freeChunk is not called because of the PoolThreadCache finalizer as otherwise this
265             // may fail due lazy class-loading in for example tomcat.
266             if (!finalizer) {
267                 switch (sizeClass) {
268                     case Normal:
269                         ++deallocationsNormal;
270                         break;
271                     case Small:
272                         ++deallocationsSmall;
273                         break;
274                     default:
275                         throw new Error();
276                 }
277             }
278             destroyChunk = !chunk.parent.free(chunk, handle, normCapacity, nioBuffer);
279             if (destroyChunk) {
280                 // all other destroyChunk calls come from the arena itself being finalized, so don't need to be counted
281                 ++pooledChunkDeallocations;
282             }
283         } finally {
284             unlock();
285         }
286         if (destroyChunk) {
287             // destroyChunk not need to be called while holding the synchronized lock.
288             destroyChunk(chunk);
289         }
290     }
291 
292     void reallocate(PooledByteBuf<T> buf, int newCapacity) {
293         assert newCapacity >= 0 && newCapacity <= buf.maxCapacity();
294 
295         final int oldCapacity;
296         final PoolChunk<T> oldChunk;
297         final ByteBuffer oldNioBuffer;
298         final long oldHandle;
299         final T oldMemory;
300         final int oldOffset;
301         final int oldMaxLength;
302         final PoolThreadCache oldCache;
303 
304         // We synchronize on the ByteBuf itself to ensure there is no "concurrent" reallocations for the same buffer.
305         // We do this to ensure the ByteBuf internal fields that are used to allocate / free are not accessed
306         // concurrently. This is important as otherwise we might end up corrupting our internal state of our data
307         // structures.
308         //
309         // Also note we don't use a Lock here but just synchronized even tho this might seem like a bad choice for Loom.
310         // This is done to minimize the overhead per ByteBuf. The time this would block another thread should be
311         // relative small and so not be a problem for Loom.
312         // See https://github.com/netty/netty/issues/13467
313         synchronized (buf) {
314             oldCapacity = buf.length;
315             if (oldCapacity == newCapacity) {
316                 return;
317             }
318 
319             oldChunk = buf.chunk;
320             oldNioBuffer = buf.tmpNioBuf;
321             oldHandle = buf.handle;
322             oldMemory = buf.memory;
323             oldOffset = buf.offset;
324             oldMaxLength = buf.maxLength;
325             oldCache = buf.cache;
326 
327             // This does not touch buf's reader/writer indices
328             allocate(parent.threadCache(), buf, newCapacity);
329         }
330         int bytesToCopy;
331         if (newCapacity > oldCapacity) {
332             bytesToCopy = oldCapacity;
333         } else {
334             buf.trimIndicesToCapacity(newCapacity);
335             bytesToCopy = newCapacity;
336         }
337         memoryCopy(oldMemory, oldOffset, buf, bytesToCopy);
338         free(oldChunk, oldNioBuffer, oldHandle, oldMaxLength, oldCache);
339     }
340 
341     @Override
342     public int numThreadCaches() {
343         return numThreadCaches.get();
344     }
345 
346     @Override
347     public int numTinySubpages() {
348         return 0;
349     }
350 
351     @Override
352     public int numSmallSubpages() {
353         return smallSubpagePools.length;
354     }
355 
356     @Override
357     public int numChunkLists() {
358         return chunkListMetrics.size();
359     }
360 
361     @Override
362     public List<PoolSubpageMetric> tinySubpages() {
363         return Collections.emptyList();
364     }
365 
366     @Override
367     public List<PoolSubpageMetric> smallSubpages() {
368         return subPageMetricList(smallSubpagePools);
369     }
370 
371     @Override
372     public List<PoolChunkListMetric> chunkLists() {
373         return chunkListMetrics;
374     }
375 
376     private static List<PoolSubpageMetric> subPageMetricList(PoolSubpage<?>[] pages) {
377         List<PoolSubpageMetric> metrics = new ArrayList<PoolSubpageMetric>();
378         for (PoolSubpage<?> head : pages) {
379             if (head.next == head) {
380                 continue;
381             }
382             PoolSubpage<?> s = head.next;
383             for (;;) {
384                 metrics.add(s);
385                 s = s.next;
386                 if (s == head) {
387                     break;
388                 }
389             }
390         }
391         return metrics;
392     }
393 
394     @Override
395     public long numAllocations() {
396         final long allocsNormal;
397         lock();
398         try {
399             allocsNormal = allocationsNormal;
400         } finally {
401             unlock();
402         }
403         return allocationsSmall.sum() + allocsNormal + allocationsHuge.sum();
404     }
405 
406     @Override
407     public long numTinyAllocations() {
408         return 0;
409     }
410 
411     @Override
412     public long numSmallAllocations() {
413         return allocationsSmall.sum();
414     }
415 
416     @Override
417     public long numNormalAllocations() {
418         lock();
419         try {
420             return allocationsNormal;
421         } finally {
422             unlock();
423         }
424     }
425 
426     @Override
427     public long numChunkAllocations() {
428         lock();
429         try {
430             return pooledChunkAllocations;
431         } finally {
432             unlock();
433         }
434     }
435 
436     @Override
437     public long numDeallocations() {
438         final long deallocs;
439         lock();
440         try {
441             deallocs = deallocationsSmall + deallocationsNormal;
442         } finally {
443             unlock();
444         }
445         return deallocs + deallocationsHuge.sum();
446     }
447 
448     @Override
449     public long numTinyDeallocations() {
450         return 0;
451     }
452 
453     @Override
454     public long numSmallDeallocations() {
455         lock();
456         try {
457             return deallocationsSmall;
458         } finally {
459             unlock();
460         }
461     }
462 
463     @Override
464     public long numNormalDeallocations() {
465         lock();
466         try {
467             return deallocationsNormal;
468         } finally {
469             unlock();
470         }
471     }
472 
473     @Override
474     public long numChunkDeallocations() {
475         lock();
476         try {
477             return pooledChunkDeallocations;
478         } finally {
479             unlock();
480         }
481     }
482 
483     @Override
484     public long numHugeAllocations() {
485         return allocationsHuge.sum();
486     }
487 
488     @Override
489     public long numHugeDeallocations() {
490         return deallocationsHuge.sum();
491     }
492 
493     @Override
494     public  long numActiveAllocations() {
495         long val = allocationsSmall.sum() + allocationsHuge.sum()
496                 - deallocationsHuge.sum();
497         lock();
498         try {
499             val += allocationsNormal - (deallocationsSmall + deallocationsNormal);
500         } finally {
501             unlock();
502         }
503         return max(val, 0);
504     }
505 
506     @Override
507     public long numActiveTinyAllocations() {
508         return 0;
509     }
510 
511     @Override
512     public long numActiveSmallAllocations() {
513         return max(numSmallAllocations() - numSmallDeallocations(), 0);
514     }
515 
516     @Override
517     public long numActiveNormalAllocations() {
518         final long val;
519         lock();
520         try {
521             val = allocationsNormal - deallocationsNormal;
522         } finally {
523             unlock();
524         }
525         return max(val, 0);
526     }
527 
528     @Override
529     public long numActiveChunks() {
530         final long val;
531         lock();
532         try {
533             val = pooledChunkAllocations - pooledChunkDeallocations;
534         } finally {
535             unlock();
536         }
537         return max(val, 0);
538     }
539 
540     @Override
541     public long numActiveHugeAllocations() {
542         return max(numHugeAllocations() - numHugeDeallocations(), 0);
543     }
544 
545     @Override
546     public long numActiveBytes() {
547         long val = activeBytesHuge.sum();
548         lock();
549         try {
550             for (int i = 0; i < chunkListMetrics.size(); i++) {
551                 for (PoolChunkMetric m: chunkListMetrics.get(i)) {
552                     val += m.chunkSize();
553                 }
554             }
555         } finally {
556             unlock();
557         }
558         return max(0, val);
559     }
560 
561     /**
562      * Return an estimate of the number of bytes that are currently pinned to buffer instances, by the arena. The
563      * pinned memory is not accessible for use by any other allocation, until the buffers using have all been released.
564      */
565     public long numPinnedBytes() {
566         long val = activeBytesHuge.sum(); // Huge chunks are exact-sized for the buffers they were allocated to.
567         for (int i = 0; i < chunkListMetrics.size(); i++) {
568             for (PoolChunkMetric m: chunkListMetrics.get(i)) {
569                 val += ((PoolChunk<?>) m).pinnedBytes();
570             }
571         }
572         return max(0, val);
573     }
574 
575     protected abstract PoolChunk<T> newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize);
576     protected abstract PoolChunk<T> newUnpooledChunk(int capacity);
577     protected abstract PooledByteBuf<T> newByteBuf(int maxCapacity);
578     protected abstract void memoryCopy(T src, int srcOffset, PooledByteBuf<T> dst, int length);
579     protected abstract void destroyChunk(PoolChunk<T> chunk);
580 
581     @Override
582     public String toString() {
583         lock();
584         try {
585             StringBuilder buf = new StringBuilder()
586                     .append("Chunk(s) at 0~25%:")
587                     .append(StringUtil.NEWLINE)
588                     .append(qInit)
589                     .append(StringUtil.NEWLINE)
590                     .append("Chunk(s) at 0~50%:")
591                     .append(StringUtil.NEWLINE)
592                     .append(q000)
593                     .append(StringUtil.NEWLINE)
594                     .append("Chunk(s) at 25~75%:")
595                     .append(StringUtil.NEWLINE)
596                     .append(q025)
597                     .append(StringUtil.NEWLINE)
598                     .append("Chunk(s) at 50~100%:")
599                     .append(StringUtil.NEWLINE)
600                     .append(q050)
601                     .append(StringUtil.NEWLINE)
602                     .append("Chunk(s) at 75~100%:")
603                     .append(StringUtil.NEWLINE)
604                     .append(q075)
605                     .append(StringUtil.NEWLINE)
606                     .append("Chunk(s) at 100%:")
607                     .append(StringUtil.NEWLINE)
608                     .append(q100)
609                     .append(StringUtil.NEWLINE)
610                     .append("small subpages:");
611             appendPoolSubPages(buf, smallSubpagePools);
612             buf.append(StringUtil.NEWLINE);
613             return buf.toString();
614         } finally {
615             unlock();
616         }
617     }
618 
619     private static void appendPoolSubPages(StringBuilder buf, PoolSubpage<?>[] subpages) {
620         for (int i = 0; i < subpages.length; i ++) {
621             PoolSubpage<?> head = subpages[i];
622             if (head.next == head || head.next == null) {
623                 continue;
624             }
625 
626             buf.append(StringUtil.NEWLINE)
627                     .append(i)
628                     .append(": ");
629             PoolSubpage<?> s = head.next;
630             while (s != null) {
631                 buf.append(s);
632                 s = s.next;
633                 if (s == head) {
634                     break;
635                 }
636             }
637         }
638     }
639 
640     @Override
641     protected final void finalize() throws Throwable {
642         try {
643             super.finalize();
644         } finally {
645             destroyPoolSubPages(smallSubpagePools);
646             destroyPoolChunkLists(qInit, q000, q025, q050, q075, q100);
647         }
648     }
649 
650     private static void destroyPoolSubPages(PoolSubpage<?>[] pages) {
651         for (PoolSubpage<?> page : pages) {
652             page.destroy();
653         }
654     }
655 
656     private void destroyPoolChunkLists(PoolChunkList<T>... chunkLists) {
657         for (PoolChunkList<T> chunkList: chunkLists) {
658             chunkList.destroy(this);
659         }
660     }
661 
662     static final class HeapArena extends PoolArena<byte[]> {
663         private final AtomicReference<PoolChunk<byte[]>> lastDestroyedChunk;
664 
665         HeapArena(PooledByteBufAllocator parent, SizeClasses sizeClass) {
666             super(parent, sizeClass);
667             lastDestroyedChunk = new AtomicReference<>();
668         }
669 
670         private static byte[] newByteArray(int size) {
671             return PlatformDependent.allocateUninitializedArray(size);
672         }
673 
674         @Override
675         boolean isDirect() {
676             return false;
677         }
678 
679         @Override
680         protected PoolChunk<byte[]> newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize) {
681             PoolChunk<byte[]> chunk = lastDestroyedChunk.getAndSet(null);
682             if (chunk != null) {
683                 assert chunk.chunkSize == chunkSize &&
684                         chunk.pageSize == pageSize &&
685                         chunk.maxPageIdx == maxPageIdx &&
686                         chunk.pageShifts == pageShifts;
687                 return chunk; // The parameters are always the same, so it's fine to reuse a previously allocated chunk.
688             }
689             return new PoolChunk<byte[]>(
690                     this, null, null, newByteArray(chunkSize), pageSize, pageShifts, chunkSize, maxPageIdx);
691         }
692 
693         @Override
694         protected PoolChunk<byte[]> newUnpooledChunk(int capacity) {
695             return new PoolChunk<byte[]>(this, null, null, newByteArray(capacity), capacity);
696         }
697 
698         @Override
699         protected void destroyChunk(PoolChunk<byte[]> chunk) {
700             PooledByteBufAllocator.onDeallocateChunk(chunk, !chunk.unpooled);
701             // Rely on GC. But keep one chunk for reuse.
702             if (!chunk.unpooled && lastDestroyedChunk.get() == null) {
703                 lastDestroyedChunk.set(chunk); // The check-and-set does not need to be atomic.
704             }
705         }
706 
707         @Override
708         protected PooledByteBuf<byte[]> newByteBuf(int maxCapacity) {
709             return HAS_UNSAFE ? PooledUnsafeHeapByteBuf.newUnsafeInstance(maxCapacity)
710                     : PooledHeapByteBuf.newInstance(maxCapacity);
711         }
712 
713         @Override
714         protected void memoryCopy(byte[] src, int srcOffset, PooledByteBuf<byte[]> dst, int length) {
715             if (length == 0) {
716                 return;
717             }
718 
719             System.arraycopy(src, srcOffset, dst.memory, dst.offset, length);
720         }
721     }
722 
723     static final class DirectArena extends PoolArena<ByteBuffer> {
724 
725         DirectArena(PooledByteBufAllocator parent, SizeClasses sizeClass) {
726             super(parent, sizeClass);
727         }
728 
729         @Override
730         boolean isDirect() {
731             return true;
732         }
733 
734         @Override
735         protected PoolChunk<ByteBuffer> newChunk(int pageSize, int maxPageIdx, int pageShifts, int chunkSize) {
736             if (sizeClass.directMemoryCacheAlignment == 0) {
737                 CleanableDirectBuffer cleanableDirectBuffer = allocateDirect(chunkSize);
738                 ByteBuffer memory = cleanableDirectBuffer.buffer();
739                 return new PoolChunk<ByteBuffer>(this, cleanableDirectBuffer, memory, memory, pageSize, pageShifts,
740                         chunkSize, maxPageIdx);
741             }
742 
743             CleanableDirectBuffer cleanableDirectBuffer = allocateDirect(
744                     chunkSize + sizeClass.directMemoryCacheAlignment);
745             final ByteBuffer base = cleanableDirectBuffer.buffer();
746             final ByteBuffer memory = PlatformDependent.alignDirectBuffer(base, sizeClass.directMemoryCacheAlignment);
747             return new PoolChunk<ByteBuffer>(this, cleanableDirectBuffer, base, memory, pageSize,
748                     pageShifts, chunkSize, maxPageIdx);
749         }
750 
751         @Override
752         protected PoolChunk<ByteBuffer> newUnpooledChunk(int capacity) {
753             if (sizeClass.directMemoryCacheAlignment == 0) {
754                 CleanableDirectBuffer cleanableDirectBuffer = allocateDirect(capacity);
755                 ByteBuffer memory = cleanableDirectBuffer.buffer();
756                 return new PoolChunk<ByteBuffer>(this, cleanableDirectBuffer, memory, memory, capacity);
757             }
758 
759             CleanableDirectBuffer cleanableDirectBuffer = allocateDirect(
760                     capacity + sizeClass.directMemoryCacheAlignment);
761             final ByteBuffer base = cleanableDirectBuffer.buffer();
762             final ByteBuffer memory = PlatformDependent.alignDirectBuffer(base, sizeClass.directMemoryCacheAlignment);
763             return new PoolChunk<ByteBuffer>(this, cleanableDirectBuffer, base, memory, capacity);
764         }
765 
766         private static CleanableDirectBuffer allocateDirect(int capacity) {
767             return PlatformDependent.allocateDirect(capacity);
768         }
769 
770         @Override
771         protected void destroyChunk(PoolChunk<ByteBuffer> chunk) {
772             PooledByteBufAllocator.onDeallocateChunk(chunk, !chunk.unpooled);
773             chunk.cleanable.clean();
774         }
775 
776         @Override
777         protected PooledByteBuf<ByteBuffer> newByteBuf(int maxCapacity) {
778             if (HAS_UNSAFE) {
779                 return PooledUnsafeDirectByteBuf.newInstance(maxCapacity);
780             } else {
781                 return PooledDirectByteBuf.newInstance(maxCapacity);
782             }
783         }
784 
785         @Override
786         protected void memoryCopy(ByteBuffer src, int srcOffset, PooledByteBuf<ByteBuffer> dstBuf, int length) {
787             if (length == 0) {
788                 return;
789             }
790 
791             if (HAS_UNSAFE) {
792                 PlatformDependent.copyMemory(
793                         PlatformDependent.directBufferAddress(src) + srcOffset,
794                         PlatformDependent.directBufferAddress(dstBuf.memory) + dstBuf.offset, length);
795             } else {
796                 // We must duplicate the NIO buffers because they may be accessed by other Netty buffers.
797                 src = src.duplicate();
798                 ByteBuffer dst = dstBuf.internalNioBuffer();
799                 src.position(srcOffset).limit(srcOffset + length);
800                 dst.position(dstBuf.offset);
801                 dst.put(src);
802             }
803         }
804     }
805 
806     void lock() {
807         lock.lock();
808     }
809 
810     void unlock() {
811         lock.unlock();
812     }
813 
814     @Override
815     public int sizeIdx2size(int sizeIdx) {
816         return sizeClass.sizeIdx2size(sizeIdx);
817     }
818 
819     @Override
820     public int sizeIdx2sizeCompute(int sizeIdx) {
821         return sizeClass.sizeIdx2sizeCompute(sizeIdx);
822     }
823 
824     @Override
825     public long pageIdx2size(int pageIdx) {
826         return sizeClass.pageIdx2size(pageIdx);
827     }
828 
829     @Override
830     public long pageIdx2sizeCompute(int pageIdx) {
831         return sizeClass.pageIdx2sizeCompute(pageIdx);
832     }
833 
834     @Override
835     public int size2SizeIdx(int size) {
836         return sizeClass.size2SizeIdx(size);
837     }
838 
839     @Override
840     public int pages2pageIdx(int pages) {
841         return sizeClass.pages2pageIdx(pages);
842     }
843 
844     @Override
845     public int pages2pageIdxFloor(int pages) {
846         return sizeClass.pages2pageIdxFloor(pages);
847     }
848 
849     @Override
850     public int normalizeSize(int size) {
851         return sizeClass.normalizeSize(size);
852     }
853 }