Skip to content

Commit b9c6737

Browse files
ChiaHungDuanChiaHungDuan
authored
[scudo] Make local cache be agnostic to the type of node in freelist (#67379)
This change moves the `TransferBatch` and `BatchGroup` out of SizeClassAllocatorLocalCache. It allows us that the node in freelist can store more blocks instead of depending on the number of blocks cached. That means we will be able to store more blocks in each node of freelist and therefore reduce the memory used by BatchClass (with little performance overhead). Note that we haven't enabled that in this patch. This is the first step of this transition.
1 parent 7a73da4 commit b9c6737

File tree

6 files changed

+167
-118
lines changed

6 files changed

+167
-118
lines changed

‎compiler-rt/lib/scudo/standalone/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ if(ANDROID)
5656
endif()
5757

5858
set(SCUDO_HEADERS
59+
allocator_common.h
5960
allocator_config.h
6061
atomic_helpers.h
6162
bytemap.h

‎compiler-rt/lib/scudo/standalone/allocator_common.h

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
//===-- allocator_common.h --------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef SCUDO_ALLOCATOR_COMMON_H_
10+
#define SCUDO_ALLOCATOR_COMMON_H_
11+
12+
#include "common.h"
13+
#include "list.h"
14+
15+
namespace scudo {
16+
17+
template <class SizeClassAllocator> struct TransferBatch {
18+
typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
19+
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
20+
21+
static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
22+
void setFromArray(CompactPtrT *Array, u16 N) {
23+
DCHECK_LE(N, MaxNumCached);
24+
Count = N;
25+
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
26+
}
27+
void appendFromArray(CompactPtrT *Array, u16 N) {
28+
DCHECK_LE(N, MaxNumCached - Count);
29+
memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
30+
// u16 will be promoted to int by arithmetic type conversion.
31+
Count = static_cast<u16>(Count + N);
32+
}
33+
void appendFromTransferBatch(TransferBatch *B, u16 N) {
34+
DCHECK_LE(N, MaxNumCached - Count);
35+
DCHECK_GE(B->Count, N);
36+
// Append from the back of `B`.
37+
memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N);
38+
// u16 will be promoted to int by arithmetic type conversion.
39+
Count = static_cast<u16>(Count + N);
40+
B->Count = static_cast<u16>(B->Count - N);
41+
}
42+
void clear() { Count = 0; }
43+
void add(CompactPtrT P) {
44+
DCHECK_LT(Count, MaxNumCached);
45+
Batch[Count++] = P;
46+
}
47+
void moveToArray(CompactPtrT *Array) {
48+
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
49+
clear();
50+
}
51+
u16 getCount() const { return Count; }
52+
bool isEmpty() const { return Count == 0U; }
53+
CompactPtrT get(u16 I) const {
54+
DCHECK_LE(I, Count);
55+
return Batch[I];
56+
}
57+
TransferBatch *Next;
58+
59+
private:
60+
CompactPtrT Batch[MaxNumCached];
61+
u16 Count;
62+
};
63+
64+
// A BatchGroup is used to collect blocks. Each group has a group id to
65+
// identify the group kind of contained blocks.
66+
template <class SizeClassAllocator> struct BatchGroup {
67+
// `Next` is used by IntrusiveList.
68+
BatchGroup *Next;
69+
// The compact base address of each group
70+
uptr CompactPtrGroupBase;
71+
// Cache value of SizeClassAllocatorLocalCache::getMaxCached()
72+
u16 MaxCachedPerBatch;
73+
// Number of blocks pushed into this group. This is an increment-only
74+
// counter.
75+
uptr PushedBlocks;
76+
// This is used to track how many bytes are not in-use since last time we
77+
// tried to release pages.
78+
uptr BytesInBGAtLastCheckpoint;
79+
// Blocks are managed by TransferBatch in a list.
80+
SinglyLinkedList<TransferBatch<SizeClassAllocator>> Batches;
81+
};
82+
83+
} // namespace scudo
84+
85+
#endif // SCUDO_ALLOCATOR_COMMON_H_

‎compiler-rt/lib/scudo/standalone/local_cache.h

+19-91
Original file line numberDiff line numberDiff line change
@@ -22,74 +22,6 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
2222
typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
2323
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
2424

25-
struct TransferBatch {
26-
static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
27-
void setFromArray(CompactPtrT *Array, u16 N) {
28-
DCHECK_LE(N, MaxNumCached);
29-
Count = N;
30-
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
31-
}
32-
void appendFromArray(CompactPtrT *Array, u16 N) {
33-
DCHECK_LE(N, MaxNumCached - Count);
34-
memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
35-
// u16 will be promoted to int by arithmetic type conversion.
36-
Count = static_cast<u16>(Count + N);
37-
}
38-
void appendFromTransferBatch(TransferBatch *B, u16 N) {
39-
DCHECK_LE(N, MaxNumCached - Count);
40-
DCHECK_GE(B->Count, N);
41-
// Append from the back of `B`.
42-
memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N);
43-
// u16 will be promoted to int by arithmetic type conversion.
44-
Count = static_cast<u16>(Count + N);
45-
B->Count = static_cast<u16>(B->Count - N);
46-
}
47-
void clear() { Count = 0; }
48-
void add(CompactPtrT P) {
49-
DCHECK_LT(Count, MaxNumCached);
50-
Batch[Count++] = P;
51-
}
52-
void copyToArray(CompactPtrT *Array) const {
53-
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
54-
}
55-
u16 getCount() const { return Count; }
56-
bool isEmpty() const { return Count == 0U; }
57-
CompactPtrT get(u16 I) const {
58-
DCHECK_LE(I, Count);
59-
return Batch[I];
60-
}
61-
static u16 getMaxCached(uptr Size) {
62-
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
63-
}
64-
TransferBatch *Next;
65-
66-
private:
67-
CompactPtrT Batch[MaxNumCached];
68-
u16 Count;
69-
};
70-
71-
// A BatchGroup is used to collect blocks. Each group has a group id to
72-
// identify the group kind of contained blocks.
73-
struct BatchGroup {
74-
// `Next` is used by IntrusiveList.
75-
BatchGroup *Next;
76-
// The compact base address of each group
77-
uptr CompactPtrGroupBase;
78-
// Cache value of TransferBatch::getMaxCached()
79-
u16 MaxCachedPerBatch;
80-
// Number of blocks pushed into this group. This is an increment-only
81-
// counter.
82-
uptr PushedBlocks;
83-
// This is used to track how many bytes are not in-use since last time we
84-
// tried to release pages.
85-
uptr BytesInBGAtLastCheckpoint;
86-
// Blocks are managed by TransferBatch in a list.
87-
SinglyLinkedList<TransferBatch> Batches;
88-
};
89-
90-
static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch),
91-
"BatchGroup uses the same class size as TransferBatch");
92-
9325
void init(GlobalStats *S, SizeClassAllocator *A) {
9426
DCHECK(isEmpty());
9527
Stats.init();
@@ -151,7 +83,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
15183
}
15284

15385
void drain() {
154-
// Drain BatchClassId last as createBatch can refill it.
86+
// Drain BatchClassId last as it may be needed while draining normal blocks.
15587
for (uptr I = 0; I < NumClasses; ++I) {
15688
if (I == BatchClassId)
15789
continue;
@@ -163,19 +95,11 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
16395
DCHECK(isEmpty());
16496
}
16597

166-
TransferBatch *createBatch(uptr ClassId, void *B) {
167-
if (ClassId != BatchClassId)
168-
B = allocate(BatchClassId);
98+
void *getBatchClassBlock() {
99+
void *B = allocate(BatchClassId);
169100
if (UNLIKELY(!B))
170101
reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
171-
return reinterpret_cast<TransferBatch *>(B);
172-
}
173-
174-
BatchGroup *createGroup() {
175-
void *Ptr = allocate(BatchClassId);
176-
if (UNLIKELY(!Ptr))
177-
reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
178-
return reinterpret_cast<BatchGroup *>(Ptr);
102+
return B;
179103
}
180104

181105
LocalStats &getStats() { return Stats; }
@@ -203,6 +127,11 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
203127
Str->append(" No block is cached.\n");
204128
}
205129

130+
static u16 getMaxCached(uptr Size) {
131+
return Min(SizeClassMap::MaxNumCachedHint,
132+
SizeClassMap::getMaxCachedHint(Size));
133+
}
134+
206135
private:
207136
static const uptr NumClasses = SizeClassMap::NumClasses;
208137
static const uptr BatchClassId = SizeClassMap::BatchClassId;
@@ -211,7 +140,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
211140
u16 MaxCount;
212141
// Note: ClassSize is zero for the transfer batch.
213142
uptr ClassSize;
214-
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
143+
CompactPtrT Chunks[2 * SizeClassMap::MaxNumCachedHint];
215144
};
216145
PerClass PerClassArray[NumClasses] = {};
217146
LocalStats Stats;
@@ -228,7 +157,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
228157
for (uptr I = 0; I < NumClasses; I++) {
229158
PerClass *P = &PerClassArray[I];
230159
const uptr Size = SizeClassAllocator::getSizeByClassId(I);
231-
P->MaxCount = static_cast<u16>(2 * TransferBatch::getMaxCached(Size));
160+
P->MaxCount = static_cast<u16>(2 * getMaxCached(Size));
232161
if (I != BatchClassId) {
233162
P->ClassSize = Size;
234163
} else {
@@ -246,15 +175,14 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
246175

247176
NOINLINE bool refill(PerClass *C, uptr ClassId) {
248177
initCacheMaybe(C);
249-
TransferBatch *B = Allocator->popBatch(this, ClassId);
250-
if (UNLIKELY(!B))
251-
return false;
252-
DCHECK_GT(B->getCount(), 0);
253-
C->Count = B->getCount();
254-
B->copyToArray(C->Chunks);
255-
B->clear();
256-
destroyBatch(ClassId, B);
257-
return true;
178+
179+
// TODO(chiahungduan): Pass the max number cached for each size class.
180+
const u16 NumBlocksRefilled =
181+
Allocator->popBlocks(this, ClassId, C->Chunks);
182+
DCHECK_LE(NumBlocksRefilled,
183+
getMaxCached(SizeClassAllocator::getSizeByClassId(ClassId)));
184+
C->Count += NumBlocksRefilled;
185+
return NumBlocksRefilled != 0;
258186
}
259187

260188
NOINLINE void drain(PerClass *C, uptr ClassId) {

‎compiler-rt/lib/scudo/standalone/primary32.h

+29-12
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#ifndef SCUDO_PRIMARY32_H_
1010
#define SCUDO_PRIMARY32_H_
1111

12+
#include "allocator_common.h"
1213
#include "bytemap.h"
1314
#include "common.h"
1415
#include "list.h"
@@ -53,8 +54,11 @@ template <typename Config> class SizeClassAllocator32 {
5354
"");
5455
typedef SizeClassAllocator32<Config> ThisT;
5556
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
56-
typedef typename CacheT::TransferBatch TransferBatch;
57-
typedef typename CacheT::BatchGroup BatchGroup;
57+
typedef TransferBatch<ThisT> TransferBatch;
58+
typedef BatchGroup<ThisT> BatchGroup;
59+
60+
static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch),
61+
"BatchGroup uses the same class size as TransferBatch");
5862

5963
static uptr getSizeByClassId(uptr ClassId) {
6064
return (ClassId == SizeClassMap::BatchClassId)
@@ -187,6 +191,21 @@ template <typename Config> class SizeClassAllocator32 {
187191
return BlockSize > PageSize;
188192
}
189193

194+
u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray) {
195+
TransferBatch *B = popBatch(C, ClassId);
196+
if (!B)
197+
return 0;
198+
199+
const u16 Count = B->getCount();
200+
DCHECK_GT(Count, 0U);
201+
B->moveToArray(ToArray);
202+
203+
if (ClassId != SizeClassMap::BatchClassId)
204+
C->deallocate(SizeClassMap::BatchClassId, B);
205+
206+
return Count;
207+
}
208+
190209
TransferBatch *popBatch(CacheT *C, uptr ClassId) {
191210
DCHECK_LT(ClassId, NumClasses);
192211
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
@@ -520,8 +539,8 @@ template <typename Config> class SizeClassAllocator32 {
520539
// from `CreateGroup` in `pushBlocksImpl`
521540
BG->PushedBlocks = 1;
522541
BG->BytesInBGAtLastCheckpoint = 0;
523-
BG->MaxCachedPerBatch = TransferBatch::getMaxCached(
524-
getSizeByClassId(SizeClassMap::BatchClassId));
542+
BG->MaxCachedPerBatch =
543+
CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId));
525544

526545
Sci->FreeListInfo.BlockList.push_front(BG);
527546
}
@@ -600,17 +619,17 @@ template <typename Config> class SizeClassAllocator32 {
600619
DCHECK_GT(Size, 0U);
601620

602621
auto CreateGroup = [&](uptr CompactPtrGroupBase) {
603-
BatchGroup *BG = C->createGroup();
622+
BatchGroup *BG = reinterpret_cast<BatchGroup *>(C->getBatchClassBlock());
604623
BG->Batches.clear();
605-
TransferBatch *TB = C->createBatch(ClassId, nullptr);
624+
TransferBatch *TB =
625+
reinterpret_cast<TransferBatch *>(C->getBatchClassBlock());
606626
TB->clear();
607627

608628
BG->CompactPtrGroupBase = CompactPtrGroupBase;
609629
BG->Batches.push_front(TB);
610630
BG->PushedBlocks = 0;
611631
BG->BytesInBGAtLastCheckpoint = 0;
612-
BG->MaxCachedPerBatch =
613-
TransferBatch::getMaxCached(getSizeByClassId(ClassId));
632+
BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId));
614633

615634
return BG;
616635
};
@@ -625,9 +644,7 @@ template <typename Config> class SizeClassAllocator32 {
625644
u16 UnusedSlots =
626645
static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
627646
if (UnusedSlots == 0) {
628-
CurBatch = C->createBatch(
629-
ClassId,
630-
reinterpret_cast<void *>(decompactPtr(ClassId, Array[I])));
647+
CurBatch = reinterpret_cast<TransferBatch *>(C->getBatchClassBlock());
631648
CurBatch->clear();
632649
Batches.push_front(CurBatch);
633650
UnusedSlots = BG->MaxCachedPerBatch;
@@ -775,7 +792,7 @@ template <typename Config> class SizeClassAllocator32 {
775792
}
776793

777794
const uptr Size = getSizeByClassId(ClassId);
778-
const u16 MaxCount = TransferBatch::getMaxCached(Size);
795+
const u16 MaxCount = CacheT::getMaxCached(Size);
779796
DCHECK_GT(MaxCount, 0U);
780797
// The maximum number of blocks we should carve in the region is dictated
781798
// by the maximum number of batches we want to fill, and the amount of

0 commit comments

Comments
 (0)