Skip to content

Commit 24b0c43

Browse files
ChiaHungDuanChiaHungDuan
authored
Reapply "[scudo] Make local cache be agnostic to the type of node in … (#68633)
…f… (#68626) This reverts commit 8dd9615. 1. Fixed the ambiguous aliasing 2. Fixed the type conversion warning
1 parent 115b6a3 commit 24b0c43

File tree

6 files changed

+240
-187
lines changed

6 files changed

+240
-187
lines changed

‎compiler-rt/lib/scudo/standalone/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ if(ANDROID)
5656
endif()
5757

5858
set(SCUDO_HEADERS
59+
allocator_common.h
5960
allocator_config.h
6061
atomic_helpers.h
6162
bytemap.h

‎compiler-rt/lib/scudo/standalone/allocator_common.h

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
//===-- allocator_common.h --------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef SCUDO_ALLOCATOR_COMMON_H_
10+
#define SCUDO_ALLOCATOR_COMMON_H_
11+
12+
#include "common.h"
13+
#include "list.h"
14+
15+
namespace scudo {
16+
17+
template <class SizeClassAllocator> struct TransferBatch {
18+
typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
19+
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
20+
21+
static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
22+
void setFromArray(CompactPtrT *Array, u16 N) {
23+
DCHECK_LE(N, MaxNumCached);
24+
Count = N;
25+
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
26+
}
27+
void appendFromArray(CompactPtrT *Array, u16 N) {
28+
DCHECK_LE(N, MaxNumCached - Count);
29+
memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
30+
// u16 will be promoted to int by arithmetic type conversion.
31+
Count = static_cast<u16>(Count + N);
32+
}
33+
void appendFromTransferBatch(TransferBatch *B, u16 N) {
34+
DCHECK_LE(N, MaxNumCached - Count);
35+
DCHECK_GE(B->Count, N);
36+
// Append from the back of `B`.
37+
memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N);
38+
// u16 will be promoted to int by arithmetic type conversion.
39+
Count = static_cast<u16>(Count + N);
40+
B->Count = static_cast<u16>(B->Count - N);
41+
}
42+
void clear() { Count = 0; }
43+
void add(CompactPtrT P) {
44+
DCHECK_LT(Count, MaxNumCached);
45+
Batch[Count++] = P;
46+
}
47+
void moveToArray(CompactPtrT *Array) {
48+
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
49+
clear();
50+
}
51+
u16 getCount() const { return Count; }
52+
bool isEmpty() const { return Count == 0U; }
53+
CompactPtrT get(u16 I) const {
54+
DCHECK_LE(I, Count);
55+
return Batch[I];
56+
}
57+
TransferBatch *Next;
58+
59+
private:
60+
CompactPtrT Batch[MaxNumCached];
61+
u16 Count;
62+
};
63+
64+
// A BatchGroup is used to collect blocks. Each group has a group id to
65+
// identify the group kind of contained blocks.
66+
template <class SizeClassAllocator> struct BatchGroup {
67+
// `Next` is used by IntrusiveList.
68+
BatchGroup *Next;
69+
// The compact base address of each group
70+
uptr CompactPtrGroupBase;
71+
// Cache value of SizeClassAllocatorLocalCache::getMaxCached()
72+
u16 MaxCachedPerBatch;
73+
// Number of blocks pushed into this group. This is an increment-only
74+
// counter.
75+
uptr PushedBlocks;
76+
// This is used to track how many bytes are not in-use since last time we
77+
// tried to release pages.
78+
uptr BytesInBGAtLastCheckpoint;
79+
// Blocks are managed by TransferBatch in a list.
80+
SinglyLinkedList<TransferBatch<SizeClassAllocator>> Batches;
81+
};
82+
83+
} // namespace scudo
84+
85+
#endif // SCUDO_ALLOCATOR_COMMON_H_

‎compiler-rt/lib/scudo/standalone/local_cache.h

+19-91
Original file line numberDiff line numberDiff line change
@@ -22,74 +22,6 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
2222
typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
2323
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
2424

25-
struct TransferBatch {
26-
static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
27-
void setFromArray(CompactPtrT *Array, u16 N) {
28-
DCHECK_LE(N, MaxNumCached);
29-
Count = N;
30-
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
31-
}
32-
void appendFromArray(CompactPtrT *Array, u16 N) {
33-
DCHECK_LE(N, MaxNumCached - Count);
34-
memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
35-
// u16 will be promoted to int by arithmetic type conversion.
36-
Count = static_cast<u16>(Count + N);
37-
}
38-
void appendFromTransferBatch(TransferBatch *B, u16 N) {
39-
DCHECK_LE(N, MaxNumCached - Count);
40-
DCHECK_GE(B->Count, N);
41-
// Append from the back of `B`.
42-
memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N);
43-
// u16 will be promoted to int by arithmetic type conversion.
44-
Count = static_cast<u16>(Count + N);
45-
B->Count = static_cast<u16>(B->Count - N);
46-
}
47-
void clear() { Count = 0; }
48-
void add(CompactPtrT P) {
49-
DCHECK_LT(Count, MaxNumCached);
50-
Batch[Count++] = P;
51-
}
52-
void copyToArray(CompactPtrT *Array) const {
53-
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
54-
}
55-
u16 getCount() const { return Count; }
56-
bool isEmpty() const { return Count == 0U; }
57-
CompactPtrT get(u16 I) const {
58-
DCHECK_LE(I, Count);
59-
return Batch[I];
60-
}
61-
static u16 getMaxCached(uptr Size) {
62-
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
63-
}
64-
TransferBatch *Next;
65-
66-
private:
67-
CompactPtrT Batch[MaxNumCached];
68-
u16 Count;
69-
};
70-
71-
// A BatchGroup is used to collect blocks. Each group has a group id to
72-
// identify the group kind of contained blocks.
73-
struct BatchGroup {
74-
// `Next` is used by IntrusiveList.
75-
BatchGroup *Next;
76-
// The compact base address of each group
77-
uptr CompactPtrGroupBase;
78-
// Cache value of TransferBatch::getMaxCached()
79-
u16 MaxCachedPerBatch;
80-
// Number of blocks pushed into this group. This is an increment-only
81-
// counter.
82-
uptr PushedBlocks;
83-
// This is used to track how many bytes are not in-use since last time we
84-
// tried to release pages.
85-
uptr BytesInBGAtLastCheckpoint;
86-
// Blocks are managed by TransferBatch in a list.
87-
SinglyLinkedList<TransferBatch> Batches;
88-
};
89-
90-
static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch),
91-
"BatchGroup uses the same class size as TransferBatch");
92-
9325
void init(GlobalStats *S, SizeClassAllocator *A) {
9426
DCHECK(isEmpty());
9527
Stats.init();
@@ -151,7 +83,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
15183
}
15284

15385
void drain() {
154-
// Drain BatchClassId last as createBatch can refill it.
86+
// Drain BatchClassId last as it may be needed while draining normal blocks.
15587
for (uptr I = 0; I < NumClasses; ++I) {
15688
if (I == BatchClassId)
15789
continue;
@@ -163,19 +95,11 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
16395
DCHECK(isEmpty());
16496
}
16597

166-
TransferBatch *createBatch(uptr ClassId, void *B) {
167-
if (ClassId != BatchClassId)
168-
B = allocate(BatchClassId);
98+
void *getBatchClassBlock() {
99+
void *B = allocate(BatchClassId);
169100
if (UNLIKELY(!B))
170101
reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
171-
return reinterpret_cast<TransferBatch *>(B);
172-
}
173-
174-
BatchGroup *createGroup() {
175-
void *Ptr = allocate(BatchClassId);
176-
if (UNLIKELY(!Ptr))
177-
reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
178-
return reinterpret_cast<BatchGroup *>(Ptr);
102+
return B;
179103
}
180104

181105
LocalStats &getStats() { return Stats; }
@@ -203,6 +127,11 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
203127
Str->append(" No block is cached.\n");
204128
}
205129

130+
static u16 getMaxCached(uptr Size) {
131+
return Min(SizeClassMap::MaxNumCachedHint,
132+
SizeClassMap::getMaxCachedHint(Size));
133+
}
134+
206135
private:
207136
static const uptr NumClasses = SizeClassMap::NumClasses;
208137
static const uptr BatchClassId = SizeClassMap::BatchClassId;
@@ -211,7 +140,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
211140
u16 MaxCount;
212141
// Note: ClassSize is zero for the transfer batch.
213142
uptr ClassSize;
214-
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
143+
CompactPtrT Chunks[2 * SizeClassMap::MaxNumCachedHint];
215144
};
216145
PerClass PerClassArray[NumClasses] = {};
217146
LocalStats Stats;
@@ -228,7 +157,7 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
228157
for (uptr I = 0; I < NumClasses; I++) {
229158
PerClass *P = &PerClassArray[I];
230159
const uptr Size = SizeClassAllocator::getSizeByClassId(I);
231-
P->MaxCount = static_cast<u16>(2 * TransferBatch::getMaxCached(Size));
160+
P->MaxCount = static_cast<u16>(2 * getMaxCached(Size));
232161
if (I != BatchClassId) {
233162
P->ClassSize = Size;
234163
} else {
@@ -246,15 +175,14 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
246175

247176
NOINLINE bool refill(PerClass *C, uptr ClassId) {
248177
initCacheMaybe(C);
249-
TransferBatch *B = Allocator->popBatch(this, ClassId);
250-
if (UNLIKELY(!B))
251-
return false;
252-
DCHECK_GT(B->getCount(), 0);
253-
C->Count = B->getCount();
254-
B->copyToArray(C->Chunks);
255-
B->clear();
256-
destroyBatch(ClassId, B);
257-
return true;
178+
179+
// TODO(chiahungduan): Pass the max number cached for each size class.
180+
const u16 NumBlocksRefilled =
181+
Allocator->popBlocks(this, ClassId, C->Chunks);
182+
DCHECK_LE(NumBlocksRefilled,
183+
getMaxCached(SizeClassAllocator::getSizeByClassId(ClassId)));
184+
C->Count = static_cast<u16>(C->Count + NumBlocksRefilled);
185+
return NumBlocksRefilled != 0;
258186
}
259187

260188
NOINLINE void drain(PerClass *C, uptr ClassId) {

0 commit comments

Comments
 (0)