archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-08 02:44:41 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

slab: handle kmalloc sheaves bootstrap

Browse source 
Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
we can simply allow them in calculate_sizes() as they are created later
than KMALLOC_NORMAL caches and can allocate sheaves and barns from
those.

For KMALLOC_NORMAL caches we perform additional step after first
creating them without sheaves. Then bootstrap_cache_sheaves() simply
allocates and initializes barns and sheaves and finally sets
s->sheaf_capacity to make them actually used.

Afterwards the only caches left without sheaves (unless SLUB_TINY or
debugging is enabled) are kmem_cache and kmem_cache_node. These are only
used when creating or destroying other kmem_caches. Thus they are not
performance critical and we can simply leave it that way.

Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
This commit is contained in:
Vlastimil Babka 2026-01-23 07:52:47 +01:00
parent f1427a1d64
commit 913ffd3a1b
1 changed files with 84 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

88
mm/slub.c
View file

@ -2593,7 +2593,8 @@ static void *setup_object(struct kmem_cache *s, void *object)
return object;
}
static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
unsigned int capacity)
{
struct slab_sheaf *sheaf;
size_t sheaf_size;
@ -2611,7 +2612,7 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
if (s->flags & SLAB_KMALLOC)
gfp |= __GFP_NO_OBJ_EXT;
sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
sheaf_size = struct_size(sheaf, objects, capacity);
sheaf = kzalloc(sheaf_size, gfp);
if (unlikely(!sheaf))
@ -2624,6 +2625,12 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
return sheaf;
}
static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
gfp_t gfp)
{
return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);
}
static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
{
kfree(sheaf);
@ -8157,8 +8164,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
if (s->flags & SLAB_RECLAIM_ACCOUNT)
s->allocflags |= __GFP_RECLAIMABLE;
/* kmalloc caches need extra care to support sheaves */
if (!is_kmalloc_cache(s))
/*
* For KMALLOC_NORMAL caches we enable sheaves later by
* bootstrap_kmalloc_sheaves() to avoid recursion
*/
if (!is_kmalloc_normal(s))
s->sheaf_capacity = calculate_sheaf_capacity(s, args);
/*
@ -8653,6 +8663,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
return s;
}
/*
* Finish the sheaves initialization done normally by init_percpu_sheaves() and
* init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
* since sheaves and barns are allocated by kmalloc.
*/
static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
{
struct kmem_cache_args empty_args = {};
unsigned int capacity;
bool failed = false;
int node, cpu;
capacity = calculate_sheaf_capacity(s, &empty_args);
/* capacity can be 0 due to debugging or SLUB_TINY */
if (!capacity)
return;
for_each_node_mask(node, slab_nodes) {
struct node_barn *barn;
barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
if (!barn) {
failed = true;
goto out;
}
barn_init(barn);
get_node(s, node)->barn = barn;
}
for_each_possible_cpu(cpu) {
struct slub_percpu_sheaves *pcs;
pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
if (!pcs->main) {
failed = true;
break;
}
}
out:
/*
* It's still early in boot so treat this like same as a failure to
* create the kmalloc cache in the first place
*/
if (failed)
panic("Out of memory when creating kmem_cache %s\n", s->name);
s->sheaf_capacity = capacity;
}
static void __init bootstrap_kmalloc_sheaves(void)
{
enum kmalloc_cache_type type;
for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
if (kmalloc_caches[type][idx])
bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
}
}
}
void __init kmem_cache_init(void)
{
static __initdata struct kmem_cache boot_kmem_cache,
@ -8696,6 +8774,8 @@ void __init kmem_cache_init(void)
setup_kmalloc_cache_index_table();
create_kmalloc_caches();
bootstrap_kmalloc_sheaves();
/* Setup random freelists for each cache */
init_freelist_randomization();