archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-07 23:04:33 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

slab fixes for 7.0-rc2

Browse source 
-----BEGIN PGP SIGNATURE-----
 
 iQFPBAABCAA5FiEEe7vIQRWZI0iWSE3xu+CwddJFiJoFAmmqujcbFIAAAAAABAAO
 bWFudTIsMi41KzEuMTIsMiwyAAoJELvgsHXSRYiakO0H/RtqrDjh7evZtXlOu5l2
 7cg2HKYeytPwlKbyerIZb7bt0rgBOVIugZWNswluvCXFWf8ypioBPUEKnkxuXtXd
 9+8pdt8GOdW6XwobvmnupEWeN7xXMygPtMABh9E5GX1flxha5DVlspL6m4RUadJ9
 Or6uo33NB9s5CQGFadb3CjTUnj5tlcKt48hvDisaxzjr1UaYrE9pauwL+UUB58Zi
 Xd38HET0a6mpBOsdPuzgrvGmtXL8dhwnNVY6aSMvpqeHY4w03iUXM08XdHbaqOW9
 X1RYzmZNOR1poxFTsnhVGPvRNuVTpktm7hqh0JnbdEHM8zF72ZmXS8OxciCGZE4H
 Zr4=
 =tIX2
 -----END PGP SIGNATURE-----

Merge tag 'slab-for-7.0-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

Pull slab fixes from Vlastimil Babka:

 - Fix for slab->stride truncation on 64k page systems due to short
   type. It was not due to races and lack of barriers in the end. (Harry
   Yoo)

 - Fix for severe performance regression due to unnecessary sheaf refill
   restrictions exposed by mempool allocation strategy. (Vlastimil
   Babka)

 - Stable fix for potential silent percpu sheaf flushing failures on
   PREEMPT_RT. (Vlastimil Babka)

* tag 'slab-for-7.0-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  mm/slab: change stride type from unsigned short to unsigned int
  mm/slab: allow sheaf refill if blocking is not allowed
  slab: distinguish lock and trylock for sheaf_flush_main()
This commit is contained in:
Linus Torvalds 2026-03-06 09:22:51 -08:00
commit 9a881ea3da
2 changed files with 52 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

10
mm/slab.h
View file

@ -59,7 +59,7 @@ struct freelist_counters {
* to save memory. In case ->stride field is not available,
* such optimizations are disabled.
*/
unsigned short stride;
unsigned int stride;
#endif
};
};
@ -559,20 +559,20 @@ static inline void put_slab_obj_exts(unsigned long obj_exts)
}
#ifdef CONFIG_64BIT
static inline void slab_set_stride(struct slab *slab, unsigned short stride)
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
{
slab->stride = stride;
}
static inline unsigned short slab_get_stride(struct slab *slab)
static inline unsigned int slab_get_stride(struct slab *slab)
{
return slab->stride;
}
#else
static inline void slab_set_stride(struct slab *slab, unsigned short stride)
static inline void slab_set_stride(struct slab *slab, unsigned int stride)
{
VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext));
}
static inline unsigned short slab_get_stride(struct slab *slab)
static inline unsigned int slab_get_stride(struct slab *slab)
{
return sizeof(struct slabobj_ext);
}

69
mm/slub.c
View file

@ -2858,19 +2858,19 @@ static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
* object pointers are moved to a on-stack array under the lock. To bound the
* stack usage, limit each batch to PCS_BATCH_MAX.
*
* returns true if at least partially flushed
* Must be called with s->cpu_sheaves->lock locked, returns with the lock
* unlocked.
*
* Returns how many objects are remaining to be flushed
*/
static bool sheaf_flush_main(struct kmem_cache *s)
static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s)
{
struct slub_percpu_sheaves *pcs;
unsigned int batch, remaining;
void *objects[PCS_BATCH_MAX];
struct slab_sheaf *sheaf;
bool ret = false;
next_batch:
if (!local_trylock(&s->cpu_sheaves->lock))
return ret;
lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
pcs = this_cpu_ptr(s->cpu_sheaves);
sheaf = pcs->main;
@ -2888,10 +2888,37 @@ next_batch:
stat_add(s, SHEAF_FLUSH, batch);
ret = true;
return remaining;
}
if (remaining)
goto next_batch;
static void sheaf_flush_main(struct kmem_cache *s)
{
unsigned int remaining;
do {
local_lock(&s->cpu_sheaves->lock);
remaining = __sheaf_flush_main_batch(s);
} while (remaining);
}
/*
* Returns true if the main sheaf was at least partially flushed.
*/
static bool sheaf_try_flush_main(struct kmem_cache *s)
{
unsigned int remaining;
bool ret = false;
do {
if (!local_trylock(&s->cpu_sheaves->lock))
return ret;
ret = true;
remaining = __sheaf_flush_main_batch(s);
} while (remaining);
return ret;
}
@ -4540,7 +4567,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
struct slab_sheaf *empty = NULL;
struct slab_sheaf *full;
struct node_barn *barn;
bool can_alloc;
bool allow_spin;
lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
@ -4561,8 +4588,9 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return NULL;
}
full = barn_replace_empty_sheaf(barn, pcs->main,
gfpflags_allow_spinning(gfp));
allow_spin = gfpflags_allow_spinning(gfp);
full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);
if (full) {
stat(s, BARN_GET);
@ -4572,9 +4600,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
stat(s, BARN_GET_FAIL);
can_alloc = gfpflags_allow_blocking(gfp);
if (can_alloc) {
if (allow_spin) {
if (pcs->spare) {
empty = pcs->spare;
pcs->spare = NULL;
@ -4584,8 +4610,9 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
}
local_unlock(&s->cpu_sheaves->lock);
pcs = NULL;
if (!can_alloc)
if (!allow_spin)
return NULL;
if (empty) {
@ -4605,11 +4632,8 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
if (!full)
return NULL;
/*
* we can reach here only when gfpflags_allow_blocking
* so this must not be an irq
*/
local_lock(&s->cpu_sheaves->lock);
if (!local_trylock(&s->cpu_sheaves->lock))
goto barn_put;
pcs = this_cpu_ptr(s->cpu_sheaves);
/*
@ -4640,6 +4664,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return pcs;
}
barn_put:
barn_put_full_sheaf(barn, full);
stat(s, BARN_PUT);
@ -5704,7 +5729,7 @@ alloc_empty:
if (put_fail)
stat(s, BARN_PUT_FAIL);
if (!sheaf_flush_main(s))
if (!sheaf_try_flush_main(s))
return NULL;
if (!local_trylock(&s->cpu_sheaves->lock))