aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
authorLinus Torvalds <torvalds@linux-foundation.org>2026-06-22 08:28:48 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-06-22 08:28:48 -0700
commit335c347686e76df9d2c7d7f61b5ea627a4c5cb4c (patch)
tree7a24a824aaffdd0945a9c8578aec1ce32e5be0f0 /mm
parent6e869de3a1b9ef9f096223e0e7f30c727de4f6bc (diff)
parent892a7864730775c3dbee2a39e9ead4fa8d4256e7 (diff)
downloadath-335c347686e76df9d2c7d7f61b5ea627a4c5cb4c.tar.gz
Merge tag 'slab-for-7.2-part2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab
Pull more slab updates from Vlastimil Babka: - Introduce and wire up a new alloc_flags parameter for modifying slab-specific behavior without adding or reusing gfp flags. Also introduce slab_alloc_context to keep function parameter bloat in check. Both are similar to what the page allocator does. kmalloc_flags() exposes alloc_flags for mm-internal users. - SLAB_ALLOC_NOLOCK flag is used to implement kmalloc_nolock() behavior without relying on lack of __GFP_RECLAIM, which caused false positives with workarounds like fd3634312a04 ("debugobject: Make it work with deferred page initialization - again"). - SLAB_ALLOC_NO_RECURSE replaces __GFP_NO_OBJ_EXT, which could have been removed, but pending memory allocation profiling changes in mm tree have grown a new user - there is however a work ongoing to replace that too, so __GFP_NO_OBJ_EXT should eventually be removed. (Vlastimil Babka) - Add kmem_buckets_alloc_track_caller() with a user to be added in the net tree (Pedro Falcato) - Fixes for kernel-doc and slabinfo (Randy Dunlap, Yichong Chen) * tag 'slab-for-7.2-part2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab: tools/mm/slabinfo: fix total_objects attribute name slab: recognize @GFP parameter as optional in kernel-doc mm/slab: add a node-track-caller variant for kmem buckets allocation mm/slab: replace __GFP_NO_OBJ_EXT with SLAB_ALLOC_NO_RECURSE for sheaves mm/slab: remove __GFP_NO_OBJ_EXT usage from alloc_slab_obj_exts() mm/slab: introduce kmalloc_flags() mm/slab: allow __GFP_NOMEMALLOC and __GFP_NOWARN for kmalloc_nolock() mm/slab: pass slab_alloc_context to __do_kmalloc_node() mm/slab: allow kmem_cache_alloc_bulk() with any gfp flags mm/slab: replace slab_alloc_node() parameters with slab_alloc_context mm/slab: pass alloc_flags through slab_post_alloc_hook() chain mm/slab: pass alloc_flags to new slab allocation mm/slab: add alloc_flags to slab_alloc_context mm/slab: replace struct partial_context with slab_alloc_context mm/slab: introduce alloc_flags and SLAB_ALLOC_NOLOCK mm/slab: introduce slab_alloc_context mm/slab: stop inlining __slab_alloc_node() mm/slab: do not init any kfence objects on allocation
Diffstat (limited to 'mm')
-rw-r--r--mm/kfence/core.c2
-rw-r--r--mm/memcontrol.c5
-rw-r--r--mm/slab.h29
-rw-r--r--mm/slub.c488
4 files changed, 327 insertions, 197 deletions
diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index ee6ae01de5aef..6577bd76954eb 100644
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -505,7 +505,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
/*
* We check slab_want_init_on_alloc() ourselves, rather than letting
- * SL*B do the initialization, as otherwise we might overwrite KFENCE's
+ * slab do the initialization, as otherwise it might overwrite KFENCE's
* redzone.
*/
if (unlikely(slab_want_init_on_alloc(gfp, cache)))
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 56cd4af082326..6dc4888a90f3f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3535,7 +3535,8 @@ static inline size_t obj_full_size(struct kmem_cache *s)
}
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
- gfp_t flags, size_t size, void **p)
+ gfp_t flags, unsigned int slab_alloc_flags,
+ size_t size, void **p)
{
size_t obj_size = obj_full_size(s);
struct obj_cgroup *objcg;
@@ -3583,7 +3584,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
slab = virt_to_slab(p[i]);
if (!slab_obj_exts(slab) &&
- alloc_slab_obj_exts(slab, s, flags, false)) {
+ alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) {
continue;
}
diff --git a/mm/slab.h b/mm/slab.h
index 1bf9c3021ae3d..281a65233795d 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -11,11 +11,35 @@
#include <linux/memcontrol.h>
#include <linux/kfence.h>
#include <linux/kasan.h>
+#include <linux/slab.h>
/*
* Internal slab definitions
*/
+/* slab's alloc_flags definitions */
+#define SLAB_ALLOC_DEFAULT 0x00 /* no flags */
+#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */
+#define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */
+#define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */
+
+static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
+{
+ return !(alloc_flags & SLAB_ALLOC_NOLOCK);
+}
+
+void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
+ unsigned int alloc_flags, int node)
+ __assume_kmalloc_alignment __alloc_size(1);
+
+static __always_inline __alloc_size(1) void *_kmalloc_flags_noprof(size_t size,
+ gfp_t flags, unsigned int alloc_flags, int node, kmalloc_token_t token)
+{
+ return __kmalloc_flags_noprof(PASS_TOKEN_PARAMS(size, token), flags, alloc_flags, node);
+}
+#define kmalloc_flags_noprof(...) _kmalloc_flags_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
+#define kmalloc_flags(...) alloc_hooks(kmalloc_flags_noprof(__VA_ARGS__))
+
#ifdef CONFIG_64BIT
# ifdef system_has_cmpxchg128
# define system_has_freelist_aba() system_has_cmpxchg128()
@@ -603,7 +627,7 @@ static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
}
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
- gfp_t gfp, bool new_slab);
+ gfp_t gfp, unsigned int alloc_flags);
#else /* CONFIG_SLAB_OBJ_EXT */
@@ -633,7 +657,8 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
#ifdef CONFIG_MEMCG
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
- gfp_t flags, size_t size, void **p);
+ gfp_t flags, unsigned int slab_alloc_flags,
+ size_t size, void **p);
void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
void **p, int objects, unsigned long obj_exts);
#endif
diff --git a/mm/slub.c b/mm/slub.c
index f87e693aca5d4..9ec774dc70096 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -213,10 +213,12 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
static DEFINE_STATIC_KEY_FALSE(strict_numa);
#endif
-/* Structure holding parameters for get_from_partial() call chain */
-struct partial_context {
- gfp_t flags;
- unsigned int orig_size;
+/* Structure holding extra parameters for slab allocations */
+struct slab_alloc_context {
+ unsigned long caller_addr;
+ size_t orig_size;
+ unsigned int alloc_flags;
+ struct list_lru *lru;
};
/* Structure holding parameters for get_partial_node_bulk() */
@@ -2045,12 +2047,16 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
#endif /* CONFIG_SLUB_DEBUG */
/*
- * The allocated objcg pointers array is not accounted directly.
+ * The allocated objcg pointers array or sheaf is not accounted directly.
* Moreover, it should not come from DMA buffer and is not readily
- * reclaimable. So those GFP bits should be masked off.
+ * reclaimable. Node restriction for the parent allocation also should
+ * not apply to the slab's internal objects, as well as __GFP_COMP used
+ * for new slab allocations.
+ * So those GFP bits should be masked off.
*/
#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \
- __GFP_ACCOUNT | __GFP_NOFAIL)
+ __GFP_ACCOUNT | __GFP_NOFAIL | \
+ __GFP_THISNODE | __GFP_COMP)
#ifdef CONFIG_SLAB_OBJ_EXT
@@ -2154,10 +2160,11 @@ static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
}
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
- gfp_t gfp, bool new_slab)
+ gfp_t gfp, unsigned int alloc_flags)
{
- bool allow_spin = gfpflags_allow_spinning(gfp);
+ const bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
unsigned int objects = objs_per_slab(s, slab);
+ bool new_slab = alloc_flags & SLAB_ALLOC_NEW_SLAB;
unsigned long new_exts;
unsigned long old_exts;
struct slabobj_ext *vec;
@@ -2165,21 +2172,13 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp &= ~OBJCGS_CLEAR_MASK;
/* Prevent recursive extension vector allocation */
- gfp |= __GFP_NO_OBJ_EXT;
+ alloc_flags |= SLAB_ALLOC_NO_RECURSE;
+ alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
sz = obj_exts_alloc_size(s, slab, gfp);
- /*
- * Note that allow_spin may be false during early boot and its
- * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting
- * architectures with cmpxchg16b, early obj_exts will be missing for
- * very early allocations on those.
- */
- if (unlikely(!allow_spin))
- vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT,
- slab_nid(slab));
- else
- vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab));
+ /* This will use kmalloc_nolock() if alloc_flags say so */
+ vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));
if (!vec) {
/*
@@ -2255,7 +2254,7 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
}
/*
- * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its
+ * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its
* corresponding extension will be NULL. alloc_tag_sub() will throw a
* warning if slab has extensions but the extension of an object is
* NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
@@ -2330,7 +2329,7 @@ static inline void init_slab_obj_exts(struct slab *slab)
}
static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
- gfp_t gfp, bool new_slab)
+ gfp_t gfp, unsigned int alloc_flags)
{
return 0;
}
@@ -2350,10 +2349,10 @@ static inline void alloc_slab_obj_exts_early(struct kmem_cache *s,
static inline unsigned long
prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
- gfp_t flags, void *p)
+ gfp_t flags, unsigned int alloc_flags, void *p)
{
if (!slab_obj_exts(slab) &&
- alloc_slab_obj_exts(slab, s, flags, false)) {
+ alloc_slab_obj_exts(slab, s, flags, alloc_flags)) {
pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
__func__, s->name);
return 0;
@@ -2365,7 +2364,8 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
/* Should be called only if mem_alloc_profiling_enabled() */
static noinline void
-__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+ unsigned int alloc_flags)
{
unsigned long obj_exts;
struct slabobj_ext *obj_ext;
@@ -2377,11 +2377,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
return;
- if (flags & __GFP_NO_OBJ_EXT)
+ if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
return;
slab = virt_to_slab(object);
- obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, object);
+ obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object);
/*
* Currently obj_exts is used only for allocation profiling.
* If other users appear then mem_alloc_profiling_enabled()
@@ -2400,10 +2400,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
}
static inline void
-alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+ unsigned int alloc_flags)
{
if (mem_alloc_profiling_enabled())
- __alloc_tagging_slab_alloc_hook(s, object, flags);
+ __alloc_tagging_slab_alloc_hook(s, object, flags, alloc_flags);
}
/* Should be called only if mem_alloc_profiling_enabled() */
@@ -2442,7 +2443,8 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
#else /* CONFIG_MEM_ALLOC_PROFILING */
static inline void
-alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+ unsigned int alloc_flags)
{
}
@@ -2460,8 +2462,9 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
static __fastpath_inline
-bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
- gfp_t flags, size_t size, void **p)
+bool memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
+ size_t size, void **p,
+ const struct slab_alloc_context *ac)
{
if (likely(!memcg_kmem_online()))
return true;
@@ -2469,7 +2472,8 @@ bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
return true;
- if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p)))
+ if (likely(__memcg_slab_post_alloc_hook(s, ac->lru, flags,
+ ac->alloc_flags, size, p)))
return true;
if (likely(size == 1)) {
@@ -2557,14 +2561,15 @@ bool memcg_slab_post_charge(void *p, gfp_t flags)
put_slab_obj_exts(obj_exts);
}
- return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p);
+ return __memcg_slab_post_alloc_hook(s, NULL, flags, SLAB_ALLOC_DEFAULT,
+ 1, &p);
}
#else /* CONFIG_MEMCG */
static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
- struct list_lru *lru,
- gfp_t flags, size_t size,
- void **p)
+ gfp_t flags,
+ size_t size, void **p,
+ const struct slab_alloc_context *ac)
{
return true;
}
@@ -2757,7 +2762,7 @@ static inline void *setup_object(struct kmem_cache *s, void *object)
}
static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
- unsigned int capacity)
+ unsigned int alloc_flags, unsigned int capacity)
{
struct slab_sheaf *sheaf;
size_t sheaf_size;
@@ -2768,10 +2773,10 @@ static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
* bucket)
*/
if (s->flags & SLAB_KMALLOC)
- gfp |= __GFP_NO_OBJ_EXT;
+ alloc_flags |= SLAB_ALLOC_NO_RECURSE;
sheaf_size = struct_size(sheaf, objects, capacity);
- sheaf = kzalloc(sheaf_size, gfp);
+ sheaf = kmalloc_flags(sheaf_size, gfp | __GFP_ZERO, alloc_flags, NUMA_NO_NODE);
if (unlikely(!sheaf))
return NULL;
@@ -2784,20 +2789,20 @@ static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
}
static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
- gfp_t gfp)
+ gfp_t gfp, unsigned int alloc_flags)
{
- if (gfp & __GFP_NO_OBJ_EXT)
+ if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
return NULL;
gfp &= ~OBJCGS_CLEAR_MASK;
- return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);
+ return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity);
}
static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
{
/*
- * If the sheaf was created with __GFP_NO_OBJ_EXT flag then its
+ * If the sheaf was created with SLAB_ALLOC_NO_RECURSE flag then its
* corresponding extension is NULL and alloc_tag_sub() will throw a
* warning, therefore replace NULL with CODETAG_EMPTY to indicate
* that the extension for this sheaf is expected to be NULL.
@@ -3351,12 +3356,14 @@ static inline void init_freelist_randomization(void) { }
#endif /* CONFIG_SLAB_FREELIST_RANDOM */
static __always_inline void account_slab(struct slab *slab, int order,
- struct kmem_cache *s, gfp_t gfp)
+ struct kmem_cache *s, gfp_t gfp,
+ unsigned int alloc_flags)
{
if (memcg_kmem_online() &&
(s->flags & SLAB_ACCOUNT) &&
!slab_obj_exts(slab))
- alloc_slab_obj_exts(slab, s, gfp, true);
+ alloc_slab_obj_exts(slab, s, gfp,
+ alloc_flags | SLAB_ALLOC_NEW_SLAB);
mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
PAGE_SIZE << order);
@@ -3377,9 +3384,10 @@ static __always_inline void unaccount_slab(struct slab *slab, int order,
}
/* Allocate and initialize a slab without building its freelist. */
-static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
+static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
+ unsigned int alloc_flags, int node)
{
- bool allow_spin = gfpflags_allow_spinning(flags);
+ bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
struct slab *slab;
struct kmem_cache_order_objects oo = s->oo;
gfp_t alloc_gfp;
@@ -3397,10 +3405,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
- /*
- * __GFP_RECLAIM could be cleared on the first allocation attempt,
- * so pass allow_spin flag directly.
- */
slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin);
if (unlikely(!slab)) {
oo = s->min;
@@ -3432,20 +3436,22 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
* to prevent the array from being overwritten.
*/
alloc_slab_obj_exts_early(s, slab);
- account_slab(slab, oo_order(oo), s, flags);
+ account_slab(slab, oo_order(oo), s, flags, alloc_flags);
return slab;
}
-static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+static struct slab *new_slab(struct kmem_cache *s, gfp_t flags,
+ unsigned int alloc_flags, int node)
{
if (unlikely(flags & GFP_SLAB_BUG_MASK))
flags = kmalloc_fix_flags(flags);
WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
- return allocate_slab(s,
- flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
+ flags &= GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK;
+
+ return allocate_slab(s, flags, alloc_flags, node);
}
static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)
@@ -3687,8 +3693,9 @@ static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab,
* and put the slab to the partial (or full) list.
*/
static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
- int orig_size, bool allow_spin)
+ const struct slab_alloc_context *ac)
{
+ bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
struct kmem_cache_node *n;
struct slab_obj_iter iter;
bool needs_add_partial;
@@ -3705,7 +3712,7 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
/* alloc_debug_processing() always expects a valid freepointer */
set_freepointer(s, object, slab->freelist);
- if (!alloc_debug_processing(s, slab, object, orig_size)) {
+ if (!alloc_debug_processing(s, slab, object, ac->orig_size)) {
/*
* It's not really expected that this would fail on a
* freshly allocated slab, but a concurrent memory
@@ -3818,7 +3825,8 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
*/
static void *get_from_partial_node(struct kmem_cache *s,
struct kmem_cache_node *n,
- struct partial_context *pc)
+ gfp_t gfp_flags,
+ const struct slab_alloc_context *ac)
{
struct slab *slab, *slab2;
unsigned long flags;
@@ -3833,7 +3841,7 @@ static void *get_from_partial_node(struct kmem_cache *s,
if (!n || !n->nr_partial)
return NULL;
- if (gfpflags_allow_spinning(pc->flags))
+ if (alloc_flags_allow_spinning(ac->alloc_flags))
spin_lock_irqsave(&n->list_lock, flags);
else if (!spin_trylock_irqsave(&n->list_lock, flags))
return NULL;
@@ -3841,12 +3849,12 @@ static void *get_from_partial_node(struct kmem_cache *s,
struct freelist_counters old, new;
- if (!pfmemalloc_match(slab, pc->flags))
+ if (!pfmemalloc_match(slab, gfp_flags))
continue;
if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
object = alloc_single_from_partial(s, n, slab,
- pc->orig_size);
+ ac->orig_size);
if (object)
break;
continue;
@@ -3880,15 +3888,16 @@ static void *get_from_partial_node(struct kmem_cache *s,
/*
* Get an object from somewhere. Search in increasing NUMA distances.
*/
-static void *get_from_any_partial(struct kmem_cache *s, struct partial_context *pc)
+static void *get_from_any_partial(struct kmem_cache *s, gfp_t gfp_flags,
+ const struct slab_alloc_context *ac)
{
#ifdef CONFIG_NUMA
struct zonelist *zonelist;
struct zoneref *z;
struct zone *zone;
- enum zone_type highest_zoneidx = gfp_zone(pc->flags);
+ enum zone_type highest_zoneidx = gfp_zone(gfp_flags);
unsigned int cpuset_mems_cookie;
- bool allow_spin = gfpflags_allow_spinning(pc->flags);
+ bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
/*
* The defrag ratio allows a configuration of the tradeoffs between
@@ -3922,16 +3931,17 @@ static void *get_from_any_partial(struct kmem_cache *s, struct partial_context *
if (allow_spin)
cpuset_mems_cookie = read_mems_allowed_begin();
- zonelist = node_zonelist(mempolicy_slab_node(), pc->flags);
+ zonelist = node_zonelist(mempolicy_slab_node(), gfp_flags);
for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
struct kmem_cache_node *n;
n = get_node(s, zone_to_nid(zone));
- if (n && cpuset_zone_allowed(zone, pc->flags) &&
+ if (n && cpuset_zone_allowed(zone, gfp_flags) &&
n->nr_partial > s->min_partial) {
- void *object = get_from_partial_node(s, n, pc);
+ void *object = get_from_partial_node(s, n,
+ gfp_flags, ac);
if (object) {
/*
@@ -3953,8 +3963,8 @@ static void *get_from_any_partial(struct kmem_cache *s, struct partial_context *
/*
* Get an object from a partial slab
*/
-static void *get_from_partial(struct kmem_cache *s, int node,
- struct partial_context *pc)
+static void *get_from_partial(struct kmem_cache *s, int node, gfp_t flags,
+ const struct slab_alloc_context *ac)
{
int searchnode = node;
void *object;
@@ -3962,11 +3972,11 @@ static void *get_from_partial(struct kmem_cache *s, int node,
if (node == NUMA_NO_NODE)
searchnode = numa_mem_id();
- object = get_from_partial_node(s, get_node(s, searchnode), pc);
- if (object || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE)))
+ object = get_from_partial_node(s, get_node(s, searchnode), flags, ac);
+ if (object || (node != NUMA_NO_NODE && (flags & __GFP_THISNODE)))
return object;
- return get_from_any_partial(s, pc);
+ return get_from_any_partial(s, flags, ac);
}
static bool has_pcs_used(int cpu, struct kmem_cache *s)
@@ -4444,45 +4454,41 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,
* slab.
*/
static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
- unsigned long addr, unsigned int orig_size)
+ const struct slab_alloc_context *ac)
{
- bool allow_spin = gfpflags_allow_spinning(gfpflags);
+ bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
+ gfp_t trynode_flags;
void *object;
struct slab *slab;
- struct partial_context pc;
bool try_thisnode = true;
stat(s, ALLOC_SLOWPATH);
new_objects:
- pc.flags = gfpflags;
+ trynode_flags = gfpflags;
/*
* When a preferred node is indicated but no __GFP_THISNODE
*
* 1) try to get a partial slab from target node only by having
- * __GFP_THISNODE in pc.flags for get_from_partial()
+ * __GFP_THISNODE in trynode_flags for get_from_partial()
* 2) if 1) failed, try to allocate a new slab from target node with
- * GPF_NOWAIT | __GFP_THISNODE opportunistically
+ * (at most) GFP_NOWAIT | __GFP_THISNODE opportunistically
* 3) if 2) failed, retry with original gfpflags which will allow
* get_from_partial() try partial lists of other nodes before
* potentially allocating new page from other nodes
*/
if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
&& try_thisnode)) {
- if (unlikely(!allow_spin))
- /* Do not upgrade gfp to NOWAIT from more restrictive mode */
- pc.flags = gfpflags | __GFP_THISNODE;
- else
- pc.flags = GFP_NOWAIT | __GFP_THISNODE;
+ trynode_flags &= GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_ACCOUNT;
+ trynode_flags |= __GFP_NOWARN | __GFP_THISNODE;
}
- pc.orig_size = orig_size;
- object = get_from_partial(s, node, &pc);
+ object = get_from_partial(s, node, trynode_flags, ac);
if (object)
goto success;
- slab = new_slab(s, pc.flags, node);
+ slab = new_slab(s, trynode_flags, ac->alloc_flags, node);
if (unlikely(!slab)) {
if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
@@ -4497,7 +4503,7 @@ new_objects:
stat(s, ALLOC_SLAB);
if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
- object = alloc_single_from_new_slab(s, slab, orig_size, allow_spin);
+ object = alloc_single_from_new_slab(s, slab, ac);
if (likely(object))
goto success;
@@ -4515,13 +4521,13 @@ new_objects:
success:
if (kmem_cache_debug_flags(s, SLAB_STORE_USER))
- set_track(s, object, TRACK_ALLOC, addr, gfpflags);
+ set_track(s, object, TRACK_ALLOC, ac->caller_addr, gfpflags);
return object;
}
-static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
- gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
+static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
+ const struct slab_alloc_context *ac)
{
void *object;
@@ -4546,7 +4552,7 @@ static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
}
#endif
- object = ___slab_alloc(s, gfpflags, node, addr, orig_size);
+ object = ___slab_alloc(s, gfpflags, node, ac);
return object;
}
@@ -4565,14 +4571,13 @@ struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
}
static __fastpath_inline
-bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
- gfp_t flags, size_t size, void **p, bool init,
- unsigned int orig_size)
+bool slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size,
+ void **p, const struct slab_alloc_context *ac)
{
+ bool init = slab_want_init_on_alloc(flags, s);
unsigned int zero_size = s->object_size;
- bool kasan_init = init;
- size_t i;
gfp_t init_flags = flags & gfp_allowed_mask;
+ bool kasan_init = false;
/*
* For kmalloc object, the allocated size (object_size) can be larger
@@ -4586,39 +4591,44 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
* orig_size if we track it.
*/
if (slub_debug_orig_size(s))
- zero_size = orig_size;
+ zero_size = ac->orig_size;
/*
- * When slab_debug is enabled, avoid memory initialization integrated
- * into KASAN and instead zero out the memory via the memset below with
- * the proper size. Otherwise, KASAN might overwrite SLUB redzones and
- * cause false-positive reports. This does not lead to a performance
+ * ARM64 can set memory tags and zero the memory using a single
+ * instruction. Since HW_TAGS KASAN uses that while tagging the object,
+ * separate zeroing is unnecessary.
+ *
+ * However, KASAN never zeroes memory when slab_debug is enabled to
+ * avoid overwriting SLUB redzones. This does not lead to a performance
* penalty on production builds, as slab_debug is not intended to be
* enabled there.
*/
- if (__slub_debug_enabled())
- kasan_init = false;
+ if (kasan_has_integrated_init() && !__slub_debug_enabled()) {
+ kasan_init = init;
+ init = false;
+ }
- /*
- * As memory initialization might be integrated into KASAN,
- * kasan_slab_alloc and initialization memset must be
- * kept together to avoid discrepancies in behavior.
- *
- * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
- */
- for (i = 0; i < size; i++) {
+ for (size_t i = 0; i < size; i++) {
p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init);
- if (p[i] && init && (!kasan_init ||
- !kasan_has_integrated_init()))
+
+ /*
+ * memset and hooks come after KASAN as p[i] might get tagged
+ *
+ * kfence zeroes the object instead of SLUB to avoid overwriting
+ * its own redzone starting at orig_size, which could happen
+ * with SLUB zeroing full s->object_size
+ */
+ if (init && p[i] && !is_kfence_address(p[i]))
memset(p[i], 0, zero_size);
- if (gfpflags_allow_spinning(flags))
+
+ if (alloc_flags_allow_spinning(ac->alloc_flags))
kmemleak_alloc_recursive(p[i], s->object_size, 1,
s->flags, init_flags);
kmsan_slab_alloc(s, p[i], init_flags);
- alloc_tagging_slab_alloc_hook(s, p[i], flags);
+ alloc_tagging_slab_alloc_hook(s, p[i], flags, ac->alloc_flags);
}
- return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
+ return memcg_slab_post_alloc_hook(s, flags, size, p, ac);
}
/*
@@ -4630,7 +4640,8 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
* unlocked.
*/
static struct slub_percpu_sheaves *
-__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp)
+__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
+ gfp_t gfp, unsigned int alloc_flags)
{
struct slab_sheaf *empty = NULL;
struct slab_sheaf *full;
@@ -4656,7 +4667,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return NULL;
}
- allow_spin = gfpflags_allow_spinning(gfp);
+ allow_spin = alloc_flags_allow_spinning(alloc_flags);
full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);
@@ -4684,7 +4695,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return NULL;
if (!empty) {
- empty = alloc_empty_sheaf(s, gfp);
+ empty = alloc_empty_sheaf(s, gfp, alloc_flags);
if (!empty)
return NULL;
}
@@ -4742,7 +4753,7 @@ barn_put:
}
static __fastpath_inline
-void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
+void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node)
{
struct slub_percpu_sheaves *pcs;
bool node_requested;
@@ -4787,7 +4798,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
pcs = this_cpu_ptr(s->cpu_sheaves);
if (unlikely(pcs->main->size == 0)) {
- pcs = __pcs_replace_empty_main(s, pcs, gfp);
+ pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags);
if (unlikely(!pcs))
return NULL;
}
@@ -4817,8 +4828,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
}
static __fastpath_inline
-unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size,
- void **p)
+unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
{
struct slub_percpu_sheaves *pcs;
struct slab_sheaf *main;
@@ -4853,7 +4863,7 @@ next_batch:
}
full = barn_replace_empty_sheaf(barn, pcs->main,
- gfpflags_allow_spinning(gfp));
+ /* allow_spin = */ true);
if (full) {
stat(s, BARN_GET);
@@ -4907,44 +4917,46 @@ do_alloc:
*
* Otherwise we can simply pick the next object from the lockless free list.
*/
-static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
- gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
+static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
+ gfp_t gfpflags, int node, const struct slab_alloc_context *ac)
{
void *object;
- bool init = false;
s = slab_pre_alloc_hook(s, gfpflags);
if (unlikely(!s))
return NULL;
- object = kfence_alloc(s, orig_size, gfpflags);
+ object = kfence_alloc(s, ac->orig_size, gfpflags);
if (unlikely(object))
goto out;
- object = alloc_from_pcs(s, gfpflags, node);
+ object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
- if (!object)
- object = __slab_alloc_node(s, gfpflags, node, addr, orig_size);
+ if (unlikely(!object))
+ object = __slab_alloc_node(s, gfpflags, node, ac);
maybe_wipe_obj_freeptr(s, object);
- init = slab_want_init_on_alloc(gfpflags, s);
out:
/*
- * When init equals 'true', like for kzalloc() family, only
- * @orig_size bytes might be zeroed instead of s->object_size
* In case this fails due to memcg_slab_post_alloc_hook(),
* object is set to NULL
*/
- slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size);
+ slab_post_alloc_hook(s, gfpflags, 1, &object, ac);
return object;
}
void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
{
- void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
- s->object_size);
+ void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
@@ -4955,8 +4967,15 @@ EXPORT_SYMBOL(kmem_cache_alloc_noprof);
void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags)
{
- void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
- s->object_size);
+ void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ .lru = lru,
+ };
+
+ ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
@@ -4988,7 +5007,14 @@ EXPORT_SYMBOL(kmem_cache_charge);
*/
void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
{
- void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
+ void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ ret = slab_alloc_node(s, gfpflags, node, &ac);
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
@@ -5042,7 +5068,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
if (unlikely(size > s->sheaf_capacity)) {
- sheaf = __alloc_empty_sheaf(s, gfp, size);
+ sheaf = __alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT, size);
if (!sheaf)
return NULL;
@@ -5087,7 +5113,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
if (!sheaf)
- sheaf = alloc_empty_sheaf(s, gfp);
+ sheaf = alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT);
if (sheaf) {
sheaf->capacity = s->sheaf_capacity;
@@ -5231,7 +5257,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
struct slab_sheaf *sheaf)
{
void *ret = NULL;
- bool init;
+ const struct slab_alloc_context ac = {
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
if (sheaf->size == 0)
goto out;
@@ -5241,10 +5270,8 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
if (likely(!ret))
ret = sheaf->objects[--sheaf->size];
- init = slab_want_init_on_alloc(gfp, s);
-
/* add __GFP_NOFAIL to force successful memcg charging */
- slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size);
+ slab_post_alloc_hook(s, gfp | __GFP_NOFAIL, 1, &ret, &ac);
out:
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE);
@@ -5312,15 +5339,16 @@ void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
EXPORT_SYMBOL(__kmalloc_large_node_noprof);
static __always_inline
-void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node,
- unsigned long caller, kmalloc_token_t token)
+void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node,
+ kmalloc_token_t token, const struct slab_alloc_context *ac)
{
+ const size_t size = ac->orig_size;
struct kmem_cache *s;
void *ret;
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = __kmalloc_large_node_noprof(size, flags, node);
- trace_kmalloc(caller, ret, size,
+ trace_kmalloc(ac->caller_addr, ret, size,
PAGE_SIZE << get_order(size), flags, node);
return ret;
}
@@ -5330,35 +5358,49 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node,
s = kmalloc_slab(size, b, flags, token);
- ret = slab_alloc_node(s, NULL, flags, node, caller, size);
+ ret = slab_alloc_node(s, flags, node, ac);
ret = kasan_kmalloc(s, ret, size, flags);
- trace_kmalloc(caller, ret, size, s->size, flags, node);
+ trace_kmalloc(ac->caller_addr, ret, size, s->size, flags, node);
return ret;
}
void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
{
- return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
- _RET_IP_, PASS_TOKEN_PARAM(token));
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node,
+ PASS_TOKEN_PARAM(token), &ac);
}
EXPORT_SYMBOL(__kmalloc_node_noprof);
void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
{
- return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_,
- PASS_TOKEN_PARAM(token));
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ return __do_kmalloc_node(NULL, flags, NUMA_NO_NODE,
+ PASS_TOKEN_PARAM(token), &ac);
}
EXPORT_SYMBOL(__kmalloc_noprof);
-void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node)
+static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags,
+ int node, const struct slab_alloc_context *ac)
{
- gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags;
- size_t orig_size = size;
struct kmem_cache *s;
bool can_retry = true;
void *ret;
+ VM_WARN_ON_ONCE(alloc_flags_allow_spinning(ac->alloc_flags));
VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO |
- __GFP_NO_OBJ_EXT));
+ __GFP_NOWARN | __GFP_NOMEMALLOC));
+
+ gfp_flags |= __GFP_NOWARN | __GFP_NOMEMALLOC;
if (unlikely(!size))
return ZERO_SIZE_PTR;
@@ -5377,7 +5419,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in
retry:
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return NULL;
- s = kmalloc_slab(size, NULL, alloc_gfp, PASS_TOKEN_PARAM(token));
+ s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token));
if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
/*
@@ -5391,7 +5433,7 @@ retry:
*/
return NULL;
- ret = alloc_from_pcs(s, alloc_gfp, node);
+ ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
if (ret)
goto success;
@@ -5401,7 +5443,7 @@ retry:
* kfence_alloc. Hence call __slab_alloc_node() (at most twice)
* and slab_post_alloc_hook() directly.
*/
- ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, orig_size);
+ ret = __slab_alloc_node(s, gfp_flags, node, ac);
/*
* It's possible we failed due to trylock as we preempted someone with
@@ -5414,8 +5456,8 @@ retry:
size = s->object_size + 1;
/*
* Another alternative is to
- * if (memcg) alloc_gfp &= ~__GFP_ACCOUNT;
- * else if (!memcg) alloc_gfp |= __GFP_ACCOUNT;
+ * if (memcg) gfp_flags &= ~__GFP_ACCOUNT;
+ * else if (!memcg) gfp_flags |= __GFP_ACCOUNT;
* to retry from bucket of the same size.
*/
can_retry = false;
@@ -5424,27 +5466,49 @@ retry:
success:
maybe_wipe_obj_freeptr(s, ret);
- slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret,
- slab_want_init_on_alloc(alloc_gfp, s), orig_size);
+ slab_post_alloc_hook(s, gfp_flags, 1, &ret, ac);
- ret = kasan_kmalloc(s, ret, orig_size, alloc_gfp);
+ ret = kasan_kmalloc(s, ret, ac->orig_size, gfp_flags);
return ret;
}
+
+void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node)
+{
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_NOLOCK,
+ };
+
+ return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token),
+ gfp_flags, node, &ac);
+}
EXPORT_SYMBOL_GPL(_kmalloc_nolock_noprof);
void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags,
int node, unsigned long caller)
{
- return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
- caller, PASS_TOKEN_PARAM(token));
+ const struct slab_alloc_context ac = {
+ .caller_addr = caller,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+ return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node,
+ PASS_TOKEN_PARAM(token), &ac);
}
EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof);
void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
- void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
- _RET_IP_, size);
+ void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
@@ -5456,7 +5520,14 @@ EXPORT_SYMBOL(__kmalloc_cache_noprof);
void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
int node, size_t size)
{
- void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
+ void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
+
+ ret = slab_alloc_node(s, gfpflags, node, &ac);
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
@@ -5465,6 +5536,30 @@ void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
}
EXPORT_SYMBOL(__kmalloc_cache_node_noprof);
+/*
+ * The only version of kmalloc_node() that takes alloc_flags and thus can
+ * determine on its own whether to handle the allocation via kmalloc_nolock() or
+ * normally
+ */
+void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
+ unsigned int alloc_flags, int node)
+{
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = alloc_flags,
+ };
+
+ if (alloc_flags_allow_spinning(alloc_flags)) {
+ return __do_kmalloc_node(NULL, flags, node,
+ PASS_TOKEN_PARAM(token), &ac);
+ } else {
+ return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token),
+ flags, node, &ac);
+ }
+}
+
+
static noinline void free_to_partial_list(
struct kmem_cache *s, struct slab *slab,
void *head, void *tail, int bulk_cnt,
@@ -5818,7 +5913,7 @@ alloc_empty:
if (!allow_spin)
return NULL;
- empty = alloc_empty_sheaf(s, GFP_NOWAIT);
+ empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT);
if (empty)
goto got_empty;
@@ -6002,7 +6097,7 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj)
local_unlock(&s->cpu_sheaves->lock);
- empty = alloc_empty_sheaf(s, GFP_NOWAIT);
+ empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT);
if (!empty)
goto fail;
@@ -6825,14 +6920,19 @@ void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long
{
bool allow_block;
void *ret;
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
/*
* It doesn't really make sense to fallback to vmalloc for sub page
* requests
*/
- ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b),
+ ret = __do_kmalloc_node(PASS_BUCKET_PARAM(b),
kmalloc_gfp_adjust(flags, size),
- node, _RET_IP_, PASS_TOKEN_PARAM(token));
+ node, PASS_TOKEN_PARAM(token), &ac);
if (ret || size <= PAGE_SIZE)
return ret;
@@ -7222,9 +7322,6 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
unsigned int refilled;
struct slab *slab;
- if (WARN_ON_ONCE(!gfpflags_allow_spinning(gfp)))
- return 0;
-
refilled = __refill_objects_node(s, p, gfp, min, max,
get_node(s, local_node),
/* allow_spin = */ true);
@@ -7238,7 +7335,7 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
new_slab:
- slab = new_slab(s, gfp, local_node);
+ slab = new_slab(s, gfp, SLAB_ALLOC_DEFAULT, local_node);
if (!slab)
goto out;
@@ -7260,10 +7357,14 @@ static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
int i;
if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
+ const struct slab_alloc_context ac = {
+ .caller_addr = _RET_IP_,
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
for (i = 0; i < size; i++) {
- p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_,
- s->object_size);
+ p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, &ac);
if (unlikely(!p[i]))
goto error;
@@ -7293,8 +7394,7 @@ error:
* Allocate @size objects from @s and places them into @p. @size must be larger
* than 0.
*
- * Interrupts must be enabled when calling this function and @flags must allow
- * spinning.
+ * Interrupts must be enabled when calling this function.
*
* Unlike alloc_pages_bulk(), this function does not check for already allocated
* objects in @p, and thus the caller does not need to zero it.
@@ -7306,6 +7406,10 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
{
unsigned int i = 0;
void *kfence_obj;
+ const struct slab_alloc_context ac = {
+ .orig_size = s->object_size,
+ .alloc_flags = SLAB_ALLOC_DEFAULT,
+ };
if (!size)
return false;
@@ -7328,7 +7432,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
size--;
}
- i = alloc_from_pcs_bulk(s, flags, size, p);
+ i = alloc_from_pcs_bulk(s, size, p);
if (i < size) {
/*
* If we ran out of memory, don't bother with freeing back to
@@ -7356,8 +7460,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
out:
/* memcg and kmem_cache debug support and memory initialization */
- return likely(slab_post_alloc_hook(s, NULL, flags, size, p,
- slab_want_init_on_alloc(flags, s), s->object_size));
+ return likely(slab_post_alloc_hook(s, flags, size, p, &ac));
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
@@ -7555,7 +7658,7 @@ static int init_percpu_sheaves(struct kmem_cache *s)
if (!s->sheaf_capacity)
pcs->main = &bootstrap_sheaf;
else
- pcs->main = alloc_empty_sheaf(s, GFP_KERNEL);
+ pcs->main = alloc_empty_sheaf(s, GFP_KERNEL, SLAB_ALLOC_DEFAULT);
if (!pcs->main)
return -ENOMEM;
@@ -7583,7 +7686,7 @@ static void early_kmem_cache_node_alloc(int node)
BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
- slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
+ slab = new_slab(kmem_cache_node, GFP_NOWAIT, SLAB_ALLOC_DEFAULT, node);
BUG_ON(!slab);
if (slab_nid(slab) != node) {
@@ -8421,7 +8524,8 @@ static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
- pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
+ pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL,
+ SLAB_ALLOC_DEFAULT, capacity);
if (!pcs->main) {
failed = true;
generated by cgit 1.3-korg (git 2.53.0) at 2026-07-01 14:04:58 +0000