aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
authorKairui Song <kasong@tencent.com>2026-05-17 23:39:48 +0800
committerAndrew Morton <akpm@linux-foundation.org>2026-06-02 15:22:22 -0700
commitcdd77f84d96675c9e8c776073df8d58d2af10607 (patch)
tree1e82d1bf0e0e46f262c48bedc1faddb2991acbcb /mm
parentbc34e87a51d9e51d398ef6d8c2c35cf1a4ff38b9 (diff)
downloadath-cdd77f84d96675c9e8c776073df8d58d2af10607.tar.gz
mm, swap: consolidate cluster allocation helpers
Swap cluster table management is spread across several narrow helpers. As a result, the allocation and fallback sequences are open-coded in multiple places. A few more per-cluster tables will be added soon, so avoid duplicating these sequences per table type. Fold the existing pairs into cluster-oriented helpers, and rename for consistency. No functional change, only a few sanity checks are slightly adjusted. Link: https://lore.kernel.org/20260517-swap-table-p4-v5-9-88ae43e064c7@tencent.com Signed-off-by: Kairui Song <kasong@tencent.com> Acked-by: Chris Li <chrisl@kernel.org> Cc: Baolin Wang <baolin.wang@linux.alibaba.com> Cc: Baoquan He <bhe@redhat.com> Cc: Barry Song <baohua@kernel.org> Cc: Chengming Zhou <chengming.zhou@linux.dev> Cc: David Hildenbrand <david@kernel.org> Cc: Hugh Dickins <hughd@google.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Kemeng Shi <shikemeng@huaweicloud.com> Cc: Lorenzo Stoakes <ljs@kernel.org> Cc: Muchun Song <muchun.song@linux.dev> Cc: Nhat Pham <nphamcs@gmail.com> Cc: Roman Gushchin <roman.gushchin@linux.dev> Cc: Shakeel Butt <shakeel.butt@linux.dev> Cc: Youngjun Park <youngjun.park@lge.com> Cc: Zi Yan <ziy@nvidia.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/swapfile.c110
1 files changed, 49 insertions, 61 deletions
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 60d8f0df3f32c..2ddabc0f3a888 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -411,20 +411,7 @@ static inline unsigned int cluster_offset(struct swap_info_struct *si,
return cluster_index(si, ci) * SWAPFILE_CLUSTER;
}
-static struct swap_table *swap_table_alloc(gfp_t gfp)
-{
- struct folio *folio;
-
- if (!SWP_TABLE_USE_PAGE)
- return kmem_cache_zalloc(swap_table_cachep, gfp);
-
- folio = folio_alloc(gfp | __GFP_ZERO, 0);
- if (folio)
- return folio_address(folio);
- return NULL;
-}
-
-static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
+static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head)
{
struct folio *folio;
@@ -432,15 +419,46 @@ static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
folio_put(folio);
}
-static void swap_table_free(struct swap_table *table)
+static void swap_cluster_free_table(struct swap_cluster_info *ci)
{
+ struct swap_table *table;
+
+ table = (struct swap_table *)rcu_dereference_protected(ci->table, true);
+ if (!table)
+ return;
+
+ rcu_assign_pointer(ci->table, NULL);
if (!SWP_TABLE_USE_PAGE) {
kmem_cache_free(swap_table_cachep, table);
return;
}
call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
- swap_table_free_folio_rcu_cb);
+ swap_cluster_free_table_folio_rcu_cb);
+}
+
+static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
+{
+ struct swap_table *table = NULL;
+ struct folio *folio;
+
+ /* The cluster must be empty and not on any list during allocation. */
+ VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
+ if (rcu_access_pointer(ci->table))
+ return 0;
+
+ if (SWP_TABLE_USE_PAGE) {
+ folio = folio_alloc(gfp | __GFP_ZERO, 0);
+ if (folio)
+ table = folio_address(folio);
+ } else {
+ table = kmem_cache_zalloc(swap_table_cachep, gfp);
+ }
+ if (!table)
+ return -ENOMEM;
+
+ rcu_assign_pointer(ci->table, table);
+ return 0;
}
/*
@@ -471,27 +489,15 @@ static void swap_cluster_assert_empty(struct swap_cluster_info *ci,
WARN_ON_ONCE(nr == SWAPFILE_CLUSTER && ci->extend_table);
}
-static void swap_cluster_free_table(struct swap_cluster_info *ci)
-{
- struct swap_table *table;
-
- /* Only empty cluster's table is allow to be freed */
- lockdep_assert_held(&ci->lock);
- table = (void *)rcu_dereference_protected(ci->table, true);
- rcu_assign_pointer(ci->table, NULL);
-
- swap_table_free(table);
-}
-
/*
* Allocate swap table for one cluster. Attempt an atomic allocation first,
* then fallback to sleeping allocation.
*/
static struct swap_cluster_info *
-swap_cluster_alloc_table(struct swap_info_struct *si,
+swap_cluster_populate(struct swap_info_struct *si,
struct swap_cluster_info *ci)
{
- struct swap_table *table;
+ int ret;
/*
* Only cluster isolation from the allocator does table allocation.
@@ -502,14 +508,9 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
lockdep_assert_held(&si->global_cluster_lock);
lockdep_assert_held(&ci->lock);
- /* The cluster must be free and was just isolated from the free list. */
- VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
-
- table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
- if (table) {
- rcu_assign_pointer(ci->table, table);
+ if (!swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ __GFP_NOWARN))
return ci;
- }
/*
* Try a sleep allocation. Each isolated free cluster may cause
@@ -521,7 +522,8 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_unlock(&si->global_cluster_lock);
local_unlock(&percpu_swap_cluster.lock);
- table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL);
+ ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
+ GFP_KERNEL);
/*
* Back to atomic context. We might have migrated to a new CPU with a
@@ -536,20 +538,11 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_lock(&si->global_cluster_lock);
spin_lock(&ci->lock);
- /* Nothing except this helper should touch a dangling empty cluster. */
- if (WARN_ON_ONCE(cluster_table_is_alloced(ci))) {
- if (table)
- swap_table_free(table);
- return ci;
- }
-
- if (!table) {
+ if (ret) {
move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
spin_unlock(&ci->lock);
return NULL;
}
-
- rcu_assign_pointer(ci->table, table);
return ci;
}
@@ -621,12 +614,11 @@ static struct swap_cluster_info *isolate_lock_cluster(
}
spin_unlock(&si->lock);
- if (found && !cluster_table_is_alloced(found)) {
- /* Only an empty free cluster's swap table can be freed. */
- VM_WARN_ON_ONCE(flags != CLUSTER_FLAG_FREE);
+ /* Cluster's table is freed when and only when it's on the free list. */
+ if (found && flags == CLUSTER_FLAG_FREE) {
VM_WARN_ON_ONCE(list != &si->free_clusters);
- VM_WARN_ON_ONCE(!cluster_is_empty(found));
- return swap_cluster_alloc_table(si, found);
+ VM_WARN_ON_ONCE(cluster_table_is_alloced(found));
+ return swap_cluster_populate(si, found);
}
return found;
@@ -769,7 +761,6 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
unsigned int ci_off = offset % SWAPFILE_CLUSTER;
unsigned long idx = offset / SWAPFILE_CLUSTER;
struct swap_cluster_info *ci;
- struct swap_table *table;
int ret = 0;
/* si->max may got shrunk by swap swap_activate() */
@@ -790,12 +781,9 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
}
ci = cluster_info + idx;
- if (!ci->table) {
- table = swap_table_alloc(GFP_KERNEL);
- if (!table)
- return -ENOMEM;
- rcu_assign_pointer(ci->table, table);
- }
+ /* Need to allocate swap table first for initial bad slot marking. */
+ if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL))
+ return -ENOMEM;
spin_lock(&ci->lock);
/* Check for duplicated bad swap slots. */
if (__swap_table_xchg(ci, ci_off, SWP_TB_BAD) != SWP_TB_NULL) {
@@ -2920,7 +2908,7 @@ static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
ci = cluster_info + i;
/* Cluster with bad marks count will have a remaining table */
spin_lock(&ci->lock);
- if (rcu_dereference_protected(ci->table, true)) {
+ if (cluster_table_is_alloced(ci)) {
swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
swap_cluster_free_table(ci);
}
generated by cgit 1.3-korg (git 2.53.0) at 2026-07-01 14:04:46 +0000