mirror of
https://github.com/torvalds/linux.git
synced 2026-01-25 15:03:52 +08:00
mm: move lesser used vma_area_struct members into the last cacheline
Move several vma_area_struct members which are rarely or never used during
page fault handling into the last cacheline to better pack vm_area_struct.
As a result vm_area_struct will fit into 3 as opposed to 4 cachelines.
New typical vm_area_struct layout:
struct vm_area_struct {
union {
struct {
long unsigned int vm_start; /* 0 8 */
long unsigned int vm_end; /* 8 8 */
}; /* 0 16 */
freeptr_t vm_freeptr; /* 0 8 */
}; /* 0 16 */
struct mm_struct * vm_mm; /* 16 8 */
pgprot_t vm_page_prot; /* 24 8 */
union {
const vm_flags_t vm_flags; /* 32 8 */
vm_flags_t __vm_flags; /* 32 8 */
}; /* 32 8 */
unsigned int vm_lock_seq; /* 40 4 */
/* XXX 4 bytes hole, try to pack */
struct list_head anon_vma_chain; /* 48 16 */
/* --- cacheline 1 boundary (64 bytes) --- */
struct anon_vma * anon_vma; /* 64 8 */
const struct vm_operations_struct * vm_ops; /* 72 8 */
long unsigned int vm_pgoff; /* 80 8 */
struct file * vm_file; /* 88 8 */
void * vm_private_data; /* 96 8 */
atomic_long_t swap_readahead_info; /* 104 8 */
struct mempolicy * vm_policy; /* 112 8 */
struct vma_numab_state * numab_state; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
refcount_t vm_refcnt (__aligned__(64)); /* 128 4 */
/* XXX 4 bytes hole, try to pack */
struct {
struct rb_node rb (__aligned__(8)); /* 136 24 */
long unsigned int rb_subtree_last; /* 160 8 */
} __attribute__((__aligned__(8))) shared; /* 136 32 */
struct anon_vma_name * anon_name; /* 168 8 */
struct vm_userfaultfd_ctx vm_userfaultfd_ctx; /* 176 8 */
/* size: 192, cachelines: 3, members: 18 */
/* sum members: 176, holes: 2, sum holes: 8 */
/* padding: 8 */
/* forced alignments: 2, forced holes: 1, sum forced holes: 4 */
} __attribute__((__aligned__(64)));
Memory consumption per 1000 VMAs becomes 48 pages:
slabinfo after vm_area_struct changes:
<name> ... <objsize> <objperslab> <pagesperslab> : ...
vm_area_struct ... 192 42 2 : ...
Link: https://lkml.kernel.org/r/20250213224655.1680278-14-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Tested-by: Shivank Garg <shivankg@amd.com>
Link: https://lkml.kernel.org/r/5e19ec93-8307-47c2-bb13-3ddf7150624e@amd.com
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Klara Modin <klarasmodin@gmail.com>
Cc: Liam R. Howlett <Liam.Howlett@Oracle.com>
Cc: Lokesh Gidra <lokeshgidra@google.com>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Minchan Kim <minchan@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Sourav Panda <souravpanda@google.com>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Suren Baghdasaryan
committed by
Andrew Morton
Andrew Morton
parent
f35ab95ca0
commit
6bef4c2f97
@@ -725,17 +725,6 @@ struct vm_area_struct {
|
||||
*/
|
||||
unsigned int vm_lock_seq;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* For areas with an address space and backing store,
|
||||
* linkage into the address_space->i_mmap interval tree.
|
||||
*
|
||||
*/
|
||||
struct {
|
||||
struct rb_node rb;
|
||||
unsigned long rb_subtree_last;
|
||||
} shared;
|
||||
|
||||
/*
|
||||
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
|
||||
* list, after a COW of one of the file pages. A MAP_SHARED vma
|
||||
@@ -755,14 +744,6 @@ struct vm_area_struct {
|
||||
struct file * vm_file; /* File we map to (can be NULL). */
|
||||
void * vm_private_data; /* was vm_pte (shared mem) */
|
||||
|
||||
#ifdef CONFIG_ANON_VMA_NAME
|
||||
/*
|
||||
* For private and shared anonymous mappings, a pointer to a null
|
||||
* terminated string containing the name given to the vma, or NULL if
|
||||
* unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
|
||||
*/
|
||||
struct anon_vma_name *anon_name;
|
||||
#endif
|
||||
#ifdef CONFIG_SWAP
|
||||
atomic_long_t swap_readahead_info;
|
||||
#endif
|
||||
@@ -775,7 +756,6 @@ struct vm_area_struct {
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
struct vma_numab_state *numab_state; /* NUMA Balancing state */
|
||||
#endif
|
||||
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
|
||||
#ifdef CONFIG_PER_VMA_LOCK
|
||||
/* Unstable RCU readers are allowed to read this. */
|
||||
refcount_t vm_refcnt ____cacheline_aligned_in_smp;
|
||||
@@ -783,6 +763,24 @@ struct vm_area_struct {
|
||||
struct lockdep_map vmlock_dep_map;
|
||||
#endif
|
||||
#endif
|
||||
/*
|
||||
* For areas with an address space and backing store,
|
||||
* linkage into the address_space->i_mmap interval tree.
|
||||
*
|
||||
*/
|
||||
struct {
|
||||
struct rb_node rb;
|
||||
unsigned long rb_subtree_last;
|
||||
} shared;
|
||||
#ifdef CONFIG_ANON_VMA_NAME
|
||||
/*
|
||||
* For private and shared anonymous mappings, a pointer to a null
|
||||
* terminated string containing the name given to the vma, or NULL if
|
||||
* unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
|
||||
*/
|
||||
struct anon_vma_name *anon_name;
|
||||
#endif
|
||||
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
|
||||
} __randomize_layout;
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
|
||||
@@ -279,16 +279,6 @@ struct vm_area_struct {
|
||||
unsigned int vm_lock_seq;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* For areas with an address space and backing store,
|
||||
* linkage into the address_space->i_mmap interval tree.
|
||||
*
|
||||
*/
|
||||
struct {
|
||||
struct rb_node rb;
|
||||
unsigned long rb_subtree_last;
|
||||
} shared;
|
||||
|
||||
/*
|
||||
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
|
||||
* list, after a COW of one of the file pages. A MAP_SHARED vma
|
||||
@@ -308,14 +298,6 @@ struct vm_area_struct {
|
||||
struct file * vm_file; /* File we map to (can be NULL). */
|
||||
void * vm_private_data; /* was vm_pte (shared mem) */
|
||||
|
||||
#ifdef CONFIG_ANON_VMA_NAME
|
||||
/*
|
||||
* For private and shared anonymous mappings, a pointer to a null
|
||||
* terminated string containing the name given to the vma, or NULL if
|
||||
* unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
|
||||
*/
|
||||
struct anon_vma_name *anon_name;
|
||||
#endif
|
||||
#ifdef CONFIG_SWAP
|
||||
atomic_long_t swap_readahead_info;
|
||||
#endif
|
||||
@@ -328,11 +310,28 @@ struct vm_area_struct {
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
struct vma_numab_state *numab_state; /* NUMA Balancing state */
|
||||
#endif
|
||||
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
|
||||
#ifdef CONFIG_PER_VMA_LOCK
|
||||
/* Unstable RCU readers are allowed to read this. */
|
||||
refcount_t vm_refcnt;
|
||||
#endif
|
||||
/*
|
||||
* For areas with an address space and backing store,
|
||||
* linkage into the address_space->i_mmap interval tree.
|
||||
*
|
||||
*/
|
||||
struct {
|
||||
struct rb_node rb;
|
||||
unsigned long rb_subtree_last;
|
||||
} shared;
|
||||
#ifdef CONFIG_ANON_VMA_NAME
|
||||
/*
|
||||
* For private and shared anonymous mappings, a pointer to a null
|
||||
* terminated string containing the name given to the vma, or NULL if
|
||||
* unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
|
||||
*/
|
||||
struct anon_vma_name *anon_name;
|
||||
#endif
|
||||
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
|
||||
} __randomize_layout;
|
||||
|
||||
struct vm_fault {};
|
||||
|
||||
Reference in New Issue
Block a user