2006-03-22 16:09:12 +08:00
|
|
|
#ifndef _LINUX_MIGRATE_H
|
|
|
|
|
#define _LINUX_MIGRATE_H
|
|
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
2007-05-07 05:50:20 +08:00
|
|
|
#include <linux/mempolicy.h>
|
2012-01-21 06:33:53 +08:00
|
|
|
#include <linux/migrate_mode.h>
|
2017-07-11 06:48:47 +08:00
|
|
|
#include <linux/hugetlb.h>
|
2006-03-22 16:09:12 +08:00
|
|
|
|
2014-06-05 07:08:25 +08:00
|
|
|
typedef struct page *new_page_t(struct page *page, unsigned long private,
|
|
|
|
|
int **reason);
|
|
|
|
|
typedef void free_page_t(struct page *page, unsigned long private);
|
2006-06-23 17:03:53 +08:00
|
|
|
|
2012-12-12 08:02:31 +08:00
|
|
|
/*
|
|
|
|
|
* Return values from addresss_space_operations.migratepage():
|
|
|
|
|
* - negative errno on page migration failure;
|
|
|
|
|
* - zero on page migration success;
|
|
|
|
|
*/
|
|
|
|
|
#define MIGRATEPAGE_SUCCESS 0
|
2014-10-10 06:29:27 +08:00
|
|
|
|
2012-10-19 21:07:31 +08:00
|
|
|
enum migrate_reason {
|
|
|
|
|
MR_COMPACTION,
|
|
|
|
|
MR_MEMORY_FAILURE,
|
|
|
|
|
MR_MEMORY_HOTPLUG,
|
|
|
|
|
MR_SYSCALL, /* also applies to cpusets */
|
|
|
|
|
MR_MEMPOLICY_MBIND,
|
2012-10-25 20:16:34 +08:00
|
|
|
MR_NUMA_MISPLACED,
|
2016-03-16 05:56:18 +08:00
|
|
|
MR_CMA,
|
|
|
|
|
MR_TYPES
|
2012-10-19 21:07:31 +08:00
|
|
|
};
|
2012-12-12 08:02:31 +08:00
|
|
|
|
2016-03-16 05:56:18 +08:00
|
|
|
/* In mm/debug.c; also keep sync with include/trace/events/migrate.h */
|
|
|
|
|
extern char *migrate_reason_names[MR_TYPES];
|
|
|
|
|
|
2017-07-11 06:48:47 +08:00
|
|
|
static inline struct page *new_page_nodemask(struct page *page,
|
|
|
|
|
int preferred_nid, nodemask_t *nodemask)
|
|
|
|
|
{
|
2017-07-13 05:36:58 +08:00
|
|
|
gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL;
|
2017-07-11 06:48:47 +08:00
|
|
|
|
|
|
|
|
if (PageHuge(page))
|
|
|
|
|
return alloc_huge_page_nodemask(page_hstate(compound_head(page)),
|
2017-07-11 06:49:11 +08:00
|
|
|
preferred_nid, nodemask);
|
2017-07-11 06:48:47 +08:00
|
|
|
|
|
|
|
|
if (PageHighMem(page) || (zone_idx(page_zone(page)) == ZONE_MOVABLE))
|
|
|
|
|
gfp_mask |= __GFP_HIGHMEM;
|
|
|
|
|
|
|
|
|
|
return __alloc_pages_nodemask(gfp_mask, 0, preferred_nid, nodemask);
|
|
|
|
|
}
|
|
|
|
|
|
2007-05-07 05:50:20 +08:00
|
|
|
#ifdef CONFIG_MIGRATION
|
2009-01-07 06:39:16 +08:00
|
|
|
|
2012-12-12 08:02:47 +08:00
|
|
|
extern void putback_movable_pages(struct list_head *l);
|
2017-05-04 05:54:45 +08:00
|
|
|
extern int migrate_page(struct address_space *mapping,
|
|
|
|
|
struct page *newpage, struct page *page,
|
|
|
|
|
enum migrate_mode mode);
|
2014-06-05 07:08:25 +08:00
|
|
|
extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
|
2013-02-23 08:35:14 +08:00
|
|
|
unsigned long private, enum migrate_mode mode, int reason);
|
2017-02-25 06:57:29 +08:00
|
|
|
extern int isolate_movable_page(struct page *page, isolate_mode_t mode);
|
mm: migrate: support non-lru movable page migration
We have allowed migration for only LRU pages until now and it was enough
to make high-order pages. But recently, embedded system(e.g., webOS,
android) uses lots of non-movable pages(e.g., zram, GPU memory) so we
have seen several reports about troubles of small high-order allocation.
For fixing the problem, there were several efforts (e,g,. enhance
compaction algorithm, SLUB fallback to 0-order page, reserved memory,
vmalloc and so on) but if there are lots of non-movable pages in system,
their solutions are void in the long run.
So, this patch is to support facility to change non-movable pages with
movable. For the feature, this patch introduces functions related to
migration to address_space_operations as well as some page flags.
If a driver want to make own pages movable, it should define three
functions which are function pointers of struct
address_space_operations.
1. bool (*isolate_page) (struct page *page, isolate_mode_t mode);
What VM expects on isolate_page function of driver is to return *true*
if driver isolates page successfully. On returing true, VM marks the
page as PG_isolated so concurrent isolation in several CPUs skip the
page for isolation. If a driver cannot isolate the page, it should
return *false*.
Once page is successfully isolated, VM uses page.lru fields so driver
shouldn't expect to preserve values in that fields.
2. int (*migratepage) (struct address_space *mapping,
struct page *newpage, struct page *oldpage, enum migrate_mode);
After isolation, VM calls migratepage of driver with isolated page. The
function of migratepage is to move content of the old page to new page
and set up fields of struct page newpage. Keep in mind that you should
indicate to the VM the oldpage is no longer movable via
__ClearPageMovable() under page_lock if you migrated the oldpage
successfully and returns 0. If driver cannot migrate the page at the
moment, driver can return -EAGAIN. On -EAGAIN, VM will retry page
migration in a short time because VM interprets -EAGAIN as "temporal
migration failure". On returning any error except -EAGAIN, VM will give
up the page migration without retrying in this time.
Driver shouldn't touch page.lru field VM using in the functions.
3. void (*putback_page)(struct page *);
If migration fails on isolated page, VM should return the isolated page
to the driver so VM calls driver's putback_page with migration failed
page. In this function, driver should put the isolated page back to the
own data structure.
4. non-lru movable page flags
There are two page flags for supporting non-lru movable page.
* PG_movable
Driver should use the below function to make page movable under
page_lock.
void __SetPageMovable(struct page *page, struct address_space *mapping)
It needs argument of address_space for registering migration family
functions which will be called by VM. Exactly speaking, PG_movable is
not a real flag of struct page. Rather than, VM reuses page->mapping's
lower bits to represent it.
#define PAGE_MAPPING_MOVABLE 0x2
page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
so driver shouldn't access page->mapping directly. Instead, driver
should use page_mapping which mask off the low two bits of page->mapping
so it can get right struct address_space.
For testing of non-lru movable page, VM supports __PageMovable function.
However, it doesn't guarantee to identify non-lru movable page because
page->mapping field is unified with other variables in struct page. As
well, if driver releases the page after isolation by VM, page->mapping
doesn't have stable value although it has PAGE_MAPPING_MOVABLE (Look at
__ClearPageMovable). But __PageMovable is cheap to catch whether page
is LRU or non-lru movable once the page has been isolated. Because LRU
pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
good for just peeking to test non-lru movable pages before more
expensive checking with lock_page in pfn scanning to select victim.
For guaranteeing non-lru movable page, VM provides PageMovable function.
Unlike __PageMovable, PageMovable functions validates page->mapping and
mapping->a_ops->isolate_page under lock_page. The lock_page prevents
sudden destroying of page->mapping.
Driver using __SetPageMovable should clear the flag via
__ClearMovablePage under page_lock before the releasing the page.
* PG_isolated
To prevent concurrent isolation among several CPUs, VM marks isolated
page as PG_isolated under lock_page. So if a CPU encounters PG_isolated
non-lru movable page, it can skip it. Driver doesn't need to manipulate
the flag because VM will set/clear it automatically. Keep in mind that
if driver sees PG_isolated page, it means the page have been isolated by
VM so it shouldn't touch page.lru field. PG_isolated is alias with
PG_reclaim flag so driver shouldn't use the flag for own purpose.
[opensource.ganesh@gmail.com: mm/compaction: remove local variable is_lru]
Link: http://lkml.kernel.org/r/20160618014841.GA7422@leo-test
Link: http://lkml.kernel.org/r/1464736881-24886-3-git-send-email-minchan@kernel.org
Signed-off-by: Gioh Kim <gi-oh.kim@profitbricks.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
Signed-off-by: Ganesh Mahendran <opensource.ganesh@gmail.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rafael Aquini <aquini@redhat.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: John Einar Reitan <john.reitan@foss.arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-07-27 06:23:05 +08:00
|
|
|
extern void putback_movable_page(struct page *page);
|
2006-06-23 17:03:53 +08:00
|
|
|
|
2006-03-22 16:09:12 +08:00
|
|
|
extern int migrate_prep(void);
|
2010-05-25 05:32:27 +08:00
|
|
|
extern int migrate_prep_local(void);
|
2010-09-08 09:19:35 +08:00
|
|
|
extern void migrate_page_copy(struct page *newpage, struct page *page);
|
|
|
|
|
extern int migrate_huge_page_move_mapping(struct address_space *mapping,
|
|
|
|
|
struct page *newpage, struct page *page);
|
2013-07-16 17:56:16 +08:00
|
|
|
extern int migrate_page_move_mapping(struct address_space *mapping,
|
|
|
|
|
struct page *newpage, struct page *page,
|
2013-12-22 06:56:08 +08:00
|
|
|
struct buffer_head *head, enum migrate_mode mode,
|
|
|
|
|
int extra_count);
|
2006-03-22 16:09:12 +08:00
|
|
|
#else
|
2009-01-07 06:39:16 +08:00
|
|
|
|
2012-12-12 08:02:47 +08:00
|
|
|
static inline void putback_movable_pages(struct list_head *l) {}
|
2014-06-05 07:08:25 +08:00
|
|
|
static inline int migrate_pages(struct list_head *l, new_page_t new,
|
|
|
|
|
free_page_t free, unsigned long private, enum migrate_mode mode,
|
|
|
|
|
int reason)
|
2013-02-23 08:35:14 +08:00
|
|
|
{ return -ENOSYS; }
|
2017-02-25 06:57:32 +08:00
|
|
|
static inline int isolate_movable_page(struct page *page, isolate_mode_t mode)
|
|
|
|
|
{ return -EBUSY; }
|
2006-03-31 18:29:56 +08:00
|
|
|
|
2006-03-22 16:09:12 +08:00
|
|
|
static inline int migrate_prep(void) { return -ENOSYS; }
|
2010-05-25 05:32:27 +08:00
|
|
|
static inline int migrate_prep_local(void) { return -ENOSYS; }
|
2006-03-22 16:09:12 +08:00
|
|
|
|
2010-09-08 09:19:35 +08:00
|
|
|
static inline void migrate_page_copy(struct page *newpage,
|
|
|
|
|
struct page *page) {}
|
|
|
|
|
|
2010-09-30 10:54:51 +08:00
|
|
|
static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
|
2010-09-08 09:19:35 +08:00
|
|
|
struct page *newpage, struct page *page)
|
|
|
|
|
{
|
|
|
|
|
return -ENOSYS;
|
|
|
|
|
}
|
|
|
|
|
|
2006-03-22 16:09:12 +08:00
|
|
|
#endif /* CONFIG_MIGRATION */
|
2012-10-25 20:16:34 +08:00
|
|
|
|
2016-07-27 06:26:50 +08:00
|
|
|
#ifdef CONFIG_COMPACTION
|
|
|
|
|
extern int PageMovable(struct page *page);
|
|
|
|
|
extern void __SetPageMovable(struct page *page, struct address_space *mapping);
|
|
|
|
|
extern void __ClearPageMovable(struct page *page);
|
|
|
|
|
#else
|
|
|
|
|
static inline int PageMovable(struct page *page) { return 0; };
|
|
|
|
|
static inline void __SetPageMovable(struct page *page,
|
|
|
|
|
struct address_space *mapping)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void __ClearPageMovable(struct page *page)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-10-25 20:16:34 +08:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
2013-12-19 09:08:42 +08:00
|
|
|
extern bool pmd_trans_migrating(pmd_t pmd);
|
2013-10-07 18:29:05 +08:00
|
|
|
extern int migrate_misplaced_page(struct page *page,
|
|
|
|
|
struct vm_area_struct *vma, int node);
|
2012-10-25 20:16:34 +08:00
|
|
|
#else
|
2013-12-19 09:08:42 +08:00
|
|
|
static inline bool pmd_trans_migrating(pmd_t pmd)
|
|
|
|
|
{
|
|
|
|
|
return false;
|
|
|
|
|
}
|
2013-10-07 18:29:05 +08:00
|
|
|
static inline int migrate_misplaced_page(struct page *page,
|
|
|
|
|
struct vm_area_struct *vma, int node)
|
2012-10-25 20:16:34 +08:00
|
|
|
{
|
|
|
|
|
return -EAGAIN; /* can't migrate now */
|
|
|
|
|
}
|
2012-12-05 17:32:56 +08:00
|
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
2012-11-19 20:35:47 +08:00
|
|
|
|
2012-12-05 17:32:56 +08:00
|
|
|
#if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
|
|
|
|
|
extern int migrate_misplaced_transhuge_page(struct mm_struct *mm,
|
|
|
|
|
struct vm_area_struct *vma,
|
|
|
|
|
pmd_t *pmd, pmd_t entry,
|
|
|
|
|
unsigned long address,
|
|
|
|
|
struct page *page, int node);
|
|
|
|
|
#else
|
2012-11-19 20:35:47 +08:00
|
|
|
static inline int migrate_misplaced_transhuge_page(struct mm_struct *mm,
|
|
|
|
|
struct vm_area_struct *vma,
|
|
|
|
|
pmd_t *pmd, pmd_t entry,
|
|
|
|
|
unsigned long address,
|
|
|
|
|
struct page *page, int node)
|
|
|
|
|
{
|
|
|
|
|
return -EAGAIN;
|
|
|
|
|
}
|
2012-12-05 17:32:56 +08:00
|
|
|
#endif /* CONFIG_NUMA_BALANCING && CONFIG_TRANSPARENT_HUGEPAGE*/
|
2012-10-25 20:16:34 +08:00
|
|
|
|
2006-03-22 16:09:12 +08:00
|
|
|
#endif /* _LINUX_MIGRATE_H */
|
