Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef _LINUX_HUGETLB_H |
| 2 | #define _LINUX_HUGETLB_H |
| 3 | |
Linus Torvalds | be93d8c | 2011-05-26 12:03:50 -0700 | [diff] [blame] | 4 | #include <linux/mm_types.h> |
Sasha Levin | 309381fea | 2014-01-23 15:52:54 -0800 | [diff] [blame] | 5 | #include <linux/mmdebug.h> |
Alexey Dobriyan | 4e950f6 | 2007-07-30 02:36:13 +0400 | [diff] [blame] | 6 | #include <linux/fs.h> |
Naoya Horiguchi | 8edf344 | 2010-05-28 09:29:15 +0900 | [diff] [blame] | 7 | #include <linux/hugetlb_inline.h> |
Aneesh Kumar K.V | abb8206 | 2012-07-31 16:42:24 -0700 | [diff] [blame] | 8 | #include <linux/cgroup.h> |
Alexey Dobriyan | 4e950f6 | 2007-07-30 02:36:13 +0400 | [diff] [blame] | 9 | |
Andrew Morton | e9ea0e2 | 2009-09-24 14:47:45 -0700 | [diff] [blame] | 10 | struct ctl_table; |
| 11 | struct user_struct; |
Aneesh Kumar K.V | 24669e5 | 2012-07-31 16:42:03 -0700 | [diff] [blame] | 12 | struct mmu_gather; |
Andrew Morton | e9ea0e2 | 2009-09-24 14:47:45 -0700 | [diff] [blame] | 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #ifdef CONFIG_HUGETLB_PAGE |
| 15 | |
| 16 | #include <linux/mempolicy.h> |
Adam Litke | 516dffd | 2007-03-01 15:46:08 -0800 | [diff] [blame] | 17 | #include <linux/shm.h> |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 18 | #include <asm/tlbflush.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | |
David Gibson | 9048162 | 2012-03-21 16:34:12 -0700 | [diff] [blame] | 20 | struct hugepage_subpool { |
| 21 | spinlock_t lock; |
| 22 | long count; |
| 23 | long max_hpages, used_hpages; |
| 24 | }; |
| 25 | |
Aneesh Kumar K.V | c3f38a3 | 2012-07-31 16:42:10 -0700 | [diff] [blame] | 26 | extern spinlock_t hugetlb_lock; |
| 27 | extern int hugetlb_max_hstate __read_mostly; |
| 28 | #define for_each_hstate(h) \ |
| 29 | for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++) |
| 30 | |
David Gibson | 9048162 | 2012-03-21 16:34:12 -0700 | [diff] [blame] | 31 | struct hugepage_subpool *hugepage_new_subpool(long nr_blocks); |
| 32 | void hugepage_put_subpool(struct hugepage_subpool *spool); |
| 33 | |
Wu Fengguang | 20a0307 | 2009-06-16 15:32:22 -0700 | [diff] [blame] | 34 | int PageHuge(struct page *page); |
| 35 | |
Mel Gorman | a1e7877 | 2008-07-23 21:27:23 -0700 | [diff] [blame] | 36 | void reset_vma_resv_huge_pages(struct vm_area_struct *vma); |
Alexey Dobriyan | 8d65af7 | 2009-09-23 15:57:19 -0700 | [diff] [blame] | 37 | int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); |
| 38 | int hugetlb_overcommit_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); |
| 39 | int hugetlb_treat_movable_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); |
Lee Schermerhorn | 06808b0 | 2009-12-14 17:58:21 -0800 | [diff] [blame] | 40 | |
| 41 | #ifdef CONFIG_NUMA |
| 42 | int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, |
| 43 | void __user *, size_t *, loff_t *); |
| 44 | #endif |
| 45 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *); |
Michel Lespinasse | 28a3571 | 2013-02-22 16:35:55 -0800 | [diff] [blame] | 47 | long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, |
| 48 | struct page **, struct vm_area_struct **, |
| 49 | unsigned long *, unsigned long *, long, unsigned int); |
Mel Gorman | 04f2cbe | 2008-07-23 21:27:25 -0700 | [diff] [blame] | 50 | void unmap_hugepage_range(struct vm_area_struct *, |
Aneesh Kumar K.V | 24669e5 | 2012-07-31 16:42:03 -0700 | [diff] [blame] | 51 | unsigned long, unsigned long, struct page *); |
Mel Gorman | d833352 | 2012-07-31 16:46:20 -0700 | [diff] [blame] | 52 | void __unmap_hugepage_range_final(struct mmu_gather *tlb, |
| 53 | struct vm_area_struct *vma, |
| 54 | unsigned long start, unsigned long end, |
| 55 | struct page *ref_page); |
Aneesh Kumar K.V | 24669e5 | 2012-07-31 16:42:03 -0700 | [diff] [blame] | 56 | void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, |
| 57 | unsigned long start, unsigned long end, |
| 58 | struct page *ref_page); |
Alexey Dobriyan | e1759c2 | 2008-10-15 23:50:22 +0400 | [diff] [blame] | 59 | void hugetlb_report_meminfo(struct seq_file *); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | int hugetlb_report_node_meminfo(int, char *); |
David Rientjes | 949f7ec | 2013-04-29 15:07:48 -0700 | [diff] [blame] | 61 | void hugetlb_show_meminfo(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | unsigned long hugetlb_total_pages(void); |
Hugh Dickins | ac9b9c6 | 2005-10-20 16:24:28 +0100 | [diff] [blame] | 63 | int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, |
Hugh Dickins | 788c7df | 2009-06-23 13:49:05 +0100 | [diff] [blame] | 64 | unsigned long address, unsigned int flags); |
Mel Gorman | a1e7877 | 2008-07-23 21:27:23 -0700 | [diff] [blame] | 65 | int hugetlb_reserve_pages(struct inode *inode, long from, long to, |
Mel Gorman | 5a6fe12 | 2009-02-10 14:02:27 +0000 | [diff] [blame] | 66 | struct vm_area_struct *vma, |
KOSAKI Motohiro | ca16d14 | 2011-05-26 19:16:19 +0900 | [diff] [blame] | 67 | vm_flags_t vm_flags); |
Chen, Kenneth W | a43a8c3 | 2006-06-23 02:03:15 -0700 | [diff] [blame] | 68 | void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed); |
Naoya Horiguchi | 6de2b1a | 2010-09-08 10:19:36 +0900 | [diff] [blame] | 69 | int dequeue_hwpoisoned_huge_page(struct page *page); |
Naoya Horiguchi | 31caf66 | 2013-09-11 14:21:59 -0700 | [diff] [blame] | 70 | bool isolate_huge_page(struct page *page, struct list_head *list); |
| 71 | void putback_active_hugepage(struct page *page); |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 72 | bool is_hugepage_active(struct page *page); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | |
Steve Capper | 3212b53 | 2013-04-23 12:35:02 +0100 | [diff] [blame] | 74 | #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE |
| 75 | pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud); |
| 76 | #endif |
| 77 | |
Mel Gorman | 396faf0 | 2007-07-17 04:03:13 -0700 | [diff] [blame] | 78 | extern unsigned long hugepages_treat_as_movable; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | extern const unsigned long hugetlb_zero, hugetlb_infinity; |
| 80 | extern int sysctl_hugetlb_shm_group; |
Jon Tollefson | 53ba51d | 2008-07-23 21:27:52 -0700 | [diff] [blame] | 81 | extern struct list_head huge_boot_pages; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 83 | /* arch callbacks */ |
| 84 | |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 85 | pte_t *huge_pte_alloc(struct mm_struct *mm, |
| 86 | unsigned long addr, unsigned long sz); |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 87 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr); |
Chen, Kenneth W | 39dde65 | 2006-12-06 20:32:03 -0800 | [diff] [blame] | 88 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep); |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 89 | struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, |
| 90 | int write); |
| 91 | struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, |
| 92 | pmd_t *pmd, int write); |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 93 | struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address, |
| 94 | pud_t *pud, int write); |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 95 | int pmd_huge(pmd_t pmd); |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 96 | int pud_huge(pud_t pmd); |
Peter Zijlstra | 7da4d64 | 2012-11-19 03:14:23 +0100 | [diff] [blame] | 97 | unsigned long hugetlb_change_protection(struct vm_area_struct *vma, |
Zhang, Yanmin | 8f86059 | 2006-03-22 00:08:50 -0800 | [diff] [blame] | 98 | unsigned long address, unsigned long end, pgprot_t newprot); |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 99 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | #else /* !CONFIG_HUGETLB_PAGE */ |
| 101 | |
Wu Fengguang | 20a0307 | 2009-06-16 15:32:22 -0700 | [diff] [blame] | 102 | static inline int PageHuge(struct page *page) |
| 103 | { |
| 104 | return 0; |
| 105 | } |
| 106 | |
Mel Gorman | a1e7877 | 2008-07-23 21:27:23 -0700 | [diff] [blame] | 107 | static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) |
| 108 | { |
| 109 | } |
| 110 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | static inline unsigned long hugetlb_total_pages(void) |
| 112 | { |
| 113 | return 0; |
| 114 | } |
| 115 | |
Adam Litke | 5b23dbe | 2007-11-14 16:59:33 -0800 | [diff] [blame] | 116 | #define follow_hugetlb_page(m,v,p,vs,a,b,i,w) ({ BUG(); 0; }) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | #define follow_huge_addr(mm, addr, write) ERR_PTR(-EINVAL) |
| 118 | #define copy_hugetlb_page_range(src, dst, vma) ({ BUG(); 0; }) |
Alexey Dobriyan | e1759c2 | 2008-10-15 23:50:22 +0400 | [diff] [blame] | 119 | static inline void hugetlb_report_meminfo(struct seq_file *m) |
| 120 | { |
| 121 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | #define hugetlb_report_node_meminfo(n, buf) 0 |
David Rientjes | 949f7ec | 2013-04-29 15:07:48 -0700 | [diff] [blame] | 123 | static inline void hugetlb_show_meminfo(void) |
| 124 | { |
| 125 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | #define follow_huge_pmd(mm, addr, pmd, write) NULL |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 127 | #define follow_huge_pud(mm, addr, pud, write) NULL |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 128 | #define prepare_hugepage_range(file, addr, len) (-EINVAL) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | #define pmd_huge(x) 0 |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 130 | #define pud_huge(x) 0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | #define is_hugepage_only_range(mm, addr, len) 0 |
David Gibson | 9da61ae | 2006-03-22 00:08:57 -0800 | [diff] [blame] | 132 | #define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; }) |
Hugh Dickins | 788c7df | 2009-06-23 13:49:05 +0100 | [diff] [blame] | 133 | #define hugetlb_fault(mm, vma, addr, flags) ({ BUG(); 0; }) |
Naoya Horiguchi | 0fe6e20 | 2010-05-28 09:29:16 +0900 | [diff] [blame] | 134 | #define huge_pte_offset(mm, address) 0 |
Aneesh Kumar K.V | 24669e5 | 2012-07-31 16:42:03 -0700 | [diff] [blame] | 135 | static inline int dequeue_hwpoisoned_huge_page(struct page *page) |
| 136 | { |
| 137 | return 0; |
| 138 | } |
| 139 | |
Naoya Horiguchi | f40386a | 2013-12-12 17:12:19 -0800 | [diff] [blame] | 140 | static inline bool isolate_huge_page(struct page *page, struct list_head *list) |
| 141 | { |
| 142 | return false; |
| 143 | } |
Naoya Horiguchi | 31caf66 | 2013-09-11 14:21:59 -0700 | [diff] [blame] | 144 | #define putback_active_hugepage(p) do {} while (0) |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 145 | #define is_hugepage_active(x) false |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | |
Peter Zijlstra | 7da4d64 | 2012-11-19 03:14:23 +0100 | [diff] [blame] | 147 | static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma, |
| 148 | unsigned long address, unsigned long end, pgprot_t newprot) |
| 149 | { |
| 150 | return 0; |
| 151 | } |
Zhang, Yanmin | 8f86059 | 2006-03-22 00:08:50 -0800 | [diff] [blame] | 152 | |
Mel Gorman | d833352 | 2012-07-31 16:46:20 -0700 | [diff] [blame] | 153 | static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb, |
| 154 | struct vm_area_struct *vma, unsigned long start, |
| 155 | unsigned long end, struct page *ref_page) |
| 156 | { |
| 157 | BUG(); |
| 158 | } |
| 159 | |
Aneesh Kumar K.V | 24669e5 | 2012-07-31 16:42:03 -0700 | [diff] [blame] | 160 | static inline void __unmap_hugepage_range(struct mmu_gather *tlb, |
| 161 | struct vm_area_struct *vma, unsigned long start, |
| 162 | unsigned long end, struct page *ref_page) |
| 163 | { |
| 164 | BUG(); |
| 165 | } |
| 166 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 167 | #endif /* !CONFIG_HUGETLB_PAGE */ |
| 168 | |
Eric B Munson | 4e52780 | 2009-09-21 17:03:47 -0700 | [diff] [blame] | 169 | #define HUGETLB_ANON_FILE "anon_hugepage" |
| 170 | |
Eric B Munson | 6bfde05 | 2009-09-21 17:03:43 -0700 | [diff] [blame] | 171 | enum { |
| 172 | /* |
| 173 | * The file will be used as an shm file so shmfs accounting rules |
| 174 | * apply |
| 175 | */ |
| 176 | HUGETLB_SHMFS_INODE = 1, |
Eric B Munson | 4e52780 | 2009-09-21 17:03:47 -0700 | [diff] [blame] | 177 | /* |
| 178 | * The file is being created on the internal vfs mount and shmfs |
| 179 | * accounting rules do not apply |
| 180 | */ |
| 181 | HUGETLB_ANONHUGE_INODE = 2, |
Eric B Munson | 6bfde05 | 2009-09-21 17:03:43 -0700 | [diff] [blame] | 182 | }; |
| 183 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | #ifdef CONFIG_HUGETLBFS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 185 | struct hugetlbfs_sb_info { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | long max_inodes; /* inodes allowed */ |
| 187 | long free_inodes; /* inodes free */ |
| 188 | spinlock_t stat_lock; |
Andi Kleen | a137e1c | 2008-07-23 21:27:43 -0700 | [diff] [blame] | 189 | struct hstate *hstate; |
David Gibson | 9048162 | 2012-03-21 16:34:12 -0700 | [diff] [blame] | 190 | struct hugepage_subpool *spool; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | }; |
| 192 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb) |
| 194 | { |
| 195 | return sb->s_fs_info; |
| 196 | } |
| 197 | |
Arjan van de Ven | 4b6f5d2 | 2006-03-28 01:56:42 -0800 | [diff] [blame] | 198 | extern const struct file_operations hugetlbfs_file_operations; |
Alexey Dobriyan | f0f37e2 | 2009-09-27 22:29:37 +0400 | [diff] [blame] | 199 | extern const struct vm_operations_struct hugetlb_vm_ops; |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 200 | struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct, |
Andi Kleen | 42d7395 | 2012-12-11 16:01:34 -0800 | [diff] [blame] | 201 | struct user_struct **user, int creat_flags, |
| 202 | int page_size_log); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | |
| 204 | static inline int is_file_hugepages(struct file *file) |
| 205 | { |
Adam Litke | 516dffd | 2007-03-01 15:46:08 -0800 | [diff] [blame] | 206 | if (file->f_op == &hugetlbfs_file_operations) |
| 207 | return 1; |
| 208 | if (is_file_shm_hugepages(file)) |
| 209 | return 1; |
| 210 | |
| 211 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 212 | } |
| 213 | |
Andi Kleen | 42d7395 | 2012-12-11 16:01:34 -0800 | [diff] [blame] | 214 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 215 | #else /* !CONFIG_HUGETLBFS */ |
| 216 | |
Stefan Richter | 1db8508 | 2009-02-10 23:27:32 +0100 | [diff] [blame] | 217 | #define is_file_hugepages(file) 0 |
Steven Truelove | 40716e2 | 2012-03-21 16:34:14 -0700 | [diff] [blame] | 218 | static inline struct file * |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 219 | hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag, |
| 220 | struct user_struct **user, int creat_flags, |
Andi Kleen | 42d7395 | 2012-12-11 16:01:34 -0800 | [diff] [blame] | 221 | int page_size_log) |
Andrew Morton | e9ea0e2 | 2009-09-24 14:47:45 -0700 | [diff] [blame] | 222 | { |
| 223 | return ERR_PTR(-ENOSYS); |
| 224 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | |
| 226 | #endif /* !CONFIG_HUGETLBFS */ |
| 227 | |
Adrian Bunk | d2ba27e8 | 2007-05-06 14:49:00 -0700 | [diff] [blame] | 228 | #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA |
| 229 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, |
| 230 | unsigned long len, unsigned long pgoff, |
| 231 | unsigned long flags); |
| 232 | #endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */ |
| 233 | |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 234 | #ifdef CONFIG_HUGETLB_PAGE |
| 235 | |
Nishanth Aravamudan | a343787 | 2008-07-23 21:27:44 -0700 | [diff] [blame] | 236 | #define HSTATE_NAME_LEN 32 |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 237 | /* Defines one hugetlb page size */ |
| 238 | struct hstate { |
Lee Schermerhorn | e8c5c82 | 2009-09-21 17:01:22 -0700 | [diff] [blame] | 239 | int next_nid_to_alloc; |
| 240 | int next_nid_to_free; |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 241 | unsigned int order; |
| 242 | unsigned long mask; |
| 243 | unsigned long max_huge_pages; |
| 244 | unsigned long nr_huge_pages; |
| 245 | unsigned long free_huge_pages; |
| 246 | unsigned long resv_huge_pages; |
| 247 | unsigned long surplus_huge_pages; |
| 248 | unsigned long nr_overcommit_huge_pages; |
Aneesh Kumar K.V | 0edaecf | 2012-07-31 16:42:07 -0700 | [diff] [blame] | 249 | struct list_head hugepage_activelist; |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 250 | struct list_head hugepage_freelists[MAX_NUMNODES]; |
| 251 | unsigned int nr_huge_pages_node[MAX_NUMNODES]; |
| 252 | unsigned int free_huge_pages_node[MAX_NUMNODES]; |
| 253 | unsigned int surplus_huge_pages_node[MAX_NUMNODES]; |
Aneesh Kumar K.V | abb8206 | 2012-07-31 16:42:24 -0700 | [diff] [blame] | 254 | #ifdef CONFIG_CGROUP_HUGETLB |
| 255 | /* cgroup control files */ |
| 256 | struct cftype cgroup_files[5]; |
| 257 | #endif |
Nishanth Aravamudan | a343787 | 2008-07-23 21:27:44 -0700 | [diff] [blame] | 258 | char name[HSTATE_NAME_LEN]; |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 259 | }; |
| 260 | |
Jon Tollefson | 53ba51d | 2008-07-23 21:27:52 -0700 | [diff] [blame] | 261 | struct huge_bootmem_page { |
| 262 | struct list_head list; |
| 263 | struct hstate *hstate; |
Becky Bruce | ee8f248 | 2011-07-25 17:11:50 -0700 | [diff] [blame] | 264 | #ifdef CONFIG_HIGHMEM |
| 265 | phys_addr_t phys; |
| 266 | #endif |
Jon Tollefson | 53ba51d | 2008-07-23 21:27:52 -0700 | [diff] [blame] | 267 | }; |
| 268 | |
Naoya Horiguchi | bf50bab | 2010-09-08 10:19:33 +0900 | [diff] [blame] | 269 | struct page *alloc_huge_page_node(struct hstate *h, int nid); |
Naoya Horiguchi | 74060e4 | 2013-09-11 14:22:06 -0700 | [diff] [blame] | 270 | struct page *alloc_huge_page_noerr(struct vm_area_struct *vma, |
| 271 | unsigned long addr, int avoid_reserve); |
Naoya Horiguchi | bf50bab | 2010-09-08 10:19:33 +0900 | [diff] [blame] | 272 | |
Jon Tollefson | 53ba51d | 2008-07-23 21:27:52 -0700 | [diff] [blame] | 273 | /* arch callback */ |
| 274 | int __init alloc_bootmem_huge_page(struct hstate *h); |
| 275 | |
Andi Kleen | e5ff215 | 2008-07-23 21:27:42 -0700 | [diff] [blame] | 276 | void __init hugetlb_add_hstate(unsigned order); |
| 277 | struct hstate *size_to_hstate(unsigned long size); |
| 278 | |
| 279 | #ifndef HUGE_MAX_HSTATE |
| 280 | #define HUGE_MAX_HSTATE 1 |
| 281 | #endif |
| 282 | |
| 283 | extern struct hstate hstates[HUGE_MAX_HSTATE]; |
| 284 | extern unsigned int default_hstate_idx; |
| 285 | |
| 286 | #define default_hstate (hstates[default_hstate_idx]) |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 287 | |
Andi Kleen | a137e1c | 2008-07-23 21:27:43 -0700 | [diff] [blame] | 288 | static inline struct hstate *hstate_inode(struct inode *i) |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 289 | { |
Andi Kleen | a137e1c | 2008-07-23 21:27:43 -0700 | [diff] [blame] | 290 | struct hugetlbfs_sb_info *hsb; |
| 291 | hsb = HUGETLBFS_SB(i->i_sb); |
| 292 | return hsb->hstate; |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 293 | } |
| 294 | |
| 295 | static inline struct hstate *hstate_file(struct file *f) |
| 296 | { |
Al Viro | 496ad9a | 2013-01-23 17:07:38 -0500 | [diff] [blame] | 297 | return hstate_inode(file_inode(f)); |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 298 | } |
| 299 | |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 300 | static inline struct hstate *hstate_sizelog(int page_size_log) |
| 301 | { |
| 302 | if (!page_size_log) |
| 303 | return &default_hstate; |
| 304 | return size_to_hstate(1 << page_size_log); |
| 305 | } |
| 306 | |
Andi Kleen | a137e1c | 2008-07-23 21:27:43 -0700 | [diff] [blame] | 307 | static inline struct hstate *hstate_vma(struct vm_area_struct *vma) |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 308 | { |
Andi Kleen | a137e1c | 2008-07-23 21:27:43 -0700 | [diff] [blame] | 309 | return hstate_file(vma->vm_file); |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 310 | } |
| 311 | |
| 312 | static inline unsigned long huge_page_size(struct hstate *h) |
| 313 | { |
| 314 | return (unsigned long)PAGE_SIZE << h->order; |
| 315 | } |
| 316 | |
Mel Gorman | 08fba69 | 2009-01-06 14:38:53 -0800 | [diff] [blame] | 317 | extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma); |
| 318 | |
Mel Gorman | 3340289 | 2009-01-06 14:38:54 -0800 | [diff] [blame] | 319 | extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); |
| 320 | |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 321 | static inline unsigned long huge_page_mask(struct hstate *h) |
| 322 | { |
| 323 | return h->mask; |
| 324 | } |
| 325 | |
| 326 | static inline unsigned int huge_page_order(struct hstate *h) |
| 327 | { |
| 328 | return h->order; |
| 329 | } |
| 330 | |
| 331 | static inline unsigned huge_page_shift(struct hstate *h) |
| 332 | { |
| 333 | return h->order + PAGE_SHIFT; |
| 334 | } |
| 335 | |
| 336 | static inline unsigned int pages_per_huge_page(struct hstate *h) |
| 337 | { |
| 338 | return 1 << h->order; |
| 339 | } |
| 340 | |
| 341 | static inline unsigned int blocks_per_huge_page(struct hstate *h) |
| 342 | { |
| 343 | return huge_page_size(h) / 512; |
| 344 | } |
| 345 | |
| 346 | #include <asm/hugetlb.h> |
| 347 | |
Chris Metcalf | d9ed9fa | 2012-04-01 14:01:34 -0400 | [diff] [blame] | 348 | #ifndef arch_make_huge_pte |
| 349 | static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, |
| 350 | struct page *page, int writable) |
| 351 | { |
| 352 | return entry; |
| 353 | } |
| 354 | #endif |
| 355 | |
Andi Kleen | e5ff215 | 2008-07-23 21:27:42 -0700 | [diff] [blame] | 356 | static inline struct hstate *page_hstate(struct page *page) |
| 357 | { |
Sasha Levin | 309381fea | 2014-01-23 15:52:54 -0800 | [diff] [blame] | 358 | VM_BUG_ON_PAGE(!PageHuge(page), page); |
Andi Kleen | e5ff215 | 2008-07-23 21:27:42 -0700 | [diff] [blame] | 359 | return size_to_hstate(PAGE_SIZE << compound_order(page)); |
| 360 | } |
| 361 | |
Andi Kleen | aa50d3a | 2010-10-06 21:45:00 +0200 | [diff] [blame] | 362 | static inline unsigned hstate_index_to_shift(unsigned index) |
| 363 | { |
| 364 | return hstates[index].order + PAGE_SHIFT; |
| 365 | } |
| 366 | |
Aneesh Kumar K.V | 972dc4d | 2012-07-31 16:42:00 -0700 | [diff] [blame] | 367 | static inline int hstate_index(struct hstate *h) |
| 368 | { |
| 369 | return h - hstates; |
| 370 | } |
| 371 | |
Zhang Yi | 13d60f4 | 2013-06-25 21:19:31 +0800 | [diff] [blame] | 372 | pgoff_t __basepage_index(struct page *page); |
| 373 | |
| 374 | /* Return page->index in PAGE_SIZE units */ |
| 375 | static inline pgoff_t basepage_index(struct page *page) |
| 376 | { |
| 377 | if (!PageCompound(page)) |
| 378 | return page->index; |
| 379 | |
| 380 | return __basepage_index(page); |
| 381 | } |
| 382 | |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 383 | extern void dissolve_free_huge_pages(unsigned long start_pfn, |
| 384 | unsigned long end_pfn); |
Naoya Horiguchi | 83467ef | 2013-09-11 14:22:11 -0700 | [diff] [blame] | 385 | int pmd_huge_support(void); |
| 386 | /* |
| 387 | * Currently hugepage migration is enabled only for pmd-based hugepage. |
| 388 | * This function will be updated when hugepage migration is more widely |
| 389 | * supported. |
| 390 | */ |
| 391 | static inline int hugepage_migration_support(struct hstate *h) |
| 392 | { |
| 393 | return pmd_huge_support() && (huge_page_shift(h) == PMD_SHIFT); |
| 394 | } |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 395 | |
Kirill A. Shutemov | cb900f4 | 2013-11-14 14:31:02 -0800 | [diff] [blame] | 396 | static inline spinlock_t *huge_pte_lockptr(struct hstate *h, |
| 397 | struct mm_struct *mm, pte_t *pte) |
| 398 | { |
| 399 | if (huge_page_size(h) == PMD_SIZE) |
| 400 | return pmd_lockptr(mm, (pmd_t *) pte); |
| 401 | VM_BUG_ON(huge_page_size(h) == PAGE_SIZE); |
| 402 | return &mm->page_table_lock; |
| 403 | } |
| 404 | |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 405 | #else /* CONFIG_HUGETLB_PAGE */ |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 406 | struct hstate {}; |
Naoya Horiguchi | bf50bab | 2010-09-08 10:19:33 +0900 | [diff] [blame] | 407 | #define alloc_huge_page_node(h, nid) NULL |
Naoya Horiguchi | 74060e4 | 2013-09-11 14:22:06 -0700 | [diff] [blame] | 408 | #define alloc_huge_page_noerr(v, a, r) NULL |
Jon Tollefson | 53ba51d | 2008-07-23 21:27:52 -0700 | [diff] [blame] | 409 | #define alloc_bootmem_huge_page(h) NULL |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 410 | #define hstate_file(f) NULL |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 411 | #define hstate_sizelog(s) NULL |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 412 | #define hstate_vma(v) NULL |
| 413 | #define hstate_inode(i) NULL |
Kirill A. Shutemov | cb900f4 | 2013-11-14 14:31:02 -0800 | [diff] [blame] | 414 | #define page_hstate(page) NULL |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 415 | #define huge_page_size(h) PAGE_SIZE |
| 416 | #define huge_page_mask(h) PAGE_MASK |
Mel Gorman | 08fba69 | 2009-01-06 14:38:53 -0800 | [diff] [blame] | 417 | #define vma_kernel_pagesize(v) PAGE_SIZE |
Mel Gorman | 3340289 | 2009-01-06 14:38:54 -0800 | [diff] [blame] | 418 | #define vma_mmu_pagesize(v) PAGE_SIZE |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 419 | #define huge_page_order(h) 0 |
| 420 | #define huge_page_shift(h) PAGE_SHIFT |
Andrea Righi | 510a35d | 2008-07-26 15:22:27 -0700 | [diff] [blame] | 421 | static inline unsigned int pages_per_huge_page(struct hstate *h) |
| 422 | { |
| 423 | return 1; |
| 424 | } |
Andi Kleen | aa50d3a | 2010-10-06 21:45:00 +0200 | [diff] [blame] | 425 | #define hstate_index_to_shift(index) 0 |
Aneesh Kumar K.V | 972dc4d | 2012-07-31 16:42:00 -0700 | [diff] [blame] | 426 | #define hstate_index(h) 0 |
Zhang Yi | 13d60f4 | 2013-06-25 21:19:31 +0800 | [diff] [blame] | 427 | |
| 428 | static inline pgoff_t basepage_index(struct page *page) |
| 429 | { |
| 430 | return page->index; |
| 431 | } |
Naoya Horiguchi | c8721bb | 2013-09-11 14:22:09 -0700 | [diff] [blame] | 432 | #define dissolve_free_huge_pages(s, e) do {} while (0) |
Naoya Horiguchi | 83467ef | 2013-09-11 14:22:11 -0700 | [diff] [blame] | 433 | #define pmd_huge_support() 0 |
| 434 | #define hugepage_migration_support(h) 0 |
Kirill A. Shutemov | cb900f4 | 2013-11-14 14:31:02 -0800 | [diff] [blame] | 435 | |
| 436 | static inline spinlock_t *huge_pte_lockptr(struct hstate *h, |
| 437 | struct mm_struct *mm, pte_t *pte) |
| 438 | { |
| 439 | return &mm->page_table_lock; |
| 440 | } |
Naoya Horiguchi | af73e4d | 2013-05-07 16:18:13 -0700 | [diff] [blame] | 441 | #endif /* CONFIG_HUGETLB_PAGE */ |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 442 | |
Kirill A. Shutemov | cb900f4 | 2013-11-14 14:31:02 -0800 | [diff] [blame] | 443 | static inline spinlock_t *huge_pte_lock(struct hstate *h, |
| 444 | struct mm_struct *mm, pte_t *pte) |
| 445 | { |
| 446 | spinlock_t *ptl; |
| 447 | |
| 448 | ptl = huge_pte_lockptr(h, mm, pte); |
| 449 | spin_lock(ptl); |
| 450 | return ptl; |
| 451 | } |
| 452 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 453 | #endif /* _LINUX_HUGETLB_H */ |