blob: 188e3e07eeeba7c0eb6555c138a16e97c3e5d787 [file] [log] [blame]
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -07001/*
2 * Handle caching attributes in page tables (PAT)
3 *
4 * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 * Suresh B Siddha <suresh.b.siddha@intel.com>
6 *
7 * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
8 */
9
Ingo Molnarad2cde12008-09-30 13:20:45 +020010#include <linux/seq_file.h>
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -070011#include <linux/bootmem.h>
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -070012#include <linux/debugfs.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020013#include <linux/kernel.h>
Ingo Molnar92b9af92009-02-28 14:09:27 +010014#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090015#include <linux/slab.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020016#include <linux/mm.h>
17#include <linux/fs.h>
Venkatesh Pallipadi335ef892009-07-10 09:57:36 -070018#include <linux/rbtree.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070019
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070020#include <asm/cacheflush.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020021#include <asm/processor.h>
22#include <asm/tlbflush.h>
Jack Steinerfd12a0d2009-11-19 14:23:41 -060023#include <asm/x86_init.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020024#include <asm/pgtable.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070025#include <asm/fcntl.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020026#include <asm/e820.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070027#include <asm/mtrr.h>
Ingo Molnarad2cde12008-09-30 13:20:45 +020028#include <asm/page.h>
29#include <asm/msr.h>
30#include <asm/pat.h>
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -070031#include <asm/io.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070032
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -080033#include "pat_internal.h"
Juergen Grossbd809af2014-11-03 14:02:03 +010034#include "mm_internal.h"
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -080035
Luis R. Rodriguez9e765612015-05-26 10:28:11 +020036#undef pr_fmt
37#define pr_fmt(fmt) "" fmt
38
Borislav Petkov9dac6292015-06-04 18:55:09 +020039static bool boot_cpu_done;
40
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +020041static int __read_mostly __pat_enabled = IS_ENABLED(CONFIG_X86_PAT);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070042
Marcin Slusarz1ee4bd92009-04-10 22:47:17 +020043static inline void pat_disable(const char *reason)
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020044{
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +020045 __pat_enabled = 0;
Luis R. Rodriguez9e765612015-05-26 10:28:11 +020046 pr_info("x86/PAT: %s\n", reason);
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020047}
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070048
Andrew Mortonbe524fb2008-05-29 00:01:28 -070049static int __init nopat(char *str)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070050{
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020051 pat_disable("PAT support disabled.");
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070052 return 0;
53}
54early_param("nopat", nopat);
55
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +020056bool pat_enabled(void)
57{
58 return !!__pat_enabled;
59}
Luis R. Rodriguezfbe71932015-05-26 10:28:16 +020060EXPORT_SYMBOL_GPL(pat_enabled);
Venki Pallipadi77b52b42008-05-05 19:09:10 -070061
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -080062int pat_debug_enable;
Ingo Molnarad2cde12008-09-30 13:20:45 +020063
Venki Pallipadi77b52b42008-05-05 19:09:10 -070064static int __init pat_debug_setup(char *str)
65{
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -080066 pat_debug_enable = 1;
Venki Pallipadi77b52b42008-05-05 19:09:10 -070067 return 0;
68}
69__setup("debugpat", pat_debug_setup);
70
Thomas Gleixner0dbcae82014-11-16 18:59:19 +010071#ifdef CONFIG_X86_PAT
72/*
Toshi Kani35a5a102015-06-04 18:55:19 +020073 * X86 PAT uses page flags arch_1 and uncached together to keep track of
74 * memory type of pages that have backing page struct.
75 *
76 * X86 PAT supports 4 different memory types:
77 * - _PAGE_CACHE_MODE_WB
78 * - _PAGE_CACHE_MODE_WC
79 * - _PAGE_CACHE_MODE_UC_MINUS
80 * - _PAGE_CACHE_MODE_WT
81 *
82 * _PAGE_CACHE_MODE_WB is the default type.
Thomas Gleixner0dbcae82014-11-16 18:59:19 +010083 */
84
Toshi Kani35a5a102015-06-04 18:55:19 +020085#define _PGMT_WB 0
Thomas Gleixner0dbcae82014-11-16 18:59:19 +010086#define _PGMT_WC (1UL << PG_arch_1)
87#define _PGMT_UC_MINUS (1UL << PG_uncached)
Toshi Kani35a5a102015-06-04 18:55:19 +020088#define _PGMT_WT (1UL << PG_uncached | 1UL << PG_arch_1)
Thomas Gleixner0dbcae82014-11-16 18:59:19 +010089#define _PGMT_MASK (1UL << PG_uncached | 1UL << PG_arch_1)
90#define _PGMT_CLEAR_MASK (~_PGMT_MASK)
91
92static inline enum page_cache_mode get_page_memtype(struct page *pg)
93{
94 unsigned long pg_flags = pg->flags & _PGMT_MASK;
95
Toshi Kani35a5a102015-06-04 18:55:19 +020096 if (pg_flags == _PGMT_WB)
97 return _PAGE_CACHE_MODE_WB;
Thomas Gleixner0dbcae82014-11-16 18:59:19 +010098 else if (pg_flags == _PGMT_WC)
99 return _PAGE_CACHE_MODE_WC;
100 else if (pg_flags == _PGMT_UC_MINUS)
101 return _PAGE_CACHE_MODE_UC_MINUS;
102 else
Toshi Kani35a5a102015-06-04 18:55:19 +0200103 return _PAGE_CACHE_MODE_WT;
Thomas Gleixner0dbcae82014-11-16 18:59:19 +0100104}
105
106static inline void set_page_memtype(struct page *pg,
107 enum page_cache_mode memtype)
108{
109 unsigned long memtype_flags;
110 unsigned long old_flags;
111 unsigned long new_flags;
112
113 switch (memtype) {
114 case _PAGE_CACHE_MODE_WC:
115 memtype_flags = _PGMT_WC;
116 break;
117 case _PAGE_CACHE_MODE_UC_MINUS:
118 memtype_flags = _PGMT_UC_MINUS;
119 break;
Toshi Kani35a5a102015-06-04 18:55:19 +0200120 case _PAGE_CACHE_MODE_WT:
121 memtype_flags = _PGMT_WT;
Thomas Gleixner0dbcae82014-11-16 18:59:19 +0100122 break;
Toshi Kani35a5a102015-06-04 18:55:19 +0200123 case _PAGE_CACHE_MODE_WB:
Thomas Gleixner0dbcae82014-11-16 18:59:19 +0100124 default:
Toshi Kani35a5a102015-06-04 18:55:19 +0200125 memtype_flags = _PGMT_WB;
Thomas Gleixner0dbcae82014-11-16 18:59:19 +0100126 break;
127 }
128
129 do {
130 old_flags = pg->flags;
131 new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
132 } while (cmpxchg(&pg->flags, old_flags, new_flags) != old_flags);
133}
134#else
135static inline enum page_cache_mode get_page_memtype(struct page *pg)
136{
137 return -1;
138}
139static inline void set_page_memtype(struct page *pg,
140 enum page_cache_mode memtype)
141{
142}
143#endif
144
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700145enum {
146 PAT_UC = 0, /* uncached */
147 PAT_WC = 1, /* Write combining */
148 PAT_WT = 4, /* Write Through */
149 PAT_WP = 5, /* Write Protected */
150 PAT_WB = 6, /* Write Back (default) */
151 PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */
152};
153
Juergen Grossbd809af2014-11-03 14:02:03 +0100154#define CM(c) (_PAGE_CACHE_MODE_ ## c)
155
156static enum page_cache_mode pat_get_cache_mode(unsigned pat_val, char *msg)
157{
158 enum page_cache_mode cache;
159 char *cache_mode;
160
161 switch (pat_val) {
162 case PAT_UC: cache = CM(UC); cache_mode = "UC "; break;
163 case PAT_WC: cache = CM(WC); cache_mode = "WC "; break;
164 case PAT_WT: cache = CM(WT); cache_mode = "WT "; break;
165 case PAT_WP: cache = CM(WP); cache_mode = "WP "; break;
166 case PAT_WB: cache = CM(WB); cache_mode = "WB "; break;
167 case PAT_UC_MINUS: cache = CM(UC_MINUS); cache_mode = "UC- "; break;
168 default: cache = CM(WB); cache_mode = "WB "; break;
169 }
170
171 memcpy(msg, cache_mode, 4);
172
173 return cache;
174}
175
176#undef CM
177
178/*
179 * Update the cache mode to pgprot translation tables according to PAT
180 * configuration.
181 * Using lower indices is preferred, so we start with highest index.
182 */
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200183void pat_init_cache_modes(u64 pat)
Juergen Grossbd809af2014-11-03 14:02:03 +0100184{
Juergen Grossbd809af2014-11-03 14:02:03 +0100185 enum page_cache_mode cache;
186 char pat_msg[33];
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200187 int i;
Juergen Grossbd809af2014-11-03 14:02:03 +0100188
Juergen Grossbd809af2014-11-03 14:02:03 +0100189 pat_msg[32] = 0;
190 for (i = 7; i >= 0; i--) {
191 cache = pat_get_cache_mode((pat >> (i * 8)) & 7,
192 pat_msg + 4 * i);
193 update_cache_mode_entry(i, cache);
194 }
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200195 pr_info("x86/PAT: Configuration [0-7]: %s\n", pat_msg);
Juergen Grossbd809af2014-11-03 14:02:03 +0100196}
197
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200198#define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700199
Borislav Petkov9dac6292015-06-04 18:55:09 +0200200static void pat_bsp_init(u64 pat)
201{
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200202 u64 tmp_pat;
203
Borislav Petkov9dac6292015-06-04 18:55:09 +0200204 if (!cpu_has_pat) {
205 pat_disable("PAT not supported by CPU.");
206 return;
207 }
208
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200209 if (!pat_enabled())
210 goto done;
211
212 rdmsrl(MSR_IA32_CR_PAT, tmp_pat);
213 if (!tmp_pat) {
Borislav Petkov9dac6292015-06-04 18:55:09 +0200214 pat_disable("PAT MSR is 0, disabled.");
215 return;
216 }
217
218 wrmsrl(MSR_IA32_CR_PAT, pat);
219
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200220done:
221 pat_init_cache_modes(pat);
Borislav Petkov9dac6292015-06-04 18:55:09 +0200222}
223
224static void pat_ap_init(u64 pat)
225{
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200226 if (!pat_enabled())
227 return;
228
Borislav Petkov9dac6292015-06-04 18:55:09 +0200229 if (!cpu_has_pat) {
230 /*
231 * If this happens we are on a secondary CPU, but switched to
232 * PAT on the boot CPU. We have no way to undo PAT.
233 */
234 panic("x86/PAT: PAT enabled, but not supported by secondary CPU\n");
235 }
236
237 wrmsrl(MSR_IA32_CR_PAT, pat);
238}
239
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700240void pat_init(void)
241{
242 u64 pat;
Toshi Kanid79a40c2015-06-04 18:55:12 +0200243 struct cpuinfo_x86 *c = &boot_cpu_data;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700244
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200245 if (!pat_enabled()) {
246 /*
247 * No PAT. Emulate the PAT table that corresponds to the two
248 * cache bits, PWT (Write Through) and PCD (Cache Disable). This
249 * setup is the same as the BIOS default setup when the system
250 * has PAT but the "nopat" boot option has been specified. This
251 * emulated PAT table is used when MSR_IA32_CR_PAT returns 0.
252 *
Toshi Kanid79a40c2015-06-04 18:55:12 +0200253 * PTE encoding:
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200254 *
255 * PCD
256 * |PWT PAT
257 * || slot
258 * 00 0 WB : _PAGE_CACHE_MODE_WB
259 * 01 1 WT : _PAGE_CACHE_MODE_WT
260 * 10 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
261 * 11 3 UC : _PAGE_CACHE_MODE_UC
262 *
263 * NOTE: When WC or WP is used, it is redirected to UC- per
264 * the default setup in __cachemode2pte_tbl[].
265 */
266 pat = PAT(0, WB) | PAT(1, WT) | PAT(2, UC_MINUS) | PAT(3, UC) |
267 PAT(4, WB) | PAT(5, WT) | PAT(6, UC_MINUS) | PAT(7, UC);
Toshi Kanid79a40c2015-06-04 18:55:12 +0200268
269 } else if ((c->x86_vendor == X86_VENDOR_INTEL) &&
270 (((c->x86 == 0x6) && (c->x86_model <= 0xd)) ||
271 ((c->x86 == 0xf) && (c->x86_model <= 0x6)))) {
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200272 /*
Toshi Kanid79a40c2015-06-04 18:55:12 +0200273 * PAT support with the lower four entries. Intel Pentium 2,
274 * 3, M, and 4 are affected by PAT errata, which makes the
275 * upper four entries unusable. To be on the safe side, we don't
276 * use those.
277 *
278 * PTE encoding:
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200279 * PAT
280 * |PCD
Toshi Kanid79a40c2015-06-04 18:55:12 +0200281 * ||PWT PAT
282 * ||| slot
283 * 000 0 WB : _PAGE_CACHE_MODE_WB
284 * 001 1 WC : _PAGE_CACHE_MODE_WC
285 * 010 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
286 * 011 3 UC : _PAGE_CACHE_MODE_UC
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200287 * PAT bit unused
Toshi Kanid79a40c2015-06-04 18:55:12 +0200288 *
289 * NOTE: When WT or WP is used, it is redirected to UC- per
290 * the default setup in __cachemode2pte_tbl[].
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200291 */
292 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
293 PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
Toshi Kanid79a40c2015-06-04 18:55:12 +0200294 } else {
295 /*
296 * Full PAT support. We put WT in slot 7 to improve
297 * robustness in the presence of errata that might cause
298 * the high PAT bit to be ignored. This way, a buggy slot 7
299 * access will hit slot 3, and slot 3 is UC, so at worst
300 * we lose performance without causing a correctness issue.
301 * Pentium 4 erratum N46 is an example for such an erratum,
302 * although we try not to use PAT at all on affected CPUs.
303 *
304 * PTE encoding:
305 * PAT
306 * |PCD
307 * ||PWT PAT
308 * ||| slot
309 * 000 0 WB : _PAGE_CACHE_MODE_WB
310 * 001 1 WC : _PAGE_CACHE_MODE_WC
311 * 010 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
312 * 011 3 UC : _PAGE_CACHE_MODE_UC
313 * 100 4 WB : Reserved
314 * 101 5 WC : Reserved
315 * 110 6 UC-: Reserved
316 * 111 7 WT : _PAGE_CACHE_MODE_WT
317 *
318 * The reserved slots are unused, but mapped to their
319 * corresponding types in the presence of PAT errata.
320 */
321 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
322 PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, WT);
Borislav Petkov9cd25aa2015-06-04 18:55:10 +0200323 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700324
Borislav Petkov9dac6292015-06-04 18:55:09 +0200325 if (!boot_cpu_done) {
326 pat_bsp_init(pat);
327 boot_cpu_done = true;
328 } else {
329 pat_ap_init(pat);
Juergen Gross9d34cfd2015-01-12 06:15:45 +0100330 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700331}
332
333#undef PAT
334
Pallipadi, Venkatesh9e41a492010-02-10 15:26:07 -0800335static DEFINE_SPINLOCK(memtype_lock); /* protects memtype accesses */
Venkatesh Pallipadi335ef892009-07-10 09:57:36 -0700336
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700337/*
338 * Does intersection of PAT memory type and MTRR memory type and returns
339 * the resulting memory type as PAT understands it.
340 * (Type in pat and mtrr will not have same value)
341 * The intersection is based on "Effective Memory Type" tables in IA-32
342 * SDM vol 3a
343 */
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100344static unsigned long pat_x_mtrr_type(u64 start, u64 end,
345 enum page_cache_mode req_type)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700346{
Venki Pallipadic26421d2008-05-29 12:01:44 -0700347 /*
348 * Look for MTRR hint to get the effective type in case where PAT
349 * request is for WB.
350 */
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100351 if (req_type == _PAGE_CACHE_MODE_WB) {
Toshi Kanib73522e2015-05-26 10:28:10 +0200352 u8 mtrr_type, uniform;
Andreas Herrmanndd0c7c42008-06-18 15:38:57 +0200353
Toshi Kanib73522e2015-05-26 10:28:10 +0200354 mtrr_type = mtrr_type_lookup(start, end, &uniform);
Suresh Siddhab6ff32d2009-04-09 14:26:51 -0700355 if (mtrr_type != MTRR_TYPE_WRBACK)
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100356 return _PAGE_CACHE_MODE_UC_MINUS;
Suresh Siddhab6ff32d2009-04-09 14:26:51 -0700357
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100358 return _PAGE_CACHE_MODE_WB;
Andreas Herrmanndd0c7c42008-06-18 15:38:57 +0200359 }
360
361 return req_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700362}
363
John Dykstrafa835232012-05-25 16:12:46 -0500364struct pagerange_state {
365 unsigned long cur_pfn;
366 int ram;
367 int not_ram;
368};
369
370static int
371pagerange_is_ram_callback(unsigned long initial_pfn, unsigned long total_nr_pages, void *arg)
372{
373 struct pagerange_state *state = arg;
374
375 state->not_ram |= initial_pfn > state->cur_pfn;
376 state->ram |= total_nr_pages > 0;
377 state->cur_pfn = initial_pfn + total_nr_pages;
378
379 return state->ram && state->not_ram;
380}
381
Yasuaki Ishimatsu3709c852010-07-22 14:57:35 +0900382static int pat_pagerange_is_ram(resource_size_t start, resource_size_t end)
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800383{
John Dykstrafa835232012-05-25 16:12:46 -0500384 int ret = 0;
385 unsigned long start_pfn = start >> PAGE_SHIFT;
386 unsigned long end_pfn = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
387 struct pagerange_state state = {start_pfn, 0, 0};
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800388
John Dykstrafa835232012-05-25 16:12:46 -0500389 /*
390 * For legacy reasons, physical address range in the legacy ISA
391 * region is tracked as non-RAM. This will allow users of
392 * /dev/mem to map portions of legacy ISA region, even when
393 * some of those portions are listed(or not even listed) with
394 * different e820 types(RAM/reserved/..)
395 */
396 if (start_pfn < ISA_END_ADDRESS >> PAGE_SHIFT)
397 start_pfn = ISA_END_ADDRESS >> PAGE_SHIFT;
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800398
John Dykstrafa835232012-05-25 16:12:46 -0500399 if (start_pfn < end_pfn) {
400 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
401 &state, pagerange_is_ram_callback);
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800402 }
403
John Dykstrafa835232012-05-25 16:12:46 -0500404 return (ret > 0) ? -1 : (state.ram ? 1 : 0);
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800405}
406
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700407/*
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700408 * For RAM pages, we use page flags to mark the pages with appropriate type.
Toshi Kani35a5a102015-06-04 18:55:19 +0200409 * The page flags are limited to four types, WB (default), WC, WT and UC-.
410 * WP request fails with -EINVAL, and UC gets redirected to UC-. Setting
411 * a new memory type is only allowed for a page mapped with the default WB
412 * type.
Toshi Kani0d69bdf2015-06-04 18:55:13 +0200413 *
414 * Here we do two passes:
415 * - Find the memtype of all the pages in the range, look for any conflicts.
416 * - In case of no conflicts, set the new memtype for pages in the range.
Suresh Siddha9542ada2008-09-24 08:53:33 -0700417 */
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100418static int reserve_ram_pages_type(u64 start, u64 end,
419 enum page_cache_mode req_type,
420 enum page_cache_mode *new_type)
Suresh Siddha9542ada2008-09-24 08:53:33 -0700421{
422 struct page *page;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700423 u64 pfn;
424
Toshi Kani35a5a102015-06-04 18:55:19 +0200425 if (req_type == _PAGE_CACHE_MODE_WP) {
Toshi Kani0d69bdf2015-06-04 18:55:13 +0200426 if (new_type)
427 *new_type = _PAGE_CACHE_MODE_UC_MINUS;
428 return -EINVAL;
429 }
430
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100431 if (req_type == _PAGE_CACHE_MODE_UC) {
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700432 /* We do not support strong UC */
433 WARN_ON_ONCE(1);
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100434 req_type = _PAGE_CACHE_MODE_UC_MINUS;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700435 }
436
437 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100438 enum page_cache_mode type;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700439
440 page = pfn_to_page(pfn);
441 type = get_page_memtype(page);
Toshi Kani35a5a102015-06-04 18:55:19 +0200442 if (type != _PAGE_CACHE_MODE_WB) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200443 pr_info("x86/PAT: reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%x, req 0x%x\n",
Bjorn Helgaas365811d2012-05-29 15:06:29 -0700444 start, end - 1, type, req_type);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700445 if (new_type)
446 *new_type = type;
447
448 return -EBUSY;
449 }
450 }
451
452 if (new_type)
453 *new_type = req_type;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700454
455 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
456 page = pfn_to_page(pfn);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700457 set_page_memtype(page, req_type);
Suresh Siddha9542ada2008-09-24 08:53:33 -0700458 }
459 return 0;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700460}
461
462static int free_ram_pages_type(u64 start, u64 end)
463{
464 struct page *page;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700465 u64 pfn;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700466
467 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
468 page = pfn_to_page(pfn);
Toshi Kani35a5a102015-06-04 18:55:19 +0200469 set_page_memtype(page, _PAGE_CACHE_MODE_WB);
Suresh Siddha9542ada2008-09-24 08:53:33 -0700470 }
471 return 0;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700472}
473
474/*
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700475 * req_type typically has one of the:
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100476 * - _PAGE_CACHE_MODE_WB
477 * - _PAGE_CACHE_MODE_WC
478 * - _PAGE_CACHE_MODE_UC_MINUS
479 * - _PAGE_CACHE_MODE_UC
Toshi Kani0d69bdf2015-06-04 18:55:13 +0200480 * - _PAGE_CACHE_MODE_WT
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700481 *
Andreas Herrmannac979912008-06-20 22:01:49 +0200482 * If new_type is NULL, function will return an error if it cannot reserve the
483 * region with req_type. If new_type is non-NULL, function will return
484 * available type in new_type in case of no error. In case of any error
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700485 * it will return a negative return value.
486 */
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100487int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
488 enum page_cache_mode *new_type)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700489{
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -0800490 struct memtype *new;
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100491 enum page_cache_mode actual_type;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700492 int is_range_ram;
Ingo Molnarad2cde12008-09-30 13:20:45 +0200493 int err = 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700494
Ingo Molnarad2cde12008-09-30 13:20:45 +0200495 BUG_ON(start >= end); /* end is exclusive */
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200496
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200497 if (!pat_enabled()) {
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700498 /* This is identical to page table setting without PAT */
Borislav Petkov7202fdb2015-06-04 18:55:11 +0200499 if (new_type)
500 *new_type = req_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700501 return 0;
502 }
503
504 /* Low ISA region is always mapped WB in page table. No need to track */
H. Peter Anvin8a271382009-11-23 14:49:20 -0800505 if (x86_platform.is_untracked_pat_range(start, end)) {
Andreas Herrmannac979912008-06-20 22:01:49 +0200506 if (new_type)
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100507 *new_type = _PAGE_CACHE_MODE_WB;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700508 return 0;
509 }
510
Suresh Siddhab6ff32d2009-04-09 14:26:51 -0700511 /*
512 * Call mtrr_lookup to get the type hint. This is an
513 * optimization for /dev/mem mmap'ers into WB memory (BIOS
514 * tools and ACPI tools). Use WB request for WB memory and use
515 * UC_MINUS otherwise.
516 */
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100517 actual_type = pat_x_mtrr_type(start, end, req_type);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700518
Suresh Siddha95971342009-01-13 10:21:30 -0800519 if (new_type)
520 *new_type = actual_type;
521
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800522 is_range_ram = pat_pagerange_is_ram(start, end);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700523 if (is_range_ram == 1) {
524
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700525 err = reserve_ram_pages_type(start, end, req_type, new_type);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700526
527 return err;
528 } else if (is_range_ram < 0) {
Suresh Siddha9542ada2008-09-24 08:53:33 -0700529 return -EINVAL;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700530 }
Suresh Siddha9542ada2008-09-24 08:53:33 -0700531
Venkatesh Pallipadi6a4f3b52010-06-10 17:45:01 -0700532 new = kzalloc(sizeof(struct memtype), GFP_KERNEL);
Andreas Herrmannac979912008-06-20 22:01:49 +0200533 if (!new)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700534 return -ENOMEM;
535
Ingo Molnarad2cde12008-09-30 13:20:45 +0200536 new->start = start;
537 new->end = end;
538 new->type = actual_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700539
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700540 spin_lock(&memtype_lock);
541
Pallipadi, Venkatesh9e41a492010-02-10 15:26:07 -0800542 err = rbt_memtype_check_insert(new, new_type);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700543 if (err) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200544 pr_info("x86/PAT: reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
545 start, end - 1,
546 cattr_name(new->type), cattr_name(req_type));
Andreas Herrmannac979912008-06-20 22:01:49 +0200547 kfree(new);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700548 spin_unlock(&memtype_lock);
Ingo Molnarad2cde12008-09-30 13:20:45 +0200549
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700550 return err;
551 }
552
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700553 spin_unlock(&memtype_lock);
Andreas Herrmann3e9c83b2008-06-20 22:04:02 +0200554
Bjorn Helgaas365811d2012-05-29 15:06:29 -0700555 dprintk("reserve_memtype added [mem %#010Lx-%#010Lx], track %s, req %s, ret %s\n",
556 start, end - 1, cattr_name(new->type), cattr_name(req_type),
Andreas Herrmann3e9c83b2008-06-20 22:04:02 +0200557 new_type ? cattr_name(*new_type) : "-");
558
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700559 return err;
560}
561
562int free_memtype(u64 start, u64 end)
563{
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700564 int err = -EINVAL;
Suresh Siddha9542ada2008-09-24 08:53:33 -0700565 int is_range_ram;
Xiaotian Feng20413f22010-05-26 09:51:10 +0800566 struct memtype *entry;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700567
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200568 if (!pat_enabled())
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700569 return 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700570
571 /* Low ISA region is always mapped WB. No need to track */
H. Peter Anvin8a271382009-11-23 14:49:20 -0800572 if (x86_platform.is_untracked_pat_range(start, end))
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700573 return 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700574
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800575 is_range_ram = pat_pagerange_is_ram(start, end);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700576 if (is_range_ram == 1) {
577
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700578 err = free_ram_pages_type(start, end);
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700579
580 return err;
581 } else if (is_range_ram < 0) {
Suresh Siddha9542ada2008-09-24 08:53:33 -0700582 return -EINVAL;
Venkatesh Pallipadif5841742009-07-10 09:57:38 -0700583 }
Suresh Siddha9542ada2008-09-24 08:53:33 -0700584
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700585 spin_lock(&memtype_lock);
Xiaotian Feng20413f22010-05-26 09:51:10 +0800586 entry = rbt_memtype_erase(start, end);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700587 spin_unlock(&memtype_lock);
588
Xiaotian Feng20413f22010-05-26 09:51:10 +0800589 if (!entry) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200590 pr_info("x86/PAT: %s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
591 current->comm, current->pid, start, end - 1);
Xiaotian Feng20413f22010-05-26 09:51:10 +0800592 return -EINVAL;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700593 }
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700594
Xiaotian Feng20413f22010-05-26 09:51:10 +0800595 kfree(entry);
596
Bjorn Helgaas365811d2012-05-29 15:06:29 -0700597 dprintk("free_memtype request [mem %#010Lx-%#010Lx]\n", start, end - 1);
Ingo Molnarad2cde12008-09-30 13:20:45 +0200598
Xiaotian Feng20413f22010-05-26 09:51:10 +0800599 return 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700600}
601
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700602
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700603/**
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700604 * lookup_memtype - Looksup the memory type for a physical address
605 * @paddr: physical address of which memory type needs to be looked up
606 *
607 * Only to be called when PAT is enabled
608 *
Juergen Gross2a374692014-11-03 14:01:55 +0100609 * Returns _PAGE_CACHE_MODE_WB, _PAGE_CACHE_MODE_WC, _PAGE_CACHE_MODE_UC_MINUS
Toshi Kani35a5a102015-06-04 18:55:19 +0200610 * or _PAGE_CACHE_MODE_WT.
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700611 */
Juergen Gross2a374692014-11-03 14:01:55 +0100612static enum page_cache_mode lookup_memtype(u64 paddr)
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700613{
Juergen Gross2a374692014-11-03 14:01:55 +0100614 enum page_cache_mode rettype = _PAGE_CACHE_MODE_WB;
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700615 struct memtype *entry;
616
H. Peter Anvin8a271382009-11-23 14:49:20 -0800617 if (x86_platform.is_untracked_pat_range(paddr, paddr + PAGE_SIZE))
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700618 return rettype;
619
620 if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
621 struct page *page;
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700622
Toshi Kani35a5a102015-06-04 18:55:19 +0200623 page = pfn_to_page(paddr >> PAGE_SHIFT);
624 return get_page_memtype(page);
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700625 }
626
627 spin_lock(&memtype_lock);
628
Pallipadi, Venkatesh9e41a492010-02-10 15:26:07 -0800629 entry = rbt_memtype_lookup(paddr);
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700630 if (entry != NULL)
631 rettype = entry->type;
632 else
Juergen Gross2a374692014-11-03 14:01:55 +0100633 rettype = _PAGE_CACHE_MODE_UC_MINUS;
Venkatesh Pallipadi637b86e2009-07-10 09:57:39 -0700634
635 spin_unlock(&memtype_lock);
636 return rettype;
637}
638
639/**
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700640 * io_reserve_memtype - Request a memory type mapping for a region of memory
641 * @start: start (physical address) of the region
642 * @end: end (physical address) of the region
643 * @type: A pointer to memtype, with requested type. On success, requested
644 * or any other compatible type that was available for the region is returned
645 *
646 * On success, returns 0
647 * On failure, returns non-zero
648 */
649int io_reserve_memtype(resource_size_t start, resource_size_t end,
Juergen Gross49a3b3c2014-11-03 14:01:54 +0100650 enum page_cache_mode *type)
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700651{
H. Peter Anvinb8551922009-08-26 17:17:51 -0700652 resource_size_t size = end - start;
Juergen Gross49a3b3c2014-11-03 14:01:54 +0100653 enum page_cache_mode req_type = *type;
654 enum page_cache_mode new_type;
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700655 int ret;
656
H. Peter Anvinb8551922009-08-26 17:17:51 -0700657 WARN_ON_ONCE(iomem_map_sanity_check(start, size));
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700658
659 ret = reserve_memtype(start, end, req_type, &new_type);
660 if (ret)
661 goto out_err;
662
H. Peter Anvinb8551922009-08-26 17:17:51 -0700663 if (!is_new_memtype_allowed(start, size, req_type, new_type))
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700664 goto out_free;
665
H. Peter Anvinb8551922009-08-26 17:17:51 -0700666 if (kernel_map_sync_memtype(start, size, new_type) < 0)
Venkatesh Pallipadi9fd126b2009-07-10 09:57:34 -0700667 goto out_free;
668
669 *type = new_type;
670 return 0;
671
672out_free:
673 free_memtype(start, end);
674 ret = -EBUSY;
675out_err:
676 return ret;
677}
678
679/**
680 * io_free_memtype - Release a memory type mapping for a region of memory
681 * @start: start (physical address) of the region
682 * @end: end (physical address) of the region
683 */
684void io_free_memtype(resource_size_t start, resource_size_t end)
685{
686 free_memtype(start, end);
687}
688
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700689pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
690 unsigned long size, pgprot_t vma_prot)
691{
692 return vma_prot;
693}
694
Ingo Molnard0926332008-07-18 00:26:59 +0200695#ifdef CONFIG_STRICT_DEVMEM
Pavel Machek1f40a8b2014-12-28 17:15:24 +0100696/* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM */
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700697static inline int range_is_allowed(unsigned long pfn, unsigned long size)
698{
699 return 1;
700}
701#else
Ravikiran G Thirumalai9e41bff2008-10-30 13:59:21 -0700702/* This check is needed to avoid cache aliasing when PAT is enabled */
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700703static inline int range_is_allowed(unsigned long pfn, unsigned long size)
704{
705 u64 from = ((u64)pfn) << PAGE_SHIFT;
706 u64 to = from + size;
707 u64 cursor = from;
708
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200709 if (!pat_enabled())
Ravikiran G Thirumalai9e41bff2008-10-30 13:59:21 -0700710 return 1;
711
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700712 while (cursor < to) {
713 if (!devmem_is_allowed(pfn)) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200714 pr_info("x86/PAT: Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx], PAT prevents it\n",
715 current->comm, from, to - 1);
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700716 return 0;
717 }
718 cursor += PAGE_SIZE;
719 pfn++;
720 }
721 return 1;
722}
Ingo Molnard0926332008-07-18 00:26:59 +0200723#endif /* CONFIG_STRICT_DEVMEM */
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700724
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700725int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
726 unsigned long size, pgprot_t *vma_prot)
727{
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100728 enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700729
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700730 if (!range_is_allowed(pfn, size))
731 return 0;
732
Christoph Hellwig6b2f3d12009-10-27 11:05:28 +0100733 if (file->f_flags & O_DSYNC)
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100734 pcm = _PAGE_CACHE_MODE_UC_MINUS;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700735
736#ifdef CONFIG_X86_32
737 /*
738 * On the PPro and successors, the MTRRs are used to set
739 * memory types for physical addresses outside main memory,
740 * so blindly setting UC or PWT on those pages is wrong.
741 * For Pentiums and earlier, the surround logic should disable
742 * caching for the high addresses through the KEN pin, but
743 * we maintain the tradition of paranoia in this code.
744 */
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200745 if (!pat_enabled() &&
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200746 !(boot_cpu_has(X86_FEATURE_MTRR) ||
747 boot_cpu_has(X86_FEATURE_K6_MTRR) ||
748 boot_cpu_has(X86_FEATURE_CYRIX_ARR) ||
749 boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) &&
750 (pfn << PAGE_SHIFT) >= __pa(high_memory)) {
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100751 pcm = _PAGE_CACHE_MODE_UC;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700752 }
753#endif
754
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700755 *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100756 cachemode2protval(pcm));
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700757 return 1;
758}
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700759
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800760/*
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800761 * Change the memory type for the physial address range in kernel identity
762 * mapping space if that range is a part of identity map.
763 */
Juergen Grossb14097b2014-11-03 14:01:58 +0100764int kernel_map_sync_memtype(u64 base, unsigned long size,
765 enum page_cache_mode pcm)
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800766{
767 unsigned long id_sz;
768
Dave Hansena25b9312013-01-22 13:24:30 -0800769 if (base > __pa(high_memory-1))
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800770 return 0;
771
Dave Hansen60f583d2013-03-07 08:31:51 -0800772 /*
773 * some areas in the middle of the kernel identity range
774 * are not mapped, like the PCI space.
775 */
776 if (!page_is_ram(base >> PAGE_SHIFT))
777 return 0;
778
Dave Hansena25b9312013-01-22 13:24:30 -0800779 id_sz = (__pa(high_memory-1) <= base + size) ?
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800780 __pa(high_memory) - base :
781 size;
782
Juergen Grossb14097b2014-11-03 14:01:58 +0100783 if (ioremap_change_attr((unsigned long)__va(base), id_sz, pcm) < 0) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200784 pr_info("x86/PAT: %s:%d ioremap_change_attr failed %s for [mem %#010Lx-%#010Lx]\n",
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800785 current->comm, current->pid,
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100786 cattr_name(pcm),
Bjorn Helgaas365811d2012-05-29 15:06:29 -0700787 base, (unsigned long long)(base + size-1));
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800788 return -EINVAL;
789 }
790 return 0;
791}
792
793/*
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800794 * Internal interface to reserve a range of physical memory with prot.
795 * Reserved non RAM regions only and after successful reserve_memtype,
796 * this func also keeps identity mapping (if any) in sync with this new prot.
797 */
venkatesh.pallipadi@intel.comcdecff62009-01-09 16:13:12 -0800798static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
799 int strict_prot)
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800800{
801 int is_ram = 0;
Venkatesh Pallipadi7880f742009-02-24 17:35:13 -0800802 int ret;
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100803 enum page_cache_mode want_pcm = pgprot2cachemode(*vma_prot);
804 enum page_cache_mode pcm = want_pcm;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800805
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800806 is_ram = pat_pagerange_is_ram(paddr, paddr + size);
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800807
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800808 /*
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700809 * reserve_pfn_range() for RAM pages. We do not refcount to keep
810 * track of number of mappings of RAM pages. We can assert that
811 * the type requested matches the type of first page in the range.
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800812 */
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700813 if (is_ram) {
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200814 if (!pat_enabled())
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700815 return 0;
816
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100817 pcm = lookup_memtype(paddr);
818 if (want_pcm != pcm) {
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200819 pr_warn("x86/PAT: %s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700820 current->comm, current->pid,
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100821 cattr_name(want_pcm),
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700822 (unsigned long long)paddr,
Bjorn Helgaas365811d2012-05-29 15:06:29 -0700823 (unsigned long long)(paddr + size - 1),
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100824 cattr_name(pcm));
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700825 *vma_prot = __pgprot((pgprot_val(*vma_prot) &
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100826 (~_PAGE_CACHE_MASK)) |
827 cachemode2protval(pcm));
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700828 }
Pallipadi, Venkatesh4bb9c5c2009-03-12 17:45:27 -0700829 return 0;
Venkatesh Pallipadid886c732009-07-10 09:57:41 -0700830 }
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800831
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100832 ret = reserve_memtype(paddr, paddr + size, want_pcm, &pcm);
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800833 if (ret)
834 return ret;
835
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100836 if (pcm != want_pcm) {
Suresh Siddha1adcaaf2009-08-17 13:23:50 -0700837 if (strict_prot ||
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100838 !is_new_memtype_allowed(paddr, size, want_pcm, pcm)) {
venkatesh.pallipadi@intel.comcdecff62009-01-09 16:13:12 -0800839 free_memtype(paddr, paddr + size);
Luis R. Rodriguez9e765612015-05-26 10:28:11 +0200840 pr_err("x86/PAT: %s:%d map pfn expected mapping type %s for [mem %#010Lx-%#010Lx], got %s\n",
841 current->comm, current->pid,
842 cattr_name(want_pcm),
843 (unsigned long long)paddr,
844 (unsigned long long)(paddr + size - 1),
845 cattr_name(pcm));
venkatesh.pallipadi@intel.comcdecff62009-01-09 16:13:12 -0800846 return -EINVAL;
847 }
848 /*
849 * We allow returning different type than the one requested in
850 * non strict case.
851 */
852 *vma_prot = __pgprot((pgprot_val(*vma_prot) &
853 (~_PAGE_CACHE_MASK)) |
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100854 cachemode2protval(pcm));
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800855 }
856
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100857 if (kernel_map_sync_memtype(paddr, size, pcm) < 0) {
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800858 free_memtype(paddr, paddr + size);
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800859 return -EINVAL;
860 }
861 return 0;
862}
863
864/*
865 * Internal interface to free a range of physical memory.
866 * Frees non RAM regions only.
867 */
868static void free_pfn_range(u64 paddr, unsigned long size)
869{
870 int is_ram;
871
Suresh Siddhabe03d9e2009-02-11 11:20:23 -0800872 is_ram = pat_pagerange_is_ram(paddr, paddr + size);
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800873 if (is_ram == 0)
874 free_memtype(paddr, paddr + size);
875}
876
877/*
Suresh Siddha5180da42012-10-08 16:28:29 -0700878 * track_pfn_copy is called when vma that is covering the pfnmap gets
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800879 * copied through copy_page_range().
880 *
881 * If the vma has a linear pfn mapping for the entire range, we get the prot
882 * from pte and reserve the entire vma range with single reserve_pfn_range call.
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800883 */
Suresh Siddha5180da42012-10-08 16:28:29 -0700884int track_pfn_copy(struct vm_area_struct *vma)
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800885{
H. Peter Anvinc1c15b62008-12-23 10:10:40 -0800886 resource_size_t paddr;
venkatesh.pallipadi@intel.com982d7892008-12-19 13:47:28 -0800887 unsigned long prot;
Pallipadi, Venkatesh4b065042009-04-08 15:37:16 -0700888 unsigned long vma_size = vma->vm_end - vma->vm_start;
venkatesh.pallipadi@intel.comcdecff62009-01-09 16:13:12 -0800889 pgprot_t pgprot;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800890
Konstantin Khlebnikovb3b9c292012-10-08 16:28:34 -0700891 if (vma->vm_flags & VM_PAT) {
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800892 /*
venkatesh.pallipadi@intel.com982d7892008-12-19 13:47:28 -0800893 * reserve the whole chunk covered by vma. We need the
894 * starting address and protection from pte.
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800895 */
Pallipadi, Venkatesh4b065042009-04-08 15:37:16 -0700896 if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800897 WARN_ON_ONCE(1);
venkatesh.pallipadi@intel.com982d7892008-12-19 13:47:28 -0800898 return -EINVAL;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800899 }
venkatesh.pallipadi@intel.comcdecff62009-01-09 16:13:12 -0800900 pgprot = __pgprot(prot);
901 return reserve_pfn_range(paddr, vma_size, &pgprot, 1);
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800902 }
903
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800904 return 0;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800905}
906
907/*
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800908 * prot is passed in as a parameter for the new mapping. If the vma has a
909 * linear pfn mapping for the entire range reserve the entire vma range with
910 * single reserve_pfn_range call.
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800911 */
Suresh Siddha5180da42012-10-08 16:28:29 -0700912int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
Konstantin Khlebnikovb3b9c292012-10-08 16:28:34 -0700913 unsigned long pfn, unsigned long addr, unsigned long size)
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800914{
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700915 resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
Juergen Gross2a374692014-11-03 14:01:55 +0100916 enum page_cache_mode pcm;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800917
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700918 /* reserve the whole chunk starting from paddr */
Konstantin Khlebnikovb3b9c292012-10-08 16:28:34 -0700919 if (addr == vma->vm_start && size == (vma->vm_end - vma->vm_start)) {
920 int ret;
921
922 ret = reserve_pfn_range(paddr, size, prot, 0);
923 if (!ret)
924 vma->vm_flags |= VM_PAT;
925 return ret;
926 }
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800927
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200928 if (!pat_enabled())
Venkatesh Pallipadi10876372009-07-10 09:57:40 -0700929 return 0;
930
Suresh Siddha5180da42012-10-08 16:28:29 -0700931 /*
932 * For anything smaller than the vma size we set prot based on the
933 * lookup.
934 */
Juergen Gross2a374692014-11-03 14:01:55 +0100935 pcm = lookup_memtype(paddr);
Suresh Siddha5180da42012-10-08 16:28:29 -0700936
937 /* Check memtype for the remaining pages */
938 while (size > PAGE_SIZE) {
939 size -= PAGE_SIZE;
940 paddr += PAGE_SIZE;
Juergen Gross2a374692014-11-03 14:01:55 +0100941 if (pcm != lookup_memtype(paddr))
Suresh Siddha5180da42012-10-08 16:28:29 -0700942 return -EINVAL;
943 }
944
945 *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
Juergen Gross2a374692014-11-03 14:01:55 +0100946 cachemode2protval(pcm));
Suresh Siddha5180da42012-10-08 16:28:29 -0700947
948 return 0;
949}
950
951int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
952 unsigned long pfn)
953{
Juergen Gross2a374692014-11-03 14:01:55 +0100954 enum page_cache_mode pcm;
Suresh Siddha5180da42012-10-08 16:28:29 -0700955
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +0200956 if (!pat_enabled())
Suresh Siddha5180da42012-10-08 16:28:29 -0700957 return 0;
958
959 /* Set prot based on lookup */
Juergen Gross2a374692014-11-03 14:01:55 +0100960 pcm = lookup_memtype((resource_size_t)pfn << PAGE_SHIFT);
Venkatesh Pallipadi10876372009-07-10 09:57:40 -0700961 *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
Juergen Gross2a374692014-11-03 14:01:55 +0100962 cachemode2protval(pcm));
Venkatesh Pallipadi10876372009-07-10 09:57:40 -0700963
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800964 return 0;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800965}
966
967/*
Suresh Siddha5180da42012-10-08 16:28:29 -0700968 * untrack_pfn is called while unmapping a pfnmap for a region.
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800969 * untrack can be called for a specific region indicated by pfn and size or
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700970 * can be for the entire vma (in which case pfn, size are zero).
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800971 */
Suresh Siddha5180da42012-10-08 16:28:29 -0700972void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
973 unsigned long size)
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800974{
H. Peter Anvinc1c15b62008-12-23 10:10:40 -0800975 resource_size_t paddr;
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700976 unsigned long prot;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800977
Konstantin Khlebnikovb3b9c292012-10-08 16:28:34 -0700978 if (!(vma->vm_flags & VM_PAT))
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800979 return;
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700980
981 /* free the chunk starting from pfn or the whole chunk */
982 paddr = (resource_size_t)pfn << PAGE_SHIFT;
983 if (!paddr && !size) {
984 if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
985 WARN_ON_ONCE(1);
986 return;
987 }
988
989 size = vma->vm_end - vma->vm_start;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800990 }
Suresh Siddhab1a86e12012-10-08 16:28:23 -0700991 free_pfn_range(paddr, size);
Konstantin Khlebnikovb3b9c292012-10-08 16:28:34 -0700992 vma->vm_flags &= ~VM_PAT;
venkatesh.pallipadi@intel.com58993292008-12-18 11:41:30 -0800993}
994
venkatesh.pallipadi@intel.com2520bd32008-12-18 11:41:32 -0800995pgprot_t pgprot_writecombine(pgprot_t prot)
996{
Borislav Petkov7202fdb2015-06-04 18:55:11 +0200997 return __pgprot(pgprot_val(prot) |
Juergen Grosse00c8cc2014-11-03 14:01:59 +0100998 cachemode2protval(_PAGE_CACHE_MODE_WC));
venkatesh.pallipadi@intel.com2520bd32008-12-18 11:41:32 -0800999}
Ingo Molnar92b9af92009-02-28 14:09:27 +01001000EXPORT_SYMBOL_GPL(pgprot_writecombine);
venkatesh.pallipadi@intel.com2520bd32008-12-18 11:41:32 -08001001
Toshi Kanid1b4bfb2015-06-04 18:55:18 +02001002pgprot_t pgprot_writethrough(pgprot_t prot)
1003{
1004 return __pgprot(pgprot_val(prot) |
1005 cachemode2protval(_PAGE_CACHE_MODE_WT));
1006}
1007EXPORT_SYMBOL_GPL(pgprot_writethrough);
1008
Andreas Herrmann012f09e2008-08-06 16:23:08 +02001009#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001010
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001011static struct memtype *memtype_get_idx(loff_t pos)
1012{
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -08001013 struct memtype *print_entry;
1014 int ret;
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001015
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -08001016 print_entry = kzalloc(sizeof(struct memtype), GFP_KERNEL);
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001017 if (!print_entry)
1018 return NULL;
1019
1020 spin_lock(&memtype_lock);
Pallipadi, Venkatesh9e41a492010-02-10 15:26:07 -08001021 ret = rbt_memtype_copy_nth_element(print_entry, pos);
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001022 spin_unlock(&memtype_lock);
Ingo Molnarad2cde12008-09-30 13:20:45 +02001023
venkatesh.pallipadi@intel.combe5a0c12010-02-10 11:57:06 -08001024 if (!ret) {
1025 return print_entry;
1026 } else {
1027 kfree(print_entry);
1028 return NULL;
1029 }
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001030}
1031
1032static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
1033{
1034 if (*pos == 0) {
1035 ++*pos;
Rasmus Villemoes37367082014-11-28 22:03:41 +01001036 seq_puts(seq, "PAT memtype list:\n");
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001037 }
1038
1039 return memtype_get_idx(*pos);
1040}
1041
1042static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1043{
1044 ++*pos;
1045 return memtype_get_idx(*pos);
1046}
1047
1048static void memtype_seq_stop(struct seq_file *seq, void *v)
1049{
1050}
1051
1052static int memtype_seq_show(struct seq_file *seq, void *v)
1053{
1054 struct memtype *print_entry = (struct memtype *)v;
1055
1056 seq_printf(seq, "%s @ 0x%Lx-0x%Lx\n", cattr_name(print_entry->type),
1057 print_entry->start, print_entry->end);
1058 kfree(print_entry);
Ingo Molnarad2cde12008-09-30 13:20:45 +02001059
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001060 return 0;
1061}
1062
Tobias Klauserd535e432009-09-04 15:53:09 +02001063static const struct seq_operations memtype_seq_ops = {
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001064 .start = memtype_seq_start,
1065 .next = memtype_seq_next,
1066 .stop = memtype_seq_stop,
1067 .show = memtype_seq_show,
1068};
1069
1070static int memtype_seq_open(struct inode *inode, struct file *file)
1071{
1072 return seq_open(file, &memtype_seq_ops);
1073}
1074
1075static const struct file_operations memtype_fops = {
1076 .open = memtype_seq_open,
1077 .read = seq_read,
1078 .llseek = seq_lseek,
1079 .release = seq_release,
1080};
1081
1082static int __init pat_memtype_list_init(void)
1083{
Luis R. Rodriguezcb32edf2015-05-26 10:28:15 +02001084 if (pat_enabled()) {
Xiaotian Fengdd4377b2009-11-26 19:53:48 +08001085 debugfs_create_file("pat_memtype_list", S_IRUSR,
1086 arch_debugfs_dir, NULL, &memtype_fops);
1087 }
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -07001088 return 0;
1089}
1090
1091late_initcall(pat_memtype_list_init);
1092
Andreas Herrmann012f09e2008-08-06 16:23:08 +02001093#endif /* CONFIG_DEBUG_FS && CONFIG_X86_PAT */