1/* $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $ */
2
3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28/*
29 * Copyright (c) 2001 Wasabi Systems, Inc.
30 * All rights reserved.
31 *
32 * Written by Frank van der Linden for Wasabi Systems, Inc.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. All advertising materials mentioning features or use of this software
43 * must display the following acknowledgement:
44 * This product includes software developed for the NetBSD Project by
45 * Wasabi Systems, Inc.
46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
47 * or promote products derived from this software without specific prior
48 * written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 */
62
63/*
64 * pmap.h: see pmap.c for the history of this pmap module.
65 */
66
67#ifndef _X86_PMAP_H_
68#define _X86_PMAP_H_
69
70/*
71 * pl*_pi: index in the ptp page for a pde mapping a VA.
72 * (pl*_i below is the index in the virtual array of all pdes per level)
73 */
74#define pl1_pi(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT)
75#define pl2_pi(VA) (((VA_SIGN_POS(VA)) & L2_MASK) >> L2_SHIFT)
76#define pl3_pi(VA) (((VA_SIGN_POS(VA)) & L3_MASK) >> L3_SHIFT)
77#define pl4_pi(VA) (((VA_SIGN_POS(VA)) & L4_MASK) >> L4_SHIFT)
78#define pl_pi(va, lvl) \
79 (((VA_SIGN_POS(va)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1])
80
81/*
82 * pl*_i: generate index into pde/pte arrays in virtual space
83 *
84 * pl_i(va, X) == plX_i(va) <= pl_i_roundup(va, X)
85 */
86#define pl1_i(VA) (((VA_SIGN_POS(VA)) & L1_FRAME) >> L1_SHIFT)
87#define pl2_i(VA) (((VA_SIGN_POS(VA)) & L2_FRAME) >> L2_SHIFT)
88#define pl3_i(VA) (((VA_SIGN_POS(VA)) & L3_FRAME) >> L3_SHIFT)
89#define pl4_i(VA) (((VA_SIGN_POS(VA)) & L4_FRAME) >> L4_SHIFT)
90#define pl_i(va, lvl) \
91 (((VA_SIGN_POS(va)) & ptp_frames[(lvl)-1]) >> ptp_shifts[(lvl)-1])
92
93#define pl_i_roundup(va, lvl) pl_i((va)+ ~ptp_frames[(lvl)-1], (lvl))
94
95/*
96 * PTP macros:
97 * a PTP's index is the PD index of the PDE that points to it
98 * a PTP's offset is the byte-offset in the PTE space that this PTP is at
99 * a PTP's VA is the first VA mapped by that PTP
100 */
101
102#define ptp_va2o(va, lvl) (pl_i(va, (lvl)+1) * PAGE_SIZE)
103
104/* size of a PDP: usually one page, except for PAE */
105#ifdef PAE
106#define PDP_SIZE 4
107#else
108#define PDP_SIZE 1
109#endif
110
111
112#if defined(_KERNEL)
113#include <sys/kcpuset.h>
114#include <x86/pmap_pv.h>
115#include <uvm/pmap/pmap_pvt.h>
116
117#define PATENTRY(n, type) (type << ((n) * 8))
118#define PAT_UC 0x0ULL
119#define PAT_WC 0x1ULL
120#define PAT_WT 0x4ULL
121#define PAT_WP 0x5ULL
122#define PAT_WB 0x6ULL
123#define PAT_UCMINUS 0x7ULL
124
125#define BTSEG_NONE 0
126#define BTSEG_TEXT 1
127#define BTSEG_RODATA 2
128#define BTSEG_DATA 3
129#define BTSPACE_NSEGS 64
130
131struct bootspace {
132 struct {
133 vaddr_t va;
134 paddr_t pa;
135 size_t sz;
136 } head;
137
138 /* Kernel segments. */
139 struct {
140 int type;
141 vaddr_t va;
142 paddr_t pa;
143 size_t sz;
144 } segs[BTSPACE_NSEGS];
145
146 /*
147 * The area used by the early kernel bootstrap. It contains the kernel
148 * symbols, the preloaded modules, the bootstrap tables, and the ISA I/O
149 * mem.
150 */
151 struct {
152 vaddr_t va;
153 paddr_t pa;
154 size_t sz;
155 } boot;
156
157 /* A magic VA usable by the bootstrap code. */
158 vaddr_t spareva;
159
160 /* Virtual address of the page directory. */
161 vaddr_t pdir;
162
163 /* Area dedicated to kernel modules (amd64 only). */
164 vaddr_t smodule;
165 vaddr_t emodule;
166};
167
168#define SLAREA_USER 0
169#define SLAREA_PTE 1
170#define SLAREA_MAIN 2
171#define SLAREA_PCPU 3
172#define SLAREA_DMAP 4
173#define SLAREA_HYPV 5
174#define SLAREA_ASAN 6
175#define SLAREA_KERN 7
176#define SLSPACE_NAREAS 8
177
178struct slotspace {
179 struct {
180 size_t sslot; /* start slot */
181 size_t nslot; /* # of slots */
182 bool active; /* area is active */
183 } area[SLSPACE_NAREAS];
184};
185
186extern struct slotspace slotspace;
187
188#ifndef MAXGDTSIZ
189#define MAXGDTSIZ 65536 /* XXX */
190#endif
191
192struct pcpu_entry {
193 uint8_t gdt[MAXGDTSIZ];
194 uint8_t tss[PAGE_SIZE];
195 uint8_t ist0[PAGE_SIZE];
196 uint8_t ist1[PAGE_SIZE];
197 uint8_t ist2[PAGE_SIZE];
198 uint8_t ist3[PAGE_SIZE];
199 uint8_t rsp0[2 * PAGE_SIZE];
200} __packed;
201
202struct pcpu_area {
203#ifdef SVS
204 uint8_t utls[PAGE_SIZE];
205#endif
206 uint8_t idt[PAGE_SIZE];
207 uint8_t ldt[PAGE_SIZE];
208 struct pcpu_entry ent[MAXCPUS];
209} __packed;
210
211extern struct pcpu_area *pcpuarea;
212
213#define PMAP_PCID_KERN 0
214#define PMAP_PCID_USER 1
215
216/*
217 * pmap data structures: see pmap.c for details of locking.
218 */
219
220/*
221 * we maintain a list of all non-kernel pmaps
222 */
223
224LIST_HEAD(pmap_head, pmap); /* struct pmap_head: head of a pmap list */
225
226/*
227 * linked list of all non-kernel pmaps
228 */
229extern struct pmap_head pmaps;
230extern kmutex_t pmaps_lock; /* protects pmaps */
231
232/*
233 * pool_cache(9) that PDPs are allocated from
234 */
235extern struct pool_cache pmap_pdp_cache;
236
237/*
238 * the pmap structure
239 *
240 * note that the pm_obj contains the lock pointer, the reference count,
241 * page list, and number of PTPs within the pmap.
242 *
243 * pm_lock is the same as the lock for vm object 0. Changes to
244 * the other objects may only be made if that lock has been taken
245 * (the other object locks are only used when uvm_pagealloc is called)
246 */
247
248struct pmap {
249 struct uvm_object pm_obj[PTP_LEVELS-1]; /* objects for lvl >= 1) */
250#define pm_lock pm_obj[0].vmobjlock
251 kmutex_t pm_obj_lock[PTP_LEVELS-1]; /* locks for pm_objs */
252 LIST_ENTRY(pmap) pm_list; /* list (lck by pm_list lock) */
253 pd_entry_t *pm_pdir; /* VA of PD (lck by object lock) */
254 paddr_t pm_pdirpa[PDP_SIZE]; /* PA of PDs (read-only after create) */
255 struct vm_page *pm_ptphint[PTP_LEVELS-1];
256 /* pointer to a PTP in our pmap */
257 struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */
258
259#if !defined(__x86_64__)
260 vaddr_t pm_hiexec; /* highest executable mapping */
261#endif /* !defined(__x86_64__) */
262 int pm_flags; /* see below */
263
264 union descriptor *pm_ldt; /* user-set LDT */
265 size_t pm_ldt_len; /* size of LDT in bytes */
266 int pm_ldt_sel; /* LDT selector */
267 kcpuset_t *pm_cpus; /* mask of CPUs using pmap */
268 kcpuset_t *pm_kernel_cpus; /* mask of CPUs using kernel part
269 of pmap */
270 kcpuset_t *pm_xen_ptp_cpus; /* mask of CPUs which have this pmap's
271 ptp mapped */
272 uint64_t pm_ncsw; /* for assertions */
273 struct vm_page *pm_gc_ptp; /* pages from pmap g/c */
274
275 /* Used by NVMM. */
276 int (*pm_enter)(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int);
277 bool (*pm_extract)(struct pmap *, vaddr_t, paddr_t *);
278 void (*pm_remove)(struct pmap *, vaddr_t, vaddr_t);
279 int (*pm_sync_pv)(struct vm_page *, vaddr_t, paddr_t, int, uint8_t *,
280 pt_entry_t *);
281 void (*pm_pp_remove_ent)(struct pmap *, struct vm_page *, pt_entry_t,
282 vaddr_t);
283 void (*pm_write_protect)(struct pmap *, vaddr_t, vaddr_t, vm_prot_t);
284 void (*pm_unwire)(struct pmap *, vaddr_t);
285
286 void (*pm_tlb_flush)(struct pmap *);
287 void *pm_data;
288};
289
290/* macro to access pm_pdirpa slots */
291#ifdef PAE
292#define pmap_pdirpa(pmap, index) \
293 ((pmap)->pm_pdirpa[l2tol3(index)] + l2tol2(index) * sizeof(pd_entry_t))
294#else
295#define pmap_pdirpa(pmap, index) \
296 ((pmap)->pm_pdirpa[0] + (index) * sizeof(pd_entry_t))
297#endif
298
299/*
300 * MD flags that we use for pmap_enter and pmap_kenter_pa:
301 */
302
303/*
304 * global kernel variables
305 */
306
307/*
308 * PDPpaddr is the physical address of the kernel's PDP.
309 * - i386 non-PAE and amd64: PDPpaddr corresponds directly to the %cr3
310 * value associated to the kernel process, proc0.
311 * - i386 PAE: it still represents the PA of the kernel's PDP (L2). Due to
312 * the L3 PD, it cannot be considered as the equivalent of a %cr3 any more.
313 * - Xen: it corresponds to the PFN of the kernel's PDP.
314 */
315extern u_long PDPpaddr;
316
317extern pd_entry_t pmap_pg_g; /* do we support PG_G? */
318extern pd_entry_t pmap_pg_nx; /* do we support PG_NX? */
319extern int pmap_largepages;
320extern long nkptp[PTP_LEVELS];
321
322/*
323 * macros
324 */
325
326#define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count)
327#define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count)
328
329#define pmap_clear_modify(pg) pmap_clear_attrs(pg, PP_ATTRS_M)
330#define pmap_clear_reference(pg) pmap_clear_attrs(pg, PP_ATTRS_U)
331#define pmap_copy(DP,SP,D,L,S) __USE(L)
332#define pmap_is_modified(pg) pmap_test_attrs(pg, PP_ATTRS_M)
333#define pmap_is_referenced(pg) pmap_test_attrs(pg, PP_ATTRS_U)
334#define pmap_move(DP,SP,D,L,S)
335#define pmap_phys_address(ppn) (x86_ptob(ppn) & ~X86_MMAP_FLAG_MASK)
336#define pmap_mmap_flags(ppn) x86_mmap_flags(ppn)
337#define pmap_valid_entry(E) ((E) & PTE_P) /* is PDE or PTE valid? */
338
339#if defined(__x86_64__) || defined(PAE)
340#define X86_MMAP_FLAG_SHIFT (64 - PGSHIFT)
341#else
342#define X86_MMAP_FLAG_SHIFT (32 - PGSHIFT)
343#endif
344
345#define X86_MMAP_FLAG_MASK 0xf
346#define X86_MMAP_FLAG_PREFETCH 0x1
347
348/*
349 * prototypes
350 */
351
352void pmap_activate(struct lwp *);
353void pmap_bootstrap(vaddr_t);
354bool pmap_clear_attrs(struct vm_page *, unsigned);
355bool pmap_pv_clear_attrs(paddr_t, unsigned);
356void pmap_deactivate(struct lwp *);
357void pmap_page_remove(struct vm_page *);
358void pmap_pv_remove(paddr_t);
359void pmap_remove(struct pmap *, vaddr_t, vaddr_t);
360bool pmap_test_attrs(struct vm_page *, unsigned);
361void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t, vm_prot_t);
362void pmap_load(void);
363paddr_t pmap_init_tmp_pgtbl(paddr_t);
364void pmap_remove_all(struct pmap *);
365void pmap_ldt_cleanup(struct lwp *);
366void pmap_ldt_sync(struct pmap *);
367void pmap_kremove_local(vaddr_t, vsize_t);
368
369#define __HAVE_PMAP_PV_TRACK 1
370void pmap_pv_init(void);
371void pmap_pv_track(paddr_t, psize_t);
372void pmap_pv_untrack(paddr_t, psize_t);
373
374void pmap_map_ptes(struct pmap *, struct pmap **, pd_entry_t **,
375 pd_entry_t * const **);
376void pmap_unmap_ptes(struct pmap *, struct pmap *);
377
378bool pmap_pdes_valid(vaddr_t, pd_entry_t * const *, pd_entry_t *,
379 int *lastlvl);
380
381u_int x86_mmap_flags(paddr_t);
382
383bool pmap_is_curpmap(struct pmap *);
384
385void pmap_ept_transform(struct pmap *);
386
387#ifndef __HAVE_DIRECT_MAP
388void pmap_vpage_cpu_init(struct cpu_info *);
389#endif
390vaddr_t slotspace_rand(int, size_t, size_t);
391
392vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */
393
394typedef enum tlbwhy {
395 TLBSHOOT_APTE,
396 TLBSHOOT_KENTER,
397 TLBSHOOT_KREMOVE,
398 TLBSHOOT_FREE_PTP1,
399 TLBSHOOT_FREE_PTP2,
400 TLBSHOOT_REMOVE_PTE,
401 TLBSHOOT_REMOVE_PTES,
402 TLBSHOOT_SYNC_PV1,
403 TLBSHOOT_SYNC_PV2,
404 TLBSHOOT_WRITE_PROTECT,
405 TLBSHOOT_ENTER,
406 TLBSHOOT_UPDATE,
407 TLBSHOOT_BUS_DMA,
408 TLBSHOOT_BUS_SPACE,
409 TLBSHOOT__MAX,
410} tlbwhy_t;
411
412void pmap_tlb_init(void);
413void pmap_tlb_cpu_init(struct cpu_info *);
414void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t);
415void pmap_tlb_shootnow(void);
416void pmap_tlb_intr(void);
417
418#define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */
419#define PMAP_FORK /* turn on pmap_fork interface */
420
421/*
422 * Do idle page zero'ing uncached to avoid polluting the cache.
423 */
424bool pmap_pageidlezero(paddr_t);
425#define PMAP_PAGEIDLEZERO(pa) pmap_pageidlezero((pa))
426
427/*
428 * inline functions
429 */
430
431/*
432 * pmap_update_pg: flush one page from the TLB (or flush the whole thing
433 * if hardware doesn't support one-page flushing)
434 */
435
436__inline static void __unused
437pmap_update_pg(vaddr_t va)
438{
439 invlpg(va);
440}
441
442/*
443 * pmap_page_protect: change the protection of all recorded mappings
444 * of a managed page
445 *
446 * => this function is a frontend for pmap_page_remove/pmap_clear_attrs
447 * => we only have to worry about making the page more protected.
448 * unprotecting a page is done on-demand at fault time.
449 */
450
451__inline static void __unused
452pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
453{
454 if ((prot & VM_PROT_WRITE) == 0) {
455 if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) {
456 (void)pmap_clear_attrs(pg, PP_ATTRS_W);
457 } else {
458 pmap_page_remove(pg);
459 }
460 }
461}
462
463/*
464 * pmap_pv_protect: change the protection of all recorded mappings
465 * of an unmanaged page
466 */
467
468__inline static void __unused
469pmap_pv_protect(paddr_t pa, vm_prot_t prot)
470{
471 if ((prot & VM_PROT_WRITE) == 0) {
472 if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) {
473 (void)pmap_pv_clear_attrs(pa, PP_ATTRS_W);
474 } else {
475 pmap_pv_remove(pa);
476 }
477 }
478}
479
480/*
481 * pmap_protect: change the protection of pages in a pmap
482 *
483 * => this function is a frontend for pmap_remove/pmap_write_protect
484 * => we only have to worry about making the page more protected.
485 * unprotecting a page is done on-demand at fault time.
486 */
487
488__inline static void __unused
489pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
490{
491 if ((prot & VM_PROT_WRITE) == 0) {
492 if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) {
493 pmap_write_protect(pmap, sva, eva, prot);
494 } else {
495 pmap_remove(pmap, sva, eva);
496 }
497 }
498}
499
500/*
501 * various address inlines
502 *
503 * vtopte: return a pointer to the PTE mapping a VA, works only for
504 * user and PT addresses
505 *
506 * kvtopte: return a pointer to the PTE mapping a kernel VA
507 */
508
509#include <lib/libkern/libkern.h>
510
511static __inline pt_entry_t * __unused
512vtopte(vaddr_t va)
513{
514
515 KASSERT(va < VM_MIN_KERNEL_ADDRESS);
516
517 return (PTE_BASE + pl1_i(va));
518}
519
520static __inline pt_entry_t * __unused
521kvtopte(vaddr_t va)
522{
523 pd_entry_t *pde;
524
525 KASSERT(va >= VM_MIN_KERNEL_ADDRESS);
526
527 pde = L2_BASE + pl2_i(va);
528 if (*pde & PG_PS)
529 return ((pt_entry_t *)pde);
530
531 return (PTE_BASE + pl1_i(va));
532}
533
534paddr_t vtophys(vaddr_t);
535vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t);
536void pmap_cpu_init_late(struct cpu_info *);
537bool sse2_idlezero_page(void *);
538
539#ifdef XENPV
540#include <sys/bitops.h>
541
542#define XPTE_MASK L1_FRAME
543/* Selects the index of a PTE in (A)PTE_BASE */
544#define XPTE_SHIFT (L1_SHIFT - ilog2(sizeof(pt_entry_t)))
545
546/* PTE access inline fuctions */
547
548/*
549 * Get the machine address of the pointed pte
550 * We use hardware MMU to get value so works only for levels 1-3
551 */
552
553static __inline paddr_t
554xpmap_ptetomach(pt_entry_t *pte)
555{
556 pt_entry_t *up_pte;
557 vaddr_t va = (vaddr_t) pte;
558
559 va = ((va & XPTE_MASK) >> XPTE_SHIFT) | (vaddr_t) PTE_BASE;
560 up_pte = (pt_entry_t *) va;
561
562 return (paddr_t) (((*up_pte) & PG_FRAME) + (((vaddr_t) pte) & (~PG_FRAME & ~VA_SIGN_MASK)));
563}
564
565/* Xen helpers to change bits of a pte */
566#define XPMAP_UPDATE_DIRECT 1 /* Update direct map entry flags too */
567
568paddr_t vtomach(vaddr_t);
569#define vtomfn(va) (vtomach(va) >> PAGE_SHIFT)
570#endif /* XENPV */
571
572/* pmap functions with machine addresses */
573void pmap_kenter_ma(vaddr_t, paddr_t, vm_prot_t, u_int);
574int pmap_enter_ma(struct pmap *, vaddr_t, paddr_t, paddr_t,
575 vm_prot_t, u_int, int);
576bool pmap_extract_ma(pmap_t, vaddr_t, paddr_t *);
577void pmap_free_ptps(struct vm_page *);
578
579paddr_t pmap_get_physpage(void);
580
581/*
582 * Hooks for the pool allocator.
583 */
584#define POOL_VTOPHYS(va) vtophys((vaddr_t) (va))
585
586#ifdef __HAVE_PCPU_AREA
587extern struct pcpu_area *pcpuarea;
588#define PDIR_SLOT_PCPU 510
589#define PMAP_PCPU_BASE (VA_SIGN_NEG((PDIR_SLOT_PCPU * NBPD_L4)))
590#endif
591
592#ifdef __HAVE_DIRECT_MAP
593
594extern vaddr_t pmap_direct_base;
595extern vaddr_t pmap_direct_end;
596
597#define PMAP_DIRECT_BASE pmap_direct_base
598#define PMAP_DIRECT_END pmap_direct_end
599
600#define PMAP_DIRECT_MAP(pa) ((vaddr_t)PMAP_DIRECT_BASE + (pa))
601#define PMAP_DIRECT_UNMAP(va) ((paddr_t)(va) - PMAP_DIRECT_BASE)
602
603/*
604 * Alternate mapping hooks for pool pages.
605 */
606#define PMAP_MAP_POOLPAGE(pa) PMAP_DIRECT_MAP((pa))
607#define PMAP_UNMAP_POOLPAGE(va) PMAP_DIRECT_UNMAP((va))
608
609void pagezero(vaddr_t);
610
611#endif /* __HAVE_DIRECT_MAP */
612
613#endif /* _KERNEL */
614
615#endif /* _X86_PMAP_H_ */
616