kern

annotate src/vm.c @ 24:53588744382c

find changesets by author, revision, files, or words in the commit message
switched the vm to use recursive page tables
author John Tsiombikas <nuclear@member.fsf.org>
date Tue, 05 Apr 2011 02:09:02 +0300
parents 5454cee245a3
children 9939a6d7a45a
rev   line source
nuclear@17 1 #include <stdio.h>
nuclear@17 2 #include <string.h>
nuclear@17 3 #include <inttypes.h>
nuclear@17 4 #include "vm.h"
nuclear@17 5 #include <stdio.h>
nuclear@17 6 #include "intr.h"
nuclear@17 7 #include "mem.h"
nuclear@17 8 #include "panic.h"
nuclear@17 9
nuclear@17 10
nuclear@17 11 #define KMEM_START 0xc0000000
nuclear@17 12 #define IDMAP_START 0xa0000
nuclear@17 13
nuclear@24 14 #define PGDIR_ADDR 0xfffff000
nuclear@24 15 #define PGTBL_BASE (0xffffffff - 4096 * 1024 + 1)
nuclear@24 16 #define PGTBL(x) ((uint32_t*)(PGTBL_BASE + PGSIZE * (x)))
nuclear@24 17
nuclear@17 18 #define ATTR_PGDIR_MASK 0x3f
nuclear@17 19 #define ATTR_PGTBL_MASK 0x1ff
nuclear@17 20 #define ADDR_PGENT_MASK 0xfffff000
nuclear@17 21
nuclear@17 22 #define PAGEFAULT 14
nuclear@17 23
nuclear@22 24
nuclear@22 25 struct page_range {
nuclear@22 26 int start, end;
nuclear@22 27 struct page_range *next;
nuclear@22 28 };
nuclear@22 29
nuclear@22 30 /* defined in vm-asm.S */
nuclear@22 31 void enable_paging(void);
nuclear@23 32 void disable_paging(void);
nuclear@23 33 int get_paging_status(void);
nuclear@22 34 void set_pgdir_addr(uint32_t addr);
nuclear@23 35 void flush_tlb(void);
nuclear@23 36 void flush_tlb_addr(uint32_t addr);
nuclear@23 37 #define flush_tlb_page(p) flush_tlb_addr(PAGE_TO_ADDR(p))
nuclear@22 38 uint32_t get_fault_addr(void);
nuclear@22 39
nuclear@23 40 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high);
nuclear@22 41 static void pgfault(int inum, uint32_t err);
nuclear@22 42 static struct page_range *alloc_node(void);
nuclear@22 43 static void free_node(struct page_range *node);
nuclear@22 44
nuclear@22 45 /* page directory */
nuclear@22 46 static uint32_t *pgdir;
nuclear@22 47
nuclear@22 48 /* 2 lists of free ranges, for kernel memory and user memory */
nuclear@22 49 static struct page_range *pglist[2];
nuclear@22 50 /* list of free page_range structures to be used in the lists */
nuclear@22 51 static struct page_range *node_pool;
nuclear@23 52 /* the first page range for the whole kernel address space, to get things started */
nuclear@23 53 static struct page_range first_node;
nuclear@22 54
nuclear@22 55
nuclear@17 56 void init_vm(struct mboot_info *mb)
nuclear@17 57 {
nuclear@19 58 uint32_t idmap_end;
nuclear@19 59
nuclear@23 60 /* initialize the physical memory map and allocator */
nuclear@17 61 init_mem(mb);
nuclear@17 62
nuclear@23 63 /* setup the page tables */
nuclear@18 64 pgdir = (uint32_t*)alloc_phys_page();
nuclear@23 65 memset(pgdir, 0, PGSIZE);
nuclear@24 66 set_pgdir_addr((uint32_t)pgdir);
nuclear@17 67
nuclear@17 68 /* map the video memory and kernel code 1-1 */
nuclear@19 69 get_kernel_mem_range(0, &idmap_end);
nuclear@19 70 map_mem_range(IDMAP_START, idmap_end - IDMAP_START, IDMAP_START, 0);
nuclear@17 71
nuclear@24 72 /* make the last page directory entry point to the page directory */
nuclear@24 73 pgdir[1023] = ((uint32_t)pgdir & ADDR_PGENT_MASK) | PG_PRESENT;
nuclear@24 74 pgdir = (uint32_t*)PGDIR_ADDR;
nuclear@24 75
nuclear@23 76 /* set the page fault handler */
nuclear@17 77 interrupt(PAGEFAULT, pgfault);
nuclear@17 78
nuclear@23 79 /* we can enable paging now */
nuclear@17 80 enable_paging();
nuclear@23 81
nuclear@23 82 /* initialize the virtual page allocator */
nuclear@23 83 node_pool = 0;
nuclear@23 84
nuclear@23 85 first_node.start = ADDR_TO_PAGE(KMEM_START);
nuclear@23 86 first_node.end = PAGE_COUNT;
nuclear@23 87 first_node.next = 0;
nuclear@23 88 pglist[MEM_KERNEL] = &first_node;
nuclear@23 89
nuclear@23 90 pglist[MEM_USER] = alloc_node();
nuclear@23 91 pglist[MEM_USER]->start = 0;
nuclear@23 92 pglist[MEM_USER]->end = ADDR_TO_PAGE(KMEM_START);
nuclear@23 93 pglist[MEM_USER]->next = 0;
nuclear@17 94 }
nuclear@17 95
nuclear@23 96 /* if ppage == -1 we allocate a physical page by calling alloc_phys_page */
nuclear@23 97 int map_page(int vpage, int ppage, unsigned int attr)
nuclear@17 98 {
nuclear@17 99 uint32_t *pgtbl;
nuclear@23 100 int diridx, pgidx, pgon;
nuclear@23 101
nuclear@23 102 pgon = get_paging_status();
nuclear@23 103
nuclear@23 104 if(ppage < 0) {
nuclear@23 105 uint32_t addr = alloc_phys_page();
nuclear@23 106 if(!addr) {
nuclear@23 107 return -1;
nuclear@23 108 }
nuclear@23 109 ppage = ADDR_TO_PAGE(addr);
nuclear@23 110 }
nuclear@23 111
nuclear@23 112 diridx = PAGE_TO_PGTBL(vpage);
nuclear@23 113 pgidx = PAGE_TO_PGTBL_PG(vpage);
nuclear@17 114
nuclear@17 115 if(!(pgdir[diridx] & PG_PRESENT)) {
nuclear@17 116 uint32_t addr = alloc_phys_page();
nuclear@24 117 pgdir[diridx] = addr | (attr & ATTR_PGDIR_MASK) | PG_PRESENT;
nuclear@24 118
nuclear@24 119 pgtbl = pgon ? PGTBL(diridx) : (uint32_t*)addr;
nuclear@18 120 memset(pgtbl, 0, PGSIZE);
nuclear@17 121 } else {
nuclear@24 122 if(pgon) {
nuclear@24 123 pgtbl = PGTBL(diridx);
nuclear@24 124 } else {
nuclear@24 125 pgtbl = (uint32_t*)(pgdir[diridx] & ADDR_PGENT_MASK);
nuclear@24 126 }
nuclear@17 127 }
nuclear@17 128
nuclear@17 129 pgtbl[pgidx] = PAGE_TO_ADDR(ppage) | (attr & ATTR_PGTBL_MASK) | PG_PRESENT;
nuclear@23 130 flush_tlb_page(vpage);
nuclear@23 131
nuclear@23 132 return 0;
nuclear@17 133 }
nuclear@17 134
nuclear@17 135 void unmap_page(int vpage)
nuclear@17 136 {
nuclear@17 137 uint32_t *pgtbl;
nuclear@17 138 int diridx = PAGE_TO_PGTBL(vpage);
nuclear@17 139 int pgidx = PAGE_TO_PGTBL_PG(vpage);
nuclear@17 140
nuclear@17 141 if(!(pgdir[diridx] & PG_PRESENT)) {
nuclear@17 142 goto err;
nuclear@17 143 }
nuclear@17 144 pgtbl = (uint32_t*)(pgdir[diridx] & ADDR_PGENT_MASK);
nuclear@17 145
nuclear@17 146 if(!(pgtbl[pgidx] & PG_PRESENT)) {
nuclear@17 147 goto err;
nuclear@17 148 }
nuclear@17 149 pgtbl[pgidx] = 0;
nuclear@23 150 flush_tlb_page(vpage);
nuclear@17 151
nuclear@17 152 return;
nuclear@17 153 err:
nuclear@17 154 printf("unmap_page(%d): page already not mapped\n", vpage);
nuclear@17 155 }
nuclear@17 156
nuclear@22 157 /* if ppg_start is -1, we allocate physical pages to map with alloc_phys_page() */
nuclear@23 158 int map_page_range(int vpg_start, int pgcount, int ppg_start, unsigned int attr)
nuclear@17 159 {
nuclear@23 160 int i, phys_pg;
nuclear@23 161 uint32_t paddr;
nuclear@17 162
nuclear@17 163 for(i=0; i<pgcount; i++) {
nuclear@23 164 if(ppg_start < 0) {
nuclear@23 165 if(!(paddr = alloc_phys_page())) {
nuclear@23 166 return -1;
nuclear@23 167 }
nuclear@23 168 phys_pg = ADDR_TO_PAGE(paddr);
nuclear@23 169 } else {
nuclear@23 170 phys_pg = ppg_start + i;
nuclear@23 171 }
nuclear@22 172
nuclear@23 173 map_page(vpg_start + i, phys_pg, attr);
nuclear@17 174 }
nuclear@23 175 return 0;
nuclear@17 176 }
nuclear@17 177
nuclear@23 178 /* if paddr is 0, we allocate physical pages with alloc_phys_page() */
nuclear@23 179 int map_mem_range(uint32_t vaddr, size_t sz, uint32_t paddr, unsigned int attr)
nuclear@17 180 {
nuclear@17 181 int vpg_start, ppg_start, num_pages;
nuclear@17 182
nuclear@23 183 if(!sz) return -1;
nuclear@17 184
nuclear@17 185 if(ADDR_TO_PGOFFS(paddr)) {
nuclear@17 186 panic("map_mem_range called with unaligned physical address: %x\n", paddr);
nuclear@17 187 }
nuclear@17 188
nuclear@17 189 vpg_start = ADDR_TO_PAGE(vaddr);
nuclear@23 190 ppg_start = paddr > 0 ? ADDR_TO_PAGE(paddr) : -1;
nuclear@17 191 num_pages = ADDR_TO_PAGE(sz) + 1;
nuclear@17 192
nuclear@23 193 return map_page_range(vpg_start, num_pages, ppg_start, attr);
nuclear@17 194 }
nuclear@17 195
nuclear@18 196 uint32_t virt_to_phys(uint32_t vaddr)
nuclear@18 197 {
nuclear@18 198 uint32_t pgaddr, *pgtbl;
nuclear@18 199 int diridx = ADDR_TO_PGTBL(vaddr);
nuclear@18 200 int pgidx = ADDR_TO_PGTBL_PG(vaddr);
nuclear@18 201
nuclear@18 202 if(!(pgdir[diridx] & PG_PRESENT)) {
nuclear@18 203 panic("virt_to_phys(%x): page table %d not present\n", vaddr, diridx);
nuclear@18 204 }
nuclear@18 205 pgtbl = (uint32_t*)(pgdir[diridx] & PGENT_ADDR_MASK);
nuclear@18 206
nuclear@18 207 if(!(pgtbl[pgidx] & PG_PRESENT)) {
nuclear@18 208 panic("virt_to_phys(%x): page %d not present\n", vaddr, ADDR_TO_PAGE(vaddr));
nuclear@18 209 }
nuclear@18 210 pgaddr = pgtbl[pgidx] & PGENT_ADDR_MASK;
nuclear@18 211
nuclear@18 212 return pgaddr | ADDR_TO_PGOFFS(vaddr);
nuclear@18 213 }
nuclear@18 214
nuclear@22 215 /* allocate a contiguous block of virtual memory pages along with
nuclear@22 216 * backing physical memory for them, and update the page table.
nuclear@22 217 */
nuclear@22 218 int pgalloc(int num, int area)
nuclear@22 219 {
nuclear@22 220 int ret = -1;
nuclear@22 221 struct page_range *node, *prev, dummy;
nuclear@22 222
nuclear@22 223 dummy.next = pglist[area];
nuclear@22 224 node = pglist[area];
nuclear@22 225 prev = &dummy;
nuclear@22 226
nuclear@22 227 while(node) {
nuclear@22 228 if(node->end - node->start >= num) {
nuclear@22 229 ret = node->start;
nuclear@22 230 node->start += num;
nuclear@22 231
nuclear@22 232 if(node->start == node->end) {
nuclear@22 233 prev->next = node->next;
nuclear@22 234 node->next = 0;
nuclear@22 235
nuclear@22 236 if(node == pglist[area]) {
nuclear@22 237 pglist[area] = 0;
nuclear@22 238 }
nuclear@22 239 free_node(node);
nuclear@22 240 }
nuclear@22 241 break;
nuclear@22 242 }
nuclear@22 243
nuclear@22 244 prev = node;
nuclear@22 245 node = node->next;
nuclear@22 246 }
nuclear@22 247
nuclear@22 248 if(ret >= 0) {
nuclear@23 249 /* allocate physical storage and map */
nuclear@23 250 if(map_page_range(ret, num, -1, 0) == -1) {
nuclear@23 251 ret = -1;
nuclear@23 252 }
nuclear@22 253 }
nuclear@22 254
nuclear@22 255 return ret;
nuclear@22 256 }
nuclear@22 257
nuclear@22 258 void pgfree(int start, int num)
nuclear@22 259 {
nuclear@23 260 int area, end;
nuclear@23 261 struct page_range *node, *new, *prev, *next;
nuclear@23 262
nuclear@23 263 if(!(new = alloc_node())) {
nuclear@23 264 panic("pgfree: can't allocate new page_range node to add the freed pages\n");
nuclear@23 265 }
nuclear@23 266 new->start = start;
nuclear@23 267 end = new->end = start + num;
nuclear@23 268
nuclear@23 269 area = PAGE_TO_ADDR(start) >= KMEM_START ? MEM_KERNEL : MEM_USER;
nuclear@23 270
nuclear@23 271 if(!pglist[area] || pglist[area]->start > start) {
nuclear@23 272 next = new->next = pglist[area];
nuclear@23 273 pglist[area] = new;
nuclear@23 274 prev = 0;
nuclear@23 275
nuclear@23 276 } else {
nuclear@23 277
nuclear@23 278 prev = 0;
nuclear@23 279 node = pglist[area];
nuclear@23 280 next = node ? node->next : 0;
nuclear@23 281
nuclear@23 282 while(node) {
nuclear@23 283 if(!next || next->start > start) {
nuclear@23 284 /* place here, after node */
nuclear@23 285 new->next = next;
nuclear@23 286 node->next = new;
nuclear@23 287 prev = node; /* needed by coalesce after the loop */
nuclear@23 288 break;
nuclear@23 289 }
nuclear@23 290
nuclear@23 291 prev = node;
nuclear@23 292 node = next;
nuclear@23 293 next = node ? node->next : 0;
nuclear@23 294 }
nuclear@23 295 }
nuclear@23 296
nuclear@23 297 coalesce(prev, new, next);
nuclear@23 298 }
nuclear@23 299
nuclear@23 300 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high)
nuclear@23 301 {
nuclear@23 302 if(high) {
nuclear@23 303 if(mid->end == high->start) {
nuclear@23 304 mid->end = high->end;
nuclear@23 305 mid->next = high->next;
nuclear@23 306 free_node(high);
nuclear@23 307 }
nuclear@23 308 }
nuclear@23 309
nuclear@23 310 if(low) {
nuclear@23 311 if(low->end == mid->start) {
nuclear@23 312 low->end += mid->end;
nuclear@23 313 low->next = mid->next;
nuclear@23 314 free_node(mid);
nuclear@23 315 }
nuclear@23 316 }
nuclear@22 317 }
nuclear@22 318
nuclear@17 319 static void pgfault(int inum, uint32_t err)
nuclear@17 320 {
nuclear@17 321 printf("~~~~ PAGE FAULT ~~~~\n");
nuclear@17 322
nuclear@17 323 printf("fault address: %x\n", get_fault_addr());
nuclear@17 324
nuclear@17 325 if(err & PG_PRESENT) {
nuclear@17 326 if(err & 8) {
nuclear@17 327 printf("reserved bit set in some paging structure\n");
nuclear@17 328 } else {
nuclear@17 329 printf("%s protection violation ", (err & PG_WRITABLE) ? "write" : "read");
nuclear@17 330 printf("in %s mode\n", err & PG_USER ? "user" : "kernel");
nuclear@17 331 }
nuclear@17 332 } else {
nuclear@17 333 printf("page not present\n");
nuclear@17 334 }
nuclear@19 335
nuclear@19 336 panic("unhandled page fault\n");
nuclear@17 337 }
nuclear@22 338
nuclear@22 339 /* --- page range list node management --- */
nuclear@23 340 #define NODES_IN_PAGE (PGSIZE / sizeof(struct page_range))
nuclear@23 341
nuclear@22 342 static struct page_range *alloc_node(void)
nuclear@22 343 {
nuclear@22 344 struct page_range *node;
nuclear@23 345 int pg, i;
nuclear@22 346
nuclear@22 347 if(node_pool) {
nuclear@22 348 node = node_pool;
nuclear@22 349 node_pool = node_pool->next;
nuclear@23 350 printf("alloc_node -> %x\n", (unsigned int)node);
nuclear@22 351 return node;
nuclear@22 352 }
nuclear@22 353
nuclear@23 354 /* no node structures in the pool, we need to allocate a new page,
nuclear@23 355 * split it up into node structures, add them in the pool, and
nuclear@23 356 * allocate one of them.
nuclear@22 357 */
nuclear@23 358 if(!(pg = pgalloc(1, MEM_KERNEL))) {
nuclear@22 359 panic("ran out of physical memory while allocating VM range structures\n");
nuclear@22 360 }
nuclear@23 361 node_pool = (struct page_range*)PAGE_TO_ADDR(pg);
nuclear@22 362
nuclear@23 363 /* link them up, skip the first as we'll just allocate it anyway */
nuclear@23 364 for(i=2; i<NODES_IN_PAGE; i++) {
nuclear@23 365 node_pool[i - 1].next = node_pool + i;
nuclear@23 366 }
nuclear@23 367 node_pool[NODES_IN_PAGE - 1].next = 0;
nuclear@23 368
nuclear@23 369 /* grab the first and return it */
nuclear@23 370 node = node_pool++;
nuclear@23 371 printf("alloc_node -> %x\n", (unsigned int)node);
nuclear@23 372 return node;
nuclear@22 373 }
nuclear@22 374
nuclear@22 375 static void free_node(struct page_range *node)
nuclear@22 376 {
nuclear@22 377 node->next = node_pool;
nuclear@22 378 node_pool = node;
nuclear@23 379 printf("free_node\n");
nuclear@22 380 }
nuclear@23 381
nuclear@23 382
nuclear@23 383 void dbg_print_vm(int area)
nuclear@23 384 {
nuclear@23 385 struct page_range *node = pglist[area];
nuclear@23 386 int last = area == MEM_USER ? 0 : ADDR_TO_PAGE(KMEM_START);
nuclear@23 387
nuclear@23 388 printf("%s vm space\n", area == MEM_USER ? "user" : "kernel");
nuclear@23 389
nuclear@23 390 while(node) {
nuclear@23 391 if(node->start > last) {
nuclear@23 392 printf(" vm-used: %x -> %x\n", PAGE_TO_ADDR(last), PAGE_TO_ADDR(node->start));
nuclear@23 393 }
nuclear@23 394
nuclear@23 395 printf(" vm-free: %x -> ", PAGE_TO_ADDR(node->start));
nuclear@23 396 if(node->end >= PAGE_COUNT) {
nuclear@23 397 printf("END\n");
nuclear@23 398 } else {
nuclear@23 399 printf("%x\n", PAGE_TO_ADDR(node->end));
nuclear@23 400 }
nuclear@23 401
nuclear@23 402 last = node->end;
nuclear@23 403 node = node->next;
nuclear@23 404 }
nuclear@23 405 }