rev |
line source |
nuclear@17
|
1 #include <stdio.h>
|
nuclear@17
|
2 #include <string.h>
|
nuclear@17
|
3 #include <inttypes.h>
|
nuclear@17
|
4 #include "vm.h"
|
nuclear@17
|
5 #include <stdio.h>
|
nuclear@17
|
6 #include "intr.h"
|
nuclear@17
|
7 #include "mem.h"
|
nuclear@17
|
8 #include "panic.h"
|
nuclear@17
|
9
|
nuclear@17
|
10
|
nuclear@17
|
11 #define KMEM_START 0xc0000000
|
nuclear@17
|
12 #define IDMAP_START 0xa0000
|
nuclear@17
|
13
|
nuclear@24
|
14 #define PGDIR_ADDR 0xfffff000
|
nuclear@24
|
15 #define PGTBL_BASE (0xffffffff - 4096 * 1024 + 1)
|
nuclear@24
|
16 #define PGTBL(x) ((uint32_t*)(PGTBL_BASE + PGSIZE * (x)))
|
nuclear@24
|
17
|
nuclear@17
|
18 #define ATTR_PGDIR_MASK 0x3f
|
nuclear@17
|
19 #define ATTR_PGTBL_MASK 0x1ff
|
nuclear@17
|
20 #define ADDR_PGENT_MASK 0xfffff000
|
nuclear@17
|
21
|
nuclear@17
|
22 #define PAGEFAULT 14
|
nuclear@17
|
23
|
nuclear@22
|
24
|
nuclear@22
|
25 struct page_range {
|
nuclear@22
|
26 int start, end;
|
nuclear@22
|
27 struct page_range *next;
|
nuclear@22
|
28 };
|
nuclear@22
|
29
|
nuclear@22
|
30 /* defined in vm-asm.S */
|
nuclear@22
|
31 void enable_paging(void);
|
nuclear@23
|
32 void disable_paging(void);
|
nuclear@23
|
33 int get_paging_status(void);
|
nuclear@22
|
34 void set_pgdir_addr(uint32_t addr);
|
nuclear@23
|
35 void flush_tlb(void);
|
nuclear@23
|
36 void flush_tlb_addr(uint32_t addr);
|
nuclear@23
|
37 #define flush_tlb_page(p) flush_tlb_addr(PAGE_TO_ADDR(p))
|
nuclear@22
|
38 uint32_t get_fault_addr(void);
|
nuclear@22
|
39
|
nuclear@23
|
40 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high);
|
nuclear@22
|
41 static void pgfault(int inum, uint32_t err);
|
nuclear@22
|
42 static struct page_range *alloc_node(void);
|
nuclear@22
|
43 static void free_node(struct page_range *node);
|
nuclear@22
|
44
|
nuclear@22
|
45 /* page directory */
|
nuclear@22
|
46 static uint32_t *pgdir;
|
nuclear@22
|
47
|
nuclear@22
|
48 /* 2 lists of free ranges, for kernel memory and user memory */
|
nuclear@22
|
49 static struct page_range *pglist[2];
|
nuclear@22
|
50 /* list of free page_range structures to be used in the lists */
|
nuclear@22
|
51 static struct page_range *node_pool;
|
nuclear@23
|
52 /* the first page range for the whole kernel address space, to get things started */
|
nuclear@23
|
53 static struct page_range first_node;
|
nuclear@22
|
54
|
nuclear@22
|
55
|
nuclear@17
|
56 void init_vm(struct mboot_info *mb)
|
nuclear@17
|
57 {
|
nuclear@19
|
58 uint32_t idmap_end;
|
nuclear@19
|
59
|
nuclear@23
|
60 /* initialize the physical memory map and allocator */
|
nuclear@17
|
61 init_mem(mb);
|
nuclear@17
|
62
|
nuclear@23
|
63 /* setup the page tables */
|
nuclear@18
|
64 pgdir = (uint32_t*)alloc_phys_page();
|
nuclear@23
|
65 memset(pgdir, 0, PGSIZE);
|
nuclear@24
|
66 set_pgdir_addr((uint32_t)pgdir);
|
nuclear@17
|
67
|
nuclear@17
|
68 /* map the video memory and kernel code 1-1 */
|
nuclear@19
|
69 get_kernel_mem_range(0, &idmap_end);
|
nuclear@19
|
70 map_mem_range(IDMAP_START, idmap_end - IDMAP_START, IDMAP_START, 0);
|
nuclear@17
|
71
|
nuclear@24
|
72 /* make the last page directory entry point to the page directory */
|
nuclear@24
|
73 pgdir[1023] = ((uint32_t)pgdir & ADDR_PGENT_MASK) | PG_PRESENT;
|
nuclear@24
|
74 pgdir = (uint32_t*)PGDIR_ADDR;
|
nuclear@24
|
75
|
nuclear@23
|
76 /* set the page fault handler */
|
nuclear@17
|
77 interrupt(PAGEFAULT, pgfault);
|
nuclear@17
|
78
|
nuclear@23
|
79 /* we can enable paging now */
|
nuclear@17
|
80 enable_paging();
|
nuclear@23
|
81
|
nuclear@23
|
82 /* initialize the virtual page allocator */
|
nuclear@23
|
83 node_pool = 0;
|
nuclear@23
|
84
|
nuclear@23
|
85 first_node.start = ADDR_TO_PAGE(KMEM_START);
|
nuclear@23
|
86 first_node.end = PAGE_COUNT;
|
nuclear@23
|
87 first_node.next = 0;
|
nuclear@23
|
88 pglist[MEM_KERNEL] = &first_node;
|
nuclear@23
|
89
|
nuclear@23
|
90 pglist[MEM_USER] = alloc_node();
|
nuclear@23
|
91 pglist[MEM_USER]->start = 0;
|
nuclear@23
|
92 pglist[MEM_USER]->end = ADDR_TO_PAGE(KMEM_START);
|
nuclear@23
|
93 pglist[MEM_USER]->next = 0;
|
nuclear@17
|
94 }
|
nuclear@17
|
95
|
nuclear@23
|
96 /* if ppage == -1 we allocate a physical page by calling alloc_phys_page */
|
nuclear@23
|
97 int map_page(int vpage, int ppage, unsigned int attr)
|
nuclear@17
|
98 {
|
nuclear@17
|
99 uint32_t *pgtbl;
|
nuclear@23
|
100 int diridx, pgidx, pgon;
|
nuclear@23
|
101
|
nuclear@23
|
102 pgon = get_paging_status();
|
nuclear@23
|
103
|
nuclear@23
|
104 if(ppage < 0) {
|
nuclear@23
|
105 uint32_t addr = alloc_phys_page();
|
nuclear@23
|
106 if(!addr) {
|
nuclear@23
|
107 return -1;
|
nuclear@23
|
108 }
|
nuclear@23
|
109 ppage = ADDR_TO_PAGE(addr);
|
nuclear@23
|
110 }
|
nuclear@23
|
111
|
nuclear@23
|
112 diridx = PAGE_TO_PGTBL(vpage);
|
nuclear@23
|
113 pgidx = PAGE_TO_PGTBL_PG(vpage);
|
nuclear@17
|
114
|
nuclear@17
|
115 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@17
|
116 uint32_t addr = alloc_phys_page();
|
nuclear@24
|
117 pgdir[diridx] = addr | (attr & ATTR_PGDIR_MASK) | PG_PRESENT;
|
nuclear@24
|
118
|
nuclear@24
|
119 pgtbl = pgon ? PGTBL(diridx) : (uint32_t*)addr;
|
nuclear@18
|
120 memset(pgtbl, 0, PGSIZE);
|
nuclear@17
|
121 } else {
|
nuclear@24
|
122 if(pgon) {
|
nuclear@24
|
123 pgtbl = PGTBL(diridx);
|
nuclear@24
|
124 } else {
|
nuclear@24
|
125 pgtbl = (uint32_t*)(pgdir[diridx] & ADDR_PGENT_MASK);
|
nuclear@24
|
126 }
|
nuclear@17
|
127 }
|
nuclear@17
|
128
|
nuclear@17
|
129 pgtbl[pgidx] = PAGE_TO_ADDR(ppage) | (attr & ATTR_PGTBL_MASK) | PG_PRESENT;
|
nuclear@23
|
130 flush_tlb_page(vpage);
|
nuclear@23
|
131
|
nuclear@23
|
132 return 0;
|
nuclear@17
|
133 }
|
nuclear@17
|
134
|
nuclear@17
|
135 void unmap_page(int vpage)
|
nuclear@17
|
136 {
|
nuclear@17
|
137 uint32_t *pgtbl;
|
nuclear@17
|
138 int diridx = PAGE_TO_PGTBL(vpage);
|
nuclear@17
|
139 int pgidx = PAGE_TO_PGTBL_PG(vpage);
|
nuclear@17
|
140
|
nuclear@17
|
141 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@17
|
142 goto err;
|
nuclear@17
|
143 }
|
nuclear@17
|
144 pgtbl = (uint32_t*)(pgdir[diridx] & ADDR_PGENT_MASK);
|
nuclear@17
|
145
|
nuclear@17
|
146 if(!(pgtbl[pgidx] & PG_PRESENT)) {
|
nuclear@17
|
147 goto err;
|
nuclear@17
|
148 }
|
nuclear@17
|
149 pgtbl[pgidx] = 0;
|
nuclear@23
|
150 flush_tlb_page(vpage);
|
nuclear@17
|
151
|
nuclear@17
|
152 return;
|
nuclear@17
|
153 err:
|
nuclear@17
|
154 printf("unmap_page(%d): page already not mapped\n", vpage);
|
nuclear@17
|
155 }
|
nuclear@17
|
156
|
nuclear@22
|
157 /* if ppg_start is -1, we allocate physical pages to map with alloc_phys_page() */
|
nuclear@23
|
158 int map_page_range(int vpg_start, int pgcount, int ppg_start, unsigned int attr)
|
nuclear@17
|
159 {
|
nuclear@23
|
160 int i, phys_pg;
|
nuclear@23
|
161 uint32_t paddr;
|
nuclear@17
|
162
|
nuclear@17
|
163 for(i=0; i<pgcount; i++) {
|
nuclear@23
|
164 if(ppg_start < 0) {
|
nuclear@23
|
165 if(!(paddr = alloc_phys_page())) {
|
nuclear@23
|
166 return -1;
|
nuclear@23
|
167 }
|
nuclear@23
|
168 phys_pg = ADDR_TO_PAGE(paddr);
|
nuclear@23
|
169 } else {
|
nuclear@23
|
170 phys_pg = ppg_start + i;
|
nuclear@23
|
171 }
|
nuclear@22
|
172
|
nuclear@23
|
173 map_page(vpg_start + i, phys_pg, attr);
|
nuclear@17
|
174 }
|
nuclear@23
|
175 return 0;
|
nuclear@17
|
176 }
|
nuclear@17
|
177
|
nuclear@23
|
178 /* if paddr is 0, we allocate physical pages with alloc_phys_page() */
|
nuclear@23
|
179 int map_mem_range(uint32_t vaddr, size_t sz, uint32_t paddr, unsigned int attr)
|
nuclear@17
|
180 {
|
nuclear@17
|
181 int vpg_start, ppg_start, num_pages;
|
nuclear@17
|
182
|
nuclear@23
|
183 if(!sz) return -1;
|
nuclear@17
|
184
|
nuclear@17
|
185 if(ADDR_TO_PGOFFS(paddr)) {
|
nuclear@17
|
186 panic("map_mem_range called with unaligned physical address: %x\n", paddr);
|
nuclear@17
|
187 }
|
nuclear@17
|
188
|
nuclear@17
|
189 vpg_start = ADDR_TO_PAGE(vaddr);
|
nuclear@23
|
190 ppg_start = paddr > 0 ? ADDR_TO_PAGE(paddr) : -1;
|
nuclear@17
|
191 num_pages = ADDR_TO_PAGE(sz) + 1;
|
nuclear@17
|
192
|
nuclear@23
|
193 return map_page_range(vpg_start, num_pages, ppg_start, attr);
|
nuclear@17
|
194 }
|
nuclear@17
|
195
|
nuclear@18
|
196 uint32_t virt_to_phys(uint32_t vaddr)
|
nuclear@18
|
197 {
|
nuclear@18
|
198 uint32_t pgaddr, *pgtbl;
|
nuclear@18
|
199 int diridx = ADDR_TO_PGTBL(vaddr);
|
nuclear@18
|
200 int pgidx = ADDR_TO_PGTBL_PG(vaddr);
|
nuclear@18
|
201
|
nuclear@18
|
202 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@18
|
203 panic("virt_to_phys(%x): page table %d not present\n", vaddr, diridx);
|
nuclear@18
|
204 }
|
nuclear@18
|
205 pgtbl = (uint32_t*)(pgdir[diridx] & PGENT_ADDR_MASK);
|
nuclear@18
|
206
|
nuclear@18
|
207 if(!(pgtbl[pgidx] & PG_PRESENT)) {
|
nuclear@18
|
208 panic("virt_to_phys(%x): page %d not present\n", vaddr, ADDR_TO_PAGE(vaddr));
|
nuclear@18
|
209 }
|
nuclear@18
|
210 pgaddr = pgtbl[pgidx] & PGENT_ADDR_MASK;
|
nuclear@18
|
211
|
nuclear@18
|
212 return pgaddr | ADDR_TO_PGOFFS(vaddr);
|
nuclear@18
|
213 }
|
nuclear@18
|
214
|
nuclear@22
|
215 /* allocate a contiguous block of virtual memory pages along with
|
nuclear@22
|
216 * backing physical memory for them, and update the page table.
|
nuclear@22
|
217 */
|
nuclear@22
|
218 int pgalloc(int num, int area)
|
nuclear@22
|
219 {
|
nuclear@22
|
220 int ret = -1;
|
nuclear@22
|
221 struct page_range *node, *prev, dummy;
|
nuclear@22
|
222
|
nuclear@22
|
223 dummy.next = pglist[area];
|
nuclear@22
|
224 node = pglist[area];
|
nuclear@22
|
225 prev = &dummy;
|
nuclear@22
|
226
|
nuclear@22
|
227 while(node) {
|
nuclear@22
|
228 if(node->end - node->start >= num) {
|
nuclear@22
|
229 ret = node->start;
|
nuclear@22
|
230 node->start += num;
|
nuclear@22
|
231
|
nuclear@22
|
232 if(node->start == node->end) {
|
nuclear@22
|
233 prev->next = node->next;
|
nuclear@22
|
234 node->next = 0;
|
nuclear@22
|
235
|
nuclear@22
|
236 if(node == pglist[area]) {
|
nuclear@22
|
237 pglist[area] = 0;
|
nuclear@22
|
238 }
|
nuclear@22
|
239 free_node(node);
|
nuclear@22
|
240 }
|
nuclear@22
|
241 break;
|
nuclear@22
|
242 }
|
nuclear@22
|
243
|
nuclear@22
|
244 prev = node;
|
nuclear@22
|
245 node = node->next;
|
nuclear@22
|
246 }
|
nuclear@22
|
247
|
nuclear@22
|
248 if(ret >= 0) {
|
nuclear@23
|
249 /* allocate physical storage and map */
|
nuclear@23
|
250 if(map_page_range(ret, num, -1, 0) == -1) {
|
nuclear@23
|
251 ret = -1;
|
nuclear@23
|
252 }
|
nuclear@22
|
253 }
|
nuclear@22
|
254
|
nuclear@22
|
255 return ret;
|
nuclear@22
|
256 }
|
nuclear@22
|
257
|
nuclear@22
|
258 void pgfree(int start, int num)
|
nuclear@22
|
259 {
|
nuclear@23
|
260 int area, end;
|
nuclear@23
|
261 struct page_range *node, *new, *prev, *next;
|
nuclear@23
|
262
|
nuclear@23
|
263 if(!(new = alloc_node())) {
|
nuclear@23
|
264 panic("pgfree: can't allocate new page_range node to add the freed pages\n");
|
nuclear@23
|
265 }
|
nuclear@23
|
266 new->start = start;
|
nuclear@23
|
267 end = new->end = start + num;
|
nuclear@23
|
268
|
nuclear@23
|
269 area = PAGE_TO_ADDR(start) >= KMEM_START ? MEM_KERNEL : MEM_USER;
|
nuclear@23
|
270
|
nuclear@23
|
271 if(!pglist[area] || pglist[area]->start > start) {
|
nuclear@23
|
272 next = new->next = pglist[area];
|
nuclear@23
|
273 pglist[area] = new;
|
nuclear@23
|
274 prev = 0;
|
nuclear@23
|
275
|
nuclear@23
|
276 } else {
|
nuclear@23
|
277
|
nuclear@23
|
278 prev = 0;
|
nuclear@23
|
279 node = pglist[area];
|
nuclear@23
|
280 next = node ? node->next : 0;
|
nuclear@23
|
281
|
nuclear@23
|
282 while(node) {
|
nuclear@23
|
283 if(!next || next->start > start) {
|
nuclear@23
|
284 /* place here, after node */
|
nuclear@23
|
285 new->next = next;
|
nuclear@23
|
286 node->next = new;
|
nuclear@23
|
287 prev = node; /* needed by coalesce after the loop */
|
nuclear@23
|
288 break;
|
nuclear@23
|
289 }
|
nuclear@23
|
290
|
nuclear@23
|
291 prev = node;
|
nuclear@23
|
292 node = next;
|
nuclear@23
|
293 next = node ? node->next : 0;
|
nuclear@23
|
294 }
|
nuclear@23
|
295 }
|
nuclear@23
|
296
|
nuclear@23
|
297 coalesce(prev, new, next);
|
nuclear@23
|
298 }
|
nuclear@23
|
299
|
nuclear@23
|
300 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high)
|
nuclear@23
|
301 {
|
nuclear@23
|
302 if(high) {
|
nuclear@23
|
303 if(mid->end == high->start) {
|
nuclear@23
|
304 mid->end = high->end;
|
nuclear@23
|
305 mid->next = high->next;
|
nuclear@23
|
306 free_node(high);
|
nuclear@23
|
307 }
|
nuclear@23
|
308 }
|
nuclear@23
|
309
|
nuclear@23
|
310 if(low) {
|
nuclear@23
|
311 if(low->end == mid->start) {
|
nuclear@23
|
312 low->end += mid->end;
|
nuclear@23
|
313 low->next = mid->next;
|
nuclear@23
|
314 free_node(mid);
|
nuclear@23
|
315 }
|
nuclear@23
|
316 }
|
nuclear@22
|
317 }
|
nuclear@22
|
318
|
nuclear@17
|
319 static void pgfault(int inum, uint32_t err)
|
nuclear@17
|
320 {
|
nuclear@17
|
321 printf("~~~~ PAGE FAULT ~~~~\n");
|
nuclear@17
|
322
|
nuclear@17
|
323 printf("fault address: %x\n", get_fault_addr());
|
nuclear@17
|
324
|
nuclear@17
|
325 if(err & PG_PRESENT) {
|
nuclear@17
|
326 if(err & 8) {
|
nuclear@17
|
327 printf("reserved bit set in some paging structure\n");
|
nuclear@17
|
328 } else {
|
nuclear@17
|
329 printf("%s protection violation ", (err & PG_WRITABLE) ? "write" : "read");
|
nuclear@17
|
330 printf("in %s mode\n", err & PG_USER ? "user" : "kernel");
|
nuclear@17
|
331 }
|
nuclear@17
|
332 } else {
|
nuclear@17
|
333 printf("page not present\n");
|
nuclear@17
|
334 }
|
nuclear@19
|
335
|
nuclear@19
|
336 panic("unhandled page fault\n");
|
nuclear@17
|
337 }
|
nuclear@22
|
338
|
nuclear@22
|
339 /* --- page range list node management --- */
|
nuclear@23
|
340 #define NODES_IN_PAGE (PGSIZE / sizeof(struct page_range))
|
nuclear@23
|
341
|
nuclear@22
|
342 static struct page_range *alloc_node(void)
|
nuclear@22
|
343 {
|
nuclear@22
|
344 struct page_range *node;
|
nuclear@23
|
345 int pg, i;
|
nuclear@22
|
346
|
nuclear@22
|
347 if(node_pool) {
|
nuclear@22
|
348 node = node_pool;
|
nuclear@22
|
349 node_pool = node_pool->next;
|
nuclear@23
|
350 printf("alloc_node -> %x\n", (unsigned int)node);
|
nuclear@22
|
351 return node;
|
nuclear@22
|
352 }
|
nuclear@22
|
353
|
nuclear@23
|
354 /* no node structures in the pool, we need to allocate a new page,
|
nuclear@23
|
355 * split it up into node structures, add them in the pool, and
|
nuclear@23
|
356 * allocate one of them.
|
nuclear@22
|
357 */
|
nuclear@23
|
358 if(!(pg = pgalloc(1, MEM_KERNEL))) {
|
nuclear@22
|
359 panic("ran out of physical memory while allocating VM range structures\n");
|
nuclear@22
|
360 }
|
nuclear@23
|
361 node_pool = (struct page_range*)PAGE_TO_ADDR(pg);
|
nuclear@22
|
362
|
nuclear@23
|
363 /* link them up, skip the first as we'll just allocate it anyway */
|
nuclear@23
|
364 for(i=2; i<NODES_IN_PAGE; i++) {
|
nuclear@23
|
365 node_pool[i - 1].next = node_pool + i;
|
nuclear@23
|
366 }
|
nuclear@23
|
367 node_pool[NODES_IN_PAGE - 1].next = 0;
|
nuclear@23
|
368
|
nuclear@23
|
369 /* grab the first and return it */
|
nuclear@23
|
370 node = node_pool++;
|
nuclear@23
|
371 printf("alloc_node -> %x\n", (unsigned int)node);
|
nuclear@23
|
372 return node;
|
nuclear@22
|
373 }
|
nuclear@22
|
374
|
nuclear@22
|
375 static void free_node(struct page_range *node)
|
nuclear@22
|
376 {
|
nuclear@22
|
377 node->next = node_pool;
|
nuclear@22
|
378 node_pool = node;
|
nuclear@23
|
379 printf("free_node\n");
|
nuclear@22
|
380 }
|
nuclear@23
|
381
|
nuclear@23
|
382
|
nuclear@23
|
383 void dbg_print_vm(int area)
|
nuclear@23
|
384 {
|
nuclear@23
|
385 struct page_range *node = pglist[area];
|
nuclear@23
|
386 int last = area == MEM_USER ? 0 : ADDR_TO_PAGE(KMEM_START);
|
nuclear@23
|
387
|
nuclear@23
|
388 printf("%s vm space\n", area == MEM_USER ? "user" : "kernel");
|
nuclear@23
|
389
|
nuclear@23
|
390 while(node) {
|
nuclear@23
|
391 if(node->start > last) {
|
nuclear@23
|
392 printf(" vm-used: %x -> %x\n", PAGE_TO_ADDR(last), PAGE_TO_ADDR(node->start));
|
nuclear@23
|
393 }
|
nuclear@23
|
394
|
nuclear@23
|
395 printf(" vm-free: %x -> ", PAGE_TO_ADDR(node->start));
|
nuclear@23
|
396 if(node->end >= PAGE_COUNT) {
|
nuclear@23
|
397 printf("END\n");
|
nuclear@23
|
398 } else {
|
nuclear@23
|
399 printf("%x\n", PAGE_TO_ADDR(node->end));
|
nuclear@23
|
400 }
|
nuclear@23
|
401
|
nuclear@23
|
402 last = node->end;
|
nuclear@23
|
403 node = node->next;
|
nuclear@23
|
404 }
|
nuclear@23
|
405 }
|