rev |
line source |
nuclear@17
|
1 #include <stdio.h>
|
nuclear@17
|
2 #include <string.h>
|
nuclear@17
|
3 #include <inttypes.h>
|
nuclear@52
|
4 #include <assert.h>
|
nuclear@52
|
5 #include "config.h"
|
nuclear@17
|
6 #include "vm.h"
|
nuclear@17
|
7 #include "intr.h"
|
nuclear@17
|
8 #include "mem.h"
|
nuclear@17
|
9 #include "panic.h"
|
nuclear@52
|
10 #include "proc.h"
|
nuclear@17
|
11
|
nuclear@17
|
12 #define IDMAP_START 0xa0000
|
nuclear@17
|
13
|
nuclear@24
|
14 #define PGDIR_ADDR 0xfffff000
|
nuclear@24
|
15 #define PGTBL_BASE (0xffffffff - 4096 * 1024 + 1)
|
nuclear@24
|
16 #define PGTBL(x) ((uint32_t*)(PGTBL_BASE + PGSIZE * (x)))
|
nuclear@24
|
17
|
nuclear@17
|
18 #define ATTR_PGDIR_MASK 0x3f
|
nuclear@17
|
19 #define ATTR_PGTBL_MASK 0x1ff
|
nuclear@17
|
20
|
nuclear@17
|
21 #define PAGEFAULT 14
|
nuclear@17
|
22
|
nuclear@22
|
23
|
nuclear@22
|
24 struct page_range {
|
nuclear@22
|
25 int start, end;
|
nuclear@22
|
26 struct page_range *next;
|
nuclear@22
|
27 };
|
nuclear@22
|
28
|
nuclear@22
|
29 /* defined in vm-asm.S */
|
nuclear@22
|
30 void enable_paging(void);
|
nuclear@23
|
31 void disable_paging(void);
|
nuclear@23
|
32 int get_paging_status(void);
|
nuclear@22
|
33 void set_pgdir_addr(uint32_t addr);
|
nuclear@23
|
34 void flush_tlb(void);
|
nuclear@23
|
35 void flush_tlb_addr(uint32_t addr);
|
nuclear@23
|
36 #define flush_tlb_page(p) flush_tlb_addr(PAGE_TO_ADDR(p))
|
nuclear@22
|
37 uint32_t get_fault_addr(void);
|
nuclear@22
|
38
|
nuclear@23
|
39 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high);
|
nuclear@52
|
40 static void pgfault(int inum);
|
nuclear@22
|
41 static struct page_range *alloc_node(void);
|
nuclear@22
|
42 static void free_node(struct page_range *node);
|
nuclear@22
|
43
|
nuclear@22
|
44 /* page directory */
|
nuclear@22
|
45 static uint32_t *pgdir;
|
nuclear@22
|
46
|
nuclear@22
|
47 /* 2 lists of free ranges, for kernel memory and user memory */
|
nuclear@22
|
48 static struct page_range *pglist[2];
|
nuclear@22
|
49 /* list of free page_range structures to be used in the lists */
|
nuclear@22
|
50 static struct page_range *node_pool;
|
nuclear@23
|
51 /* the first page range for the whole kernel address space, to get things started */
|
nuclear@23
|
52 static struct page_range first_node;
|
nuclear@22
|
53
|
nuclear@22
|
54
|
nuclear@26
|
55 void init_vm(void)
|
nuclear@17
|
56 {
|
nuclear@19
|
57 uint32_t idmap_end;
|
nuclear@47
|
58 int i, kmem_start_pg, pgtbl_base_pg;
|
nuclear@19
|
59
|
nuclear@23
|
60 /* setup the page tables */
|
nuclear@18
|
61 pgdir = (uint32_t*)alloc_phys_page();
|
nuclear@23
|
62 memset(pgdir, 0, PGSIZE);
|
nuclear@24
|
63 set_pgdir_addr((uint32_t)pgdir);
|
nuclear@17
|
64
|
nuclear@17
|
65 /* map the video memory and kernel code 1-1 */
|
nuclear@19
|
66 get_kernel_mem_range(0, &idmap_end);
|
nuclear@19
|
67 map_mem_range(IDMAP_START, idmap_end - IDMAP_START, IDMAP_START, 0);
|
nuclear@17
|
68
|
nuclear@24
|
69 /* make the last page directory entry point to the page directory */
|
nuclear@68
|
70 pgdir[1023] = ((uint32_t)pgdir & PGENT_ADDR_MASK) | PG_PRESENT;
|
nuclear@24
|
71 pgdir = (uint32_t*)PGDIR_ADDR;
|
nuclear@24
|
72
|
nuclear@23
|
73 /* set the page fault handler */
|
nuclear@17
|
74 interrupt(PAGEFAULT, pgfault);
|
nuclear@17
|
75
|
nuclear@23
|
76 /* we can enable paging now */
|
nuclear@17
|
77 enable_paging();
|
nuclear@23
|
78
|
nuclear@23
|
79 /* initialize the virtual page allocator */
|
nuclear@23
|
80 node_pool = 0;
|
nuclear@23
|
81
|
nuclear@47
|
82 kmem_start_pg = ADDR_TO_PAGE(KMEM_START);
|
nuclear@47
|
83 pgtbl_base_pg = ADDR_TO_PAGE(PGTBL_BASE);
|
nuclear@47
|
84
|
nuclear@47
|
85 first_node.start = kmem_start_pg;
|
nuclear@47
|
86 first_node.end = pgtbl_base_pg;
|
nuclear@23
|
87 first_node.next = 0;
|
nuclear@23
|
88 pglist[MEM_KERNEL] = &first_node;
|
nuclear@23
|
89
|
nuclear@23
|
90 pglist[MEM_USER] = alloc_node();
|
nuclear@26
|
91 pglist[MEM_USER]->start = ADDR_TO_PAGE(idmap_end);
|
nuclear@47
|
92 pglist[MEM_USER]->end = kmem_start_pg;
|
nuclear@23
|
93 pglist[MEM_USER]->next = 0;
|
nuclear@47
|
94
|
nuclear@47
|
95 /* temporaroly map something into every 1024th page of the kernel address
|
nuclear@47
|
96 * space to force pre-allocation of all the kernel page-tables
|
nuclear@47
|
97 */
|
nuclear@47
|
98 for(i=kmem_start_pg; i<pgtbl_base_pg; i+=1024) {
|
nuclear@47
|
99 /* if there's already something mapped here, leave it alone */
|
nuclear@47
|
100 if(virt_to_phys_page(i) == -1) {
|
nuclear@47
|
101 map_page(i, 0, 0);
|
nuclear@47
|
102 unmap_page(i);
|
nuclear@47
|
103 }
|
nuclear@47
|
104 }
|
nuclear@17
|
105 }
|
nuclear@17
|
106
|
nuclear@23
|
107 /* if ppage == -1 we allocate a physical page by calling alloc_phys_page */
|
nuclear@23
|
108 int map_page(int vpage, int ppage, unsigned int attr)
|
nuclear@17
|
109 {
|
nuclear@17
|
110 uint32_t *pgtbl;
|
nuclear@25
|
111 int diridx, pgidx, pgon, intr_state;
|
nuclear@25
|
112
|
nuclear@25
|
113 intr_state = get_intr_state();
|
nuclear@25
|
114 disable_intr();
|
nuclear@23
|
115
|
nuclear@23
|
116 pgon = get_paging_status();
|
nuclear@23
|
117
|
nuclear@23
|
118 if(ppage < 0) {
|
nuclear@23
|
119 uint32_t addr = alloc_phys_page();
|
nuclear@23
|
120 if(!addr) {
|
nuclear@25
|
121 set_intr_state(intr_state);
|
nuclear@23
|
122 return -1;
|
nuclear@23
|
123 }
|
nuclear@23
|
124 ppage = ADDR_TO_PAGE(addr);
|
nuclear@23
|
125 }
|
nuclear@23
|
126
|
nuclear@23
|
127 diridx = PAGE_TO_PGTBL(vpage);
|
nuclear@23
|
128 pgidx = PAGE_TO_PGTBL_PG(vpage);
|
nuclear@17
|
129
|
nuclear@17
|
130 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@55
|
131 /* no page table present, we must allocate one */
|
nuclear@17
|
132 uint32_t addr = alloc_phys_page();
|
nuclear@55
|
133
|
nuclear@55
|
134 /* make sure all page directory entries in the below the kernel vm
|
nuclear@55
|
135 * split have the user and writable bits set, otherwise further user
|
nuclear@55
|
136 * mappings on the same 4mb block will be unusable in user space.
|
nuclear@55
|
137 */
|
nuclear@55
|
138 unsigned int pgdir_attr = attr;
|
nuclear@55
|
139 if(vpage < ADDR_TO_PAGE(KMEM_START)) {
|
nuclear@55
|
140 pgdir_attr |= PG_USER | PG_WRITABLE;
|
nuclear@55
|
141 }
|
nuclear@55
|
142
|
nuclear@55
|
143 pgdir[diridx] = addr | (pgdir_attr & ATTR_PGDIR_MASK) | PG_PRESENT;
|
nuclear@24
|
144
|
nuclear@24
|
145 pgtbl = pgon ? PGTBL(diridx) : (uint32_t*)addr;
|
nuclear@18
|
146 memset(pgtbl, 0, PGSIZE);
|
nuclear@17
|
147 } else {
|
nuclear@24
|
148 if(pgon) {
|
nuclear@24
|
149 pgtbl = PGTBL(diridx);
|
nuclear@24
|
150 } else {
|
nuclear@68
|
151 pgtbl = (uint32_t*)(pgdir[diridx] & PGENT_ADDR_MASK);
|
nuclear@24
|
152 }
|
nuclear@17
|
153 }
|
nuclear@17
|
154
|
nuclear@17
|
155 pgtbl[pgidx] = PAGE_TO_ADDR(ppage) | (attr & ATTR_PGTBL_MASK) | PG_PRESENT;
|
nuclear@23
|
156 flush_tlb_page(vpage);
|
nuclear@23
|
157
|
nuclear@25
|
158 set_intr_state(intr_state);
|
nuclear@23
|
159 return 0;
|
nuclear@17
|
160 }
|
nuclear@17
|
161
|
nuclear@43
|
162 int unmap_page(int vpage)
|
nuclear@17
|
163 {
|
nuclear@17
|
164 uint32_t *pgtbl;
|
nuclear@43
|
165 int res = 0;
|
nuclear@17
|
166 int diridx = PAGE_TO_PGTBL(vpage);
|
nuclear@17
|
167 int pgidx = PAGE_TO_PGTBL_PG(vpage);
|
nuclear@17
|
168
|
nuclear@25
|
169 int intr_state = get_intr_state();
|
nuclear@25
|
170 disable_intr();
|
nuclear@25
|
171
|
nuclear@17
|
172 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@17
|
173 goto err;
|
nuclear@17
|
174 }
|
nuclear@26
|
175 pgtbl = PGTBL(diridx);
|
nuclear@17
|
176
|
nuclear@17
|
177 if(!(pgtbl[pgidx] & PG_PRESENT)) {
|
nuclear@17
|
178 goto err;
|
nuclear@17
|
179 }
|
nuclear@17
|
180 pgtbl[pgidx] = 0;
|
nuclear@23
|
181 flush_tlb_page(vpage);
|
nuclear@17
|
182
|
nuclear@25
|
183 if(0) {
|
nuclear@17
|
184 err:
|
nuclear@25
|
185 printf("unmap_page(%d): page already not mapped\n", vpage);
|
nuclear@43
|
186 res = -1;
|
nuclear@25
|
187 }
|
nuclear@25
|
188 set_intr_state(intr_state);
|
nuclear@43
|
189 return res;
|
nuclear@17
|
190 }
|
nuclear@17
|
191
|
nuclear@22
|
192 /* if ppg_start is -1, we allocate physical pages to map with alloc_phys_page() */
|
nuclear@23
|
193 int map_page_range(int vpg_start, int pgcount, int ppg_start, unsigned int attr)
|
nuclear@17
|
194 {
|
nuclear@23
|
195 int i, phys_pg;
|
nuclear@17
|
196
|
nuclear@17
|
197 for(i=0; i<pgcount; i++) {
|
nuclear@26
|
198 phys_pg = ppg_start < 0 ? -1 : ppg_start + i;
|
nuclear@23
|
199 map_page(vpg_start + i, phys_pg, attr);
|
nuclear@17
|
200 }
|
nuclear@23
|
201 return 0;
|
nuclear@17
|
202 }
|
nuclear@17
|
203
|
nuclear@43
|
204 int unmap_page_range(int vpg_start, int pgcount)
|
nuclear@43
|
205 {
|
nuclear@43
|
206 int i, res = 0;
|
nuclear@43
|
207
|
nuclear@43
|
208 for(i=0; i<pgcount; i++) {
|
nuclear@43
|
209 if(unmap_page(vpg_start + i) == -1) {
|
nuclear@43
|
210 res = -1;
|
nuclear@43
|
211 }
|
nuclear@43
|
212 }
|
nuclear@43
|
213 return res;
|
nuclear@43
|
214 }
|
nuclear@43
|
215
|
nuclear@23
|
216 /* if paddr is 0, we allocate physical pages with alloc_phys_page() */
|
nuclear@23
|
217 int map_mem_range(uint32_t vaddr, size_t sz, uint32_t paddr, unsigned int attr)
|
nuclear@17
|
218 {
|
nuclear@17
|
219 int vpg_start, ppg_start, num_pages;
|
nuclear@17
|
220
|
nuclear@23
|
221 if(!sz) return -1;
|
nuclear@17
|
222
|
nuclear@17
|
223 if(ADDR_TO_PGOFFS(paddr)) {
|
nuclear@17
|
224 panic("map_mem_range called with unaligned physical address: %x\n", paddr);
|
nuclear@17
|
225 }
|
nuclear@17
|
226
|
nuclear@17
|
227 vpg_start = ADDR_TO_PAGE(vaddr);
|
nuclear@23
|
228 ppg_start = paddr > 0 ? ADDR_TO_PAGE(paddr) : -1;
|
nuclear@17
|
229 num_pages = ADDR_TO_PAGE(sz) + 1;
|
nuclear@17
|
230
|
nuclear@23
|
231 return map_page_range(vpg_start, num_pages, ppg_start, attr);
|
nuclear@17
|
232 }
|
nuclear@17
|
233
|
nuclear@18
|
234 uint32_t virt_to_phys(uint32_t vaddr)
|
nuclear@18
|
235 {
|
nuclear@43
|
236 int pg;
|
nuclear@43
|
237 uint32_t pgaddr;
|
nuclear@43
|
238
|
nuclear@43
|
239 if((pg = virt_to_phys_page(ADDR_TO_PAGE(vaddr))) == -1) {
|
nuclear@43
|
240 return 0;
|
nuclear@43
|
241 }
|
nuclear@43
|
242 pgaddr = PAGE_TO_ADDR(pg);
|
nuclear@43
|
243
|
nuclear@43
|
244 return pgaddr | ADDR_TO_PGOFFS(vaddr);
|
nuclear@43
|
245 }
|
nuclear@43
|
246
|
nuclear@43
|
247 int virt_to_phys_page(int vpg)
|
nuclear@43
|
248 {
|
nuclear@18
|
249 uint32_t pgaddr, *pgtbl;
|
nuclear@43
|
250 int diridx, pgidx;
|
nuclear@43
|
251
|
nuclear@43
|
252 if(vpg < 0 || vpg >= PAGE_COUNT) {
|
nuclear@43
|
253 return -1;
|
nuclear@43
|
254 }
|
nuclear@43
|
255
|
nuclear@43
|
256 diridx = PAGE_TO_PGTBL(vpg);
|
nuclear@43
|
257 pgidx = PAGE_TO_PGTBL_PG(vpg);
|
nuclear@18
|
258
|
nuclear@18
|
259 if(!(pgdir[diridx] & PG_PRESENT)) {
|
nuclear@43
|
260 return -1;
|
nuclear@18
|
261 }
|
nuclear@26
|
262 pgtbl = PGTBL(diridx);
|
nuclear@18
|
263
|
nuclear@18
|
264 if(!(pgtbl[pgidx] & PG_PRESENT)) {
|
nuclear@43
|
265 return -1;
|
nuclear@18
|
266 }
|
nuclear@18
|
267 pgaddr = pgtbl[pgidx] & PGENT_ADDR_MASK;
|
nuclear@43
|
268 return ADDR_TO_PAGE(pgaddr);
|
nuclear@18
|
269 }
|
nuclear@18
|
270
|
nuclear@22
|
271 /* allocate a contiguous block of virtual memory pages along with
|
nuclear@22
|
272 * backing physical memory for them, and update the page table.
|
nuclear@22
|
273 */
|
nuclear@22
|
274 int pgalloc(int num, int area)
|
nuclear@22
|
275 {
|
nuclear@25
|
276 int intr_state, ret = -1;
|
nuclear@22
|
277 struct page_range *node, *prev, dummy;
|
nuclear@22
|
278
|
nuclear@25
|
279 intr_state = get_intr_state();
|
nuclear@25
|
280 disable_intr();
|
nuclear@25
|
281
|
nuclear@22
|
282 dummy.next = pglist[area];
|
nuclear@22
|
283 node = pglist[area];
|
nuclear@22
|
284 prev = &dummy;
|
nuclear@22
|
285
|
nuclear@22
|
286 while(node) {
|
nuclear@22
|
287 if(node->end - node->start >= num) {
|
nuclear@22
|
288 ret = node->start;
|
nuclear@22
|
289 node->start += num;
|
nuclear@22
|
290
|
nuclear@22
|
291 if(node->start == node->end) {
|
nuclear@22
|
292 prev->next = node->next;
|
nuclear@22
|
293 node->next = 0;
|
nuclear@22
|
294
|
nuclear@22
|
295 if(node == pglist[area]) {
|
nuclear@22
|
296 pglist[area] = 0;
|
nuclear@22
|
297 }
|
nuclear@22
|
298 free_node(node);
|
nuclear@22
|
299 }
|
nuclear@22
|
300 break;
|
nuclear@22
|
301 }
|
nuclear@22
|
302
|
nuclear@22
|
303 prev = node;
|
nuclear@22
|
304 node = node->next;
|
nuclear@22
|
305 }
|
nuclear@22
|
306
|
nuclear@22
|
307 if(ret >= 0) {
|
nuclear@55
|
308 /*unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : PG_GLOBAL;*/
|
nuclear@55
|
309 unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : 0;
|
nuclear@55
|
310
|
nuclear@23
|
311 /* allocate physical storage and map */
|
nuclear@45
|
312 if(map_page_range(ret, num, -1, attr) == -1) {
|
nuclear@45
|
313 ret = -1;
|
nuclear@45
|
314 }
|
nuclear@45
|
315 }
|
nuclear@45
|
316
|
nuclear@45
|
317 set_intr_state(intr_state);
|
nuclear@45
|
318 return ret;
|
nuclear@45
|
319 }
|
nuclear@45
|
320
|
nuclear@45
|
321 int pgalloc_vrange(int start, int num)
|
nuclear@45
|
322 {
|
nuclear@45
|
323 struct page_range *node, *prev, dummy;
|
nuclear@45
|
324 int area, intr_state, ret = -1;
|
nuclear@45
|
325
|
nuclear@45
|
326 area = (start >= ADDR_TO_PAGE(KMEM_START)) ? MEM_KERNEL : MEM_USER;
|
nuclear@47
|
327 if(area == MEM_USER && start + num > ADDR_TO_PAGE(KMEM_START)) {
|
nuclear@45
|
328 printf("pgalloc_vrange: invalid range request crossing user/kernel split\n");
|
nuclear@45
|
329 return -1;
|
nuclear@45
|
330 }
|
nuclear@45
|
331
|
nuclear@45
|
332 intr_state = get_intr_state();
|
nuclear@45
|
333 disable_intr();
|
nuclear@45
|
334
|
nuclear@45
|
335 dummy.next = pglist[area];
|
nuclear@45
|
336 node = pglist[area];
|
nuclear@45
|
337 prev = &dummy;
|
nuclear@45
|
338
|
nuclear@45
|
339 /* check to see if the requested VM range is available */
|
nuclear@45
|
340 node = pglist[area];
|
nuclear@45
|
341 while(node) {
|
nuclear@45
|
342 if(start >= node->start && start + num <= node->end) {
|
nuclear@49
|
343 ret = start; /* can do .. */
|
nuclear@49
|
344
|
nuclear@49
|
345 if(start == node->start) {
|
nuclear@49
|
346 /* adjacent to the start of the range */
|
nuclear@49
|
347 node->start += num;
|
nuclear@49
|
348 } else if(start + num == node->end) {
|
nuclear@49
|
349 /* adjacent to the end of the range */
|
nuclear@49
|
350 node->end = start;
|
nuclear@49
|
351 } else {
|
nuclear@49
|
352 /* somewhere in the middle, which means we need
|
nuclear@49
|
353 * to allocate a new page_range
|
nuclear@49
|
354 */
|
nuclear@49
|
355 struct page_range *newnode;
|
nuclear@49
|
356
|
nuclear@49
|
357 if(!(newnode = alloc_node())) {
|
nuclear@49
|
358 panic("pgalloc_vrange failed to allocate new page_range while splitting a range in half... bummer\n");
|
nuclear@49
|
359 }
|
nuclear@49
|
360 newnode->start = start + num;
|
nuclear@49
|
361 newnode->end = node->end;
|
nuclear@49
|
362 newnode->next = node->next;
|
nuclear@49
|
363
|
nuclear@49
|
364 node->end = start;
|
nuclear@49
|
365 node->next = newnode;
|
nuclear@49
|
366 /* no need to check for null nodes at this point, there's
|
nuclear@49
|
367 * certainly stuff at the begining and the end, otherwise we
|
nuclear@49
|
368 * wouldn't be here. so break out of it.
|
nuclear@49
|
369 */
|
nuclear@49
|
370 break;
|
nuclear@49
|
371 }
|
nuclear@45
|
372
|
nuclear@45
|
373 if(node->start == node->end) {
|
nuclear@45
|
374 prev->next = node->next;
|
nuclear@45
|
375 node->next = 0;
|
nuclear@45
|
376
|
nuclear@45
|
377 if(node == pglist[area]) {
|
nuclear@45
|
378 pglist[area] = 0;
|
nuclear@45
|
379 }
|
nuclear@45
|
380 free_node(node);
|
nuclear@45
|
381 }
|
nuclear@45
|
382 break;
|
nuclear@45
|
383 }
|
nuclear@45
|
384
|
nuclear@45
|
385 prev = node;
|
nuclear@45
|
386 node = node->next;
|
nuclear@45
|
387 }
|
nuclear@45
|
388
|
nuclear@45
|
389 if(ret >= 0) {
|
nuclear@55
|
390 /*unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : PG_GLOBAL;*/
|
nuclear@55
|
391 unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : 0;
|
nuclear@55
|
392
|
nuclear@45
|
393 /* allocate physical storage and map */
|
nuclear@45
|
394 if(map_page_range(ret, num, -1, attr) == -1) {
|
nuclear@23
|
395 ret = -1;
|
nuclear@23
|
396 }
|
nuclear@22
|
397 }
|
nuclear@22
|
398
|
nuclear@25
|
399 set_intr_state(intr_state);
|
nuclear@22
|
400 return ret;
|
nuclear@22
|
401 }
|
nuclear@22
|
402
|
nuclear@22
|
403 void pgfree(int start, int num)
|
nuclear@22
|
404 {
|
nuclear@33
|
405 int i, area, intr_state;
|
nuclear@23
|
406 struct page_range *node, *new, *prev, *next;
|
nuclear@23
|
407
|
nuclear@25
|
408 intr_state = get_intr_state();
|
nuclear@25
|
409 disable_intr();
|
nuclear@25
|
410
|
nuclear@26
|
411 for(i=0; i<num; i++) {
|
nuclear@43
|
412 int phys_pg = virt_to_phys_page(start + i);
|
nuclear@43
|
413 if(phys_pg != -1) {
|
nuclear@43
|
414 free_phys_page(phys_pg);
|
nuclear@26
|
415 }
|
nuclear@26
|
416 }
|
nuclear@26
|
417
|
nuclear@23
|
418 if(!(new = alloc_node())) {
|
nuclear@23
|
419 panic("pgfree: can't allocate new page_range node to add the freed pages\n");
|
nuclear@23
|
420 }
|
nuclear@23
|
421 new->start = start;
|
nuclear@33
|
422 new->end = start + num;
|
nuclear@23
|
423
|
nuclear@23
|
424 area = PAGE_TO_ADDR(start) >= KMEM_START ? MEM_KERNEL : MEM_USER;
|
nuclear@23
|
425
|
nuclear@23
|
426 if(!pglist[area] || pglist[area]->start > start) {
|
nuclear@23
|
427 next = new->next = pglist[area];
|
nuclear@23
|
428 pglist[area] = new;
|
nuclear@23
|
429 prev = 0;
|
nuclear@23
|
430
|
nuclear@23
|
431 } else {
|
nuclear@23
|
432
|
nuclear@23
|
433 prev = 0;
|
nuclear@23
|
434 node = pglist[area];
|
nuclear@23
|
435 next = node ? node->next : 0;
|
nuclear@23
|
436
|
nuclear@23
|
437 while(node) {
|
nuclear@23
|
438 if(!next || next->start > start) {
|
nuclear@23
|
439 /* place here, after node */
|
nuclear@23
|
440 new->next = next;
|
nuclear@23
|
441 node->next = new;
|
nuclear@23
|
442 prev = node; /* needed by coalesce after the loop */
|
nuclear@23
|
443 break;
|
nuclear@23
|
444 }
|
nuclear@23
|
445
|
nuclear@23
|
446 prev = node;
|
nuclear@23
|
447 node = next;
|
nuclear@23
|
448 next = node ? node->next : 0;
|
nuclear@23
|
449 }
|
nuclear@23
|
450 }
|
nuclear@23
|
451
|
nuclear@23
|
452 coalesce(prev, new, next);
|
nuclear@25
|
453 set_intr_state(intr_state);
|
nuclear@23
|
454 }
|
nuclear@23
|
455
|
nuclear@23
|
456 static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high)
|
nuclear@23
|
457 {
|
nuclear@23
|
458 if(high) {
|
nuclear@23
|
459 if(mid->end == high->start) {
|
nuclear@23
|
460 mid->end = high->end;
|
nuclear@23
|
461 mid->next = high->next;
|
nuclear@23
|
462 free_node(high);
|
nuclear@23
|
463 }
|
nuclear@23
|
464 }
|
nuclear@23
|
465
|
nuclear@23
|
466 if(low) {
|
nuclear@23
|
467 if(low->end == mid->start) {
|
nuclear@23
|
468 low->end += mid->end;
|
nuclear@23
|
469 low->next = mid->next;
|
nuclear@23
|
470 free_node(mid);
|
nuclear@23
|
471 }
|
nuclear@23
|
472 }
|
nuclear@22
|
473 }
|
nuclear@22
|
474
|
nuclear@52
|
475 static void pgfault(int inum)
|
nuclear@17
|
476 {
|
nuclear@52
|
477 struct intr_frame *frm = get_intr_frame();
|
nuclear@52
|
478 uint32_t fault_addr = get_fault_addr();
|
nuclear@52
|
479
|
nuclear@52
|
480 /* the fault occured in user space */
|
nuclear@55
|
481 if(frm->err & PG_USER) {
|
nuclear@52
|
482 int fault_page = ADDR_TO_PAGE(fault_addr);
|
nuclear@52
|
483 struct process *proc = get_current_proc();
|
nuclear@55
|
484 printf("DBG: page fault in user space\n");
|
nuclear@52
|
485 assert(proc);
|
nuclear@52
|
486
|
nuclear@52
|
487 if(frm->err & PG_PRESENT) {
|
nuclear@52
|
488 /* it's not due to a missing page, just panic */
|
nuclear@52
|
489 goto unhandled;
|
nuclear@52
|
490 }
|
nuclear@52
|
491
|
nuclear@52
|
492 /* detect if it's an automatic stack growth deal */
|
nuclear@55
|
493 if(fault_page < proc->user_stack_pg && proc->user_stack_pg - fault_page < USTACK_MAXGROW) {
|
nuclear@55
|
494 int num_pages = proc->user_stack_pg - fault_page;
|
nuclear@52
|
495 printf("growing user (%d) stack by %d pages\n", proc->id, num_pages);
|
nuclear@52
|
496
|
nuclear@52
|
497 if(pgalloc_vrange(fault_page, num_pages) != fault_page) {
|
nuclear@52
|
498 printf("failed to allocate VM for stack growth\n");
|
nuclear@52
|
499 /* TODO: in the future we'd SIGSEGV the process here, for now just panic */
|
nuclear@52
|
500 goto unhandled;
|
nuclear@52
|
501 }
|
nuclear@55
|
502 proc->user_stack_pg = fault_page;
|
nuclear@52
|
503 return;
|
nuclear@52
|
504 }
|
nuclear@52
|
505 }
|
nuclear@52
|
506
|
nuclear@52
|
507 unhandled:
|
nuclear@17
|
508 printf("~~~~ PAGE FAULT ~~~~\n");
|
nuclear@52
|
509 printf("fault address: %x\n", fault_addr);
|
nuclear@17
|
510
|
nuclear@51
|
511 if(frm->err & PG_PRESENT) {
|
nuclear@51
|
512 if(frm->err & 8) {
|
nuclear@17
|
513 printf("reserved bit set in some paging structure\n");
|
nuclear@17
|
514 } else {
|
nuclear@55
|
515 printf("%s protection violation ", (frm->err & PG_WRITABLE) ? "WRITE" : "READ");
|
nuclear@55
|
516 printf("in %s mode\n", (frm->err & PG_USER) ? "user" : "kernel");
|
nuclear@17
|
517 }
|
nuclear@17
|
518 } else {
|
nuclear@17
|
519 printf("page not present\n");
|
nuclear@17
|
520 }
|
nuclear@19
|
521
|
nuclear@19
|
522 panic("unhandled page fault\n");
|
nuclear@17
|
523 }
|
nuclear@22
|
524
|
nuclear@22
|
525 /* --- page range list node management --- */
|
nuclear@23
|
526 #define NODES_IN_PAGE (PGSIZE / sizeof(struct page_range))
|
nuclear@23
|
527
|
nuclear@22
|
528 static struct page_range *alloc_node(void)
|
nuclear@22
|
529 {
|
nuclear@22
|
530 struct page_range *node;
|
nuclear@23
|
531 int pg, i;
|
nuclear@22
|
532
|
nuclear@22
|
533 if(node_pool) {
|
nuclear@22
|
534 node = node_pool;
|
nuclear@22
|
535 node_pool = node_pool->next;
|
nuclear@47
|
536 /*printf("alloc_node -> %x\n", (unsigned int)node);*/
|
nuclear@22
|
537 return node;
|
nuclear@22
|
538 }
|
nuclear@22
|
539
|
nuclear@23
|
540 /* no node structures in the pool, we need to allocate a new page,
|
nuclear@23
|
541 * split it up into node structures, add them in the pool, and
|
nuclear@23
|
542 * allocate one of them.
|
nuclear@22
|
543 */
|
nuclear@23
|
544 if(!(pg = pgalloc(1, MEM_KERNEL))) {
|
nuclear@22
|
545 panic("ran out of physical memory while allocating VM range structures\n");
|
nuclear@22
|
546 }
|
nuclear@23
|
547 node_pool = (struct page_range*)PAGE_TO_ADDR(pg);
|
nuclear@22
|
548
|
nuclear@23
|
549 /* link them up, skip the first as we'll just allocate it anyway */
|
nuclear@23
|
550 for(i=2; i<NODES_IN_PAGE; i++) {
|
nuclear@23
|
551 node_pool[i - 1].next = node_pool + i;
|
nuclear@23
|
552 }
|
nuclear@23
|
553 node_pool[NODES_IN_PAGE - 1].next = 0;
|
nuclear@23
|
554
|
nuclear@23
|
555 /* grab the first and return it */
|
nuclear@23
|
556 node = node_pool++;
|
nuclear@47
|
557 /*printf("alloc_node -> %x\n", (unsigned int)node);*/
|
nuclear@23
|
558 return node;
|
nuclear@22
|
559 }
|
nuclear@22
|
560
|
nuclear@22
|
561 static void free_node(struct page_range *node)
|
nuclear@22
|
562 {
|
nuclear@22
|
563 node->next = node_pool;
|
nuclear@22
|
564 node_pool = node;
|
nuclear@47
|
565 /*printf("free_node\n");*/
|
nuclear@22
|
566 }
|
nuclear@23
|
567
|
nuclear@47
|
568 /* clone_vm makes a copy of the current page tables, thus duplicating the
|
nuclear@47
|
569 * virtual address space.
|
nuclear@47
|
570 *
|
nuclear@47
|
571 * For the kernel part of the address space (last 256 page directory entries)
|
nuclear@47
|
572 * we don't want to diplicate the page tables, just point all page directory
|
nuclear@47
|
573 * entries to the same set of page tables.
|
nuclear@43
|
574 *
|
nuclear@57
|
575 * If "cow" is non-zero it also marks the shared user-space pages as
|
nuclear@57
|
576 * read-only, to implement copy-on-write.
|
nuclear@57
|
577 *
|
nuclear@43
|
578 * Returns the physical address of the new page directory.
|
nuclear@43
|
579 */
|
nuclear@57
|
580 uint32_t clone_vm(int cow)
|
nuclear@43
|
581 {
|
nuclear@57
|
582 int i, j, dirpg, tblpg, kstart_dirent;
|
nuclear@43
|
583 uint32_t paddr;
|
nuclear@43
|
584 uint32_t *ndir, *ntbl;
|
nuclear@43
|
585
|
nuclear@47
|
586 /* allocate the new page directory */
|
nuclear@43
|
587 if((dirpg = pgalloc(1, MEM_KERNEL)) == -1) {
|
nuclear@43
|
588 panic("clone_vmem: failed to allocate page directory page\n");
|
nuclear@43
|
589 }
|
nuclear@43
|
590 ndir = (uint32_t*)PAGE_TO_ADDR(dirpg);
|
nuclear@43
|
591
|
nuclear@47
|
592 /* allocate a virtual page for temporarily mapping all new
|
nuclear@47
|
593 * page tables while we populate them.
|
nuclear@47
|
594 */
|
nuclear@43
|
595 if((tblpg = pgalloc(1, MEM_KERNEL)) == -1) {
|
nuclear@43
|
596 panic("clone_vmem: failed to allocate page table page\n");
|
nuclear@43
|
597 }
|
nuclear@43
|
598 ntbl = (uint32_t*)PAGE_TO_ADDR(tblpg);
|
nuclear@43
|
599
|
nuclear@43
|
600 /* we will allocate physical pages and map them to this virtual page
|
nuclear@57
|
601 * as needed in the loop below. we don't need the physical page allocated
|
nuclear@57
|
602 * by pgalloc.
|
nuclear@43
|
603 */
|
nuclear@49
|
604 free_phys_page(virt_to_phys((uint32_t)ntbl));
|
nuclear@43
|
605
|
nuclear@48
|
606 kstart_dirent = ADDR_TO_PAGE(KMEM_START) / 1024;
|
nuclear@47
|
607
|
nuclear@47
|
608 /* user space */
|
nuclear@48
|
609 for(i=0; i<kstart_dirent; i++) {
|
nuclear@43
|
610 if(pgdir[i] & PG_PRESENT) {
|
nuclear@64
|
611 if(cow) {
|
nuclear@64
|
612 /* first go through all the entries of the existing
|
nuclear@64
|
613 * page table and unset the writable bits.
|
nuclear@64
|
614 */
|
nuclear@64
|
615 for(j=0; j<1024; j++) {
|
nuclear@68
|
616 clear_page_bit(i * 1024 + j, PG_WRITABLE, PAGE_ONLY);
|
nuclear@68
|
617 /*PGTBL(i)[j] &= ~(uint32_t)PG_WRITABLE;*/
|
nuclear@64
|
618 }
|
nuclear@57
|
619 }
|
nuclear@57
|
620
|
nuclear@57
|
621 /* allocate a page table for the clone */
|
nuclear@43
|
622 paddr = alloc_phys_page();
|
nuclear@43
|
623
|
nuclear@43
|
624 /* copy the page table */
|
nuclear@57
|
625 map_page(tblpg, ADDR_TO_PAGE(paddr), 0);
|
nuclear@43
|
626 memcpy(ntbl, PGTBL(i), PGSIZE);
|
nuclear@43
|
627
|
nuclear@43
|
628 /* set the new page directory entry */
|
nuclear@43
|
629 ndir[i] = paddr | (pgdir[i] & PGOFFS_MASK);
|
nuclear@43
|
630 } else {
|
nuclear@43
|
631 ndir[i] = 0;
|
nuclear@43
|
632 }
|
nuclear@43
|
633 }
|
nuclear@43
|
634
|
nuclear@55
|
635 /* for the kernel space we'll just use the same page tables */
|
nuclear@48
|
636 for(i=kstart_dirent; i<1024; i++) {
|
nuclear@49
|
637 ndir[i] = pgdir[i];
|
nuclear@47
|
638 }
|
nuclear@47
|
639
|
nuclear@64
|
640 if(cow) {
|
nuclear@64
|
641 /* we just changed all the page protection bits, so we need to flush the TLB */
|
nuclear@64
|
642 flush_tlb();
|
nuclear@64
|
643 }
|
nuclear@57
|
644
|
nuclear@49
|
645 paddr = virt_to_phys((uint32_t)ndir);
|
nuclear@43
|
646
|
nuclear@57
|
647 /* unmap before freeing the virtual pages, to avoid deallocating the physical pages */
|
nuclear@43
|
648 unmap_page(dirpg);
|
nuclear@43
|
649 unmap_page(tblpg);
|
nuclear@43
|
650
|
nuclear@43
|
651 pgfree(dirpg, 1);
|
nuclear@43
|
652 pgfree(tblpg, 1);
|
nuclear@43
|
653
|
nuclear@43
|
654 return paddr;
|
nuclear@43
|
655 }
|
nuclear@57
|
656
|
nuclear@57
|
657 int get_page_bit(int pgnum, uint32_t bit, int wholepath)
|
nuclear@57
|
658 {
|
nuclear@57
|
659 int tidx = PAGE_TO_PGTBL(pgnum);
|
nuclear@57
|
660 int tent = PAGE_TO_PGTBL_PG(pgnum);
|
nuclear@57
|
661 uint32_t *pgtbl = PGTBL(tidx);
|
nuclear@57
|
662
|
nuclear@57
|
663 if(wholepath) {
|
nuclear@57
|
664 if((pgdir[tidx] & bit) == 0) {
|
nuclear@57
|
665 return 0;
|
nuclear@57
|
666 }
|
nuclear@57
|
667 }
|
nuclear@57
|
668
|
nuclear@57
|
669 return pgtbl[tent] & bit;
|
nuclear@57
|
670 }
|
nuclear@57
|
671
|
nuclear@57
|
672 void set_page_bit(int pgnum, uint32_t bit, int wholepath)
|
nuclear@57
|
673 {
|
nuclear@57
|
674 int tidx = PAGE_TO_PGTBL(pgnum);
|
nuclear@57
|
675 int tent = PAGE_TO_PGTBL_PG(pgnum);
|
nuclear@57
|
676 uint32_t *pgtbl = PGTBL(tidx);
|
nuclear@57
|
677
|
nuclear@57
|
678 if(wholepath) {
|
nuclear@57
|
679 pgdir[tidx] |= bit;
|
nuclear@57
|
680 }
|
nuclear@57
|
681 pgtbl[tent] |= bit;
|
nuclear@57
|
682
|
nuclear@57
|
683 flush_tlb_page(pgnum);
|
nuclear@57
|
684 }
|
nuclear@57
|
685
|
nuclear@57
|
686 void clear_page_bit(int pgnum, uint32_t bit, int wholepath)
|
nuclear@57
|
687 {
|
nuclear@57
|
688 int tidx = PAGE_TO_PGTBL(pgnum);
|
nuclear@57
|
689 int tent = PAGE_TO_PGTBL_PG(pgnum);
|
nuclear@57
|
690 uint32_t *pgtbl = PGTBL(tidx);
|
nuclear@57
|
691
|
nuclear@57
|
692 if(wholepath) {
|
nuclear@57
|
693 pgdir[tidx] &= ~bit;
|
nuclear@57
|
694 }
|
nuclear@57
|
695
|
nuclear@57
|
696 pgtbl[tent] &= ~bit;
|
nuclear@57
|
697
|
nuclear@57
|
698 flush_tlb_page(pgnum);
|
nuclear@57
|
699 }
|
nuclear@43
|
700
|
nuclear@43
|
701
|
nuclear@68
|
702 #define USER_PGDIR_ENTRIES PAGE_TO_PGTBL(KMEM_START_PAGE)
|
nuclear@68
|
703 int cons_vmmap(struct rbtree *vmmap)
|
nuclear@68
|
704 {
|
nuclear@68
|
705 int i, j;
|
nuclear@68
|
706
|
nuclear@68
|
707 rb_init(vmmap, RB_KEY_INT);
|
nuclear@68
|
708
|
nuclear@68
|
709 for(i=0; i<USER_PGDIR_ENTRIES; i++) {
|
nuclear@68
|
710 if(pgdir[i] & PG_PRESENT) {
|
nuclear@68
|
711 /* page table is present, iterate through its 1024 pages */
|
nuclear@68
|
712 uint32_t *pgtbl = PGTBL(i);
|
nuclear@68
|
713
|
nuclear@68
|
714 for(j=0; j<1024; j++) {
|
nuclear@68
|
715 if(pgtbl[j] & PG_PRESENT) {
|
nuclear@68
|
716 struct vm_page *vmp;
|
nuclear@68
|
717
|
nuclear@68
|
718 if(!(vmp = malloc(sizeof *vmp))) {
|
nuclear@68
|
719 panic("cons_vmap failed to allocate memory");
|
nuclear@68
|
720 }
|
nuclear@68
|
721 vmp->vpage = i * 1024 + j;
|
nuclear@68
|
722 vmp->ppage = ADDR_TO_PAGE(pgtbl[j] & PGENT_ADDR_MASK);
|
nuclear@68
|
723 vmp->flags = pgtbl[j] & ATTR_PGTBL_MASK;
|
nuclear@68
|
724 vmp->nref = 1; /* when first created assume no sharing */
|
nuclear@68
|
725
|
nuclear@68
|
726 rb_inserti(vmmap, vmp->ppage, vmp);
|
nuclear@68
|
727 }
|
nuclear@68
|
728 }
|
nuclear@68
|
729 }
|
nuclear@68
|
730 }
|
nuclear@68
|
731
|
nuclear@68
|
732 return 0;
|
nuclear@68
|
733 }
|
nuclear@68
|
734
|
nuclear@68
|
735
|
nuclear@23
|
736 void dbg_print_vm(int area)
|
nuclear@23
|
737 {
|
nuclear@25
|
738 struct page_range *node;
|
nuclear@25
|
739 int last, intr_state;
|
nuclear@25
|
740
|
nuclear@25
|
741 intr_state = get_intr_state();
|
nuclear@25
|
742 disable_intr();
|
nuclear@25
|
743
|
nuclear@25
|
744 node = pglist[area];
|
nuclear@25
|
745 last = area == MEM_USER ? 0 : ADDR_TO_PAGE(KMEM_START);
|
nuclear@23
|
746
|
nuclear@23
|
747 printf("%s vm space\n", area == MEM_USER ? "user" : "kernel");
|
nuclear@23
|
748
|
nuclear@23
|
749 while(node) {
|
nuclear@23
|
750 if(node->start > last) {
|
nuclear@23
|
751 printf(" vm-used: %x -> %x\n", PAGE_TO_ADDR(last), PAGE_TO_ADDR(node->start));
|
nuclear@23
|
752 }
|
nuclear@23
|
753
|
nuclear@23
|
754 printf(" vm-free: %x -> ", PAGE_TO_ADDR(node->start));
|
nuclear@23
|
755 if(node->end >= PAGE_COUNT) {
|
nuclear@23
|
756 printf("END\n");
|
nuclear@23
|
757 } else {
|
nuclear@23
|
758 printf("%x\n", PAGE_TO_ADDR(node->end));
|
nuclear@23
|
759 }
|
nuclear@23
|
760
|
nuclear@23
|
761 last = node->end;
|
nuclear@23
|
762 node = node->next;
|
nuclear@23
|
763 }
|
nuclear@25
|
764
|
nuclear@25
|
765 set_intr_state(intr_state);
|
nuclear@23
|
766 }
|