nuclear@17: #include nuclear@17: #include nuclear@17: #include nuclear@52: #include nuclear@52: #include "config.h" nuclear@17: #include "vm.h" nuclear@17: #include "intr.h" nuclear@17: #include "mem.h" nuclear@17: #include "panic.h" nuclear@52: #include "proc.h" nuclear@17: nuclear@17: #define IDMAP_START 0xa0000 nuclear@17: nuclear@24: #define PGDIR_ADDR 0xfffff000 nuclear@24: #define PGTBL_BASE (0xffffffff - 4096 * 1024 + 1) nuclear@24: #define PGTBL(x) ((uint32_t*)(PGTBL_BASE + PGSIZE * (x))) nuclear@24: nuclear@17: #define ATTR_PGDIR_MASK 0x3f nuclear@17: #define ATTR_PGTBL_MASK 0x1ff nuclear@17: #define ADDR_PGENT_MASK 0xfffff000 nuclear@17: nuclear@17: #define PAGEFAULT 14 nuclear@17: nuclear@22: nuclear@22: struct page_range { nuclear@22: int start, end; nuclear@22: struct page_range *next; nuclear@22: }; nuclear@22: nuclear@22: /* defined in vm-asm.S */ nuclear@22: void enable_paging(void); nuclear@23: void disable_paging(void); nuclear@23: int get_paging_status(void); nuclear@22: void set_pgdir_addr(uint32_t addr); nuclear@23: void flush_tlb(void); nuclear@23: void flush_tlb_addr(uint32_t addr); nuclear@23: #define flush_tlb_page(p) flush_tlb_addr(PAGE_TO_ADDR(p)) nuclear@22: uint32_t get_fault_addr(void); nuclear@22: nuclear@23: static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high); nuclear@52: static void pgfault(int inum); nuclear@22: static struct page_range *alloc_node(void); nuclear@22: static void free_node(struct page_range *node); nuclear@22: nuclear@22: /* page directory */ nuclear@22: static uint32_t *pgdir; nuclear@22: nuclear@22: /* 2 lists of free ranges, for kernel memory and user memory */ nuclear@22: static struct page_range *pglist[2]; nuclear@22: /* list of free page_range structures to be used in the lists */ nuclear@22: static struct page_range *node_pool; nuclear@23: /* the first page range for the whole kernel address space, to get things started */ nuclear@23: static struct page_range first_node; nuclear@22: nuclear@22: nuclear@26: void init_vm(void) nuclear@17: { nuclear@19: uint32_t idmap_end; nuclear@47: int i, kmem_start_pg, pgtbl_base_pg; nuclear@19: nuclear@23: /* setup the page tables */ nuclear@18: pgdir = (uint32_t*)alloc_phys_page(); nuclear@23: memset(pgdir, 0, PGSIZE); nuclear@24: set_pgdir_addr((uint32_t)pgdir); nuclear@17: nuclear@17: /* map the video memory and kernel code 1-1 */ nuclear@19: get_kernel_mem_range(0, &idmap_end); nuclear@19: map_mem_range(IDMAP_START, idmap_end - IDMAP_START, IDMAP_START, 0); nuclear@17: nuclear@24: /* make the last page directory entry point to the page directory */ nuclear@24: pgdir[1023] = ((uint32_t)pgdir & ADDR_PGENT_MASK) | PG_PRESENT; nuclear@24: pgdir = (uint32_t*)PGDIR_ADDR; nuclear@24: nuclear@23: /* set the page fault handler */ nuclear@17: interrupt(PAGEFAULT, pgfault); nuclear@17: nuclear@23: /* we can enable paging now */ nuclear@17: enable_paging(); nuclear@23: nuclear@23: /* initialize the virtual page allocator */ nuclear@23: node_pool = 0; nuclear@23: nuclear@47: kmem_start_pg = ADDR_TO_PAGE(KMEM_START); nuclear@47: pgtbl_base_pg = ADDR_TO_PAGE(PGTBL_BASE); nuclear@47: nuclear@47: first_node.start = kmem_start_pg; nuclear@47: first_node.end = pgtbl_base_pg; nuclear@23: first_node.next = 0; nuclear@23: pglist[MEM_KERNEL] = &first_node; nuclear@23: nuclear@23: pglist[MEM_USER] = alloc_node(); nuclear@26: pglist[MEM_USER]->start = ADDR_TO_PAGE(idmap_end); nuclear@47: pglist[MEM_USER]->end = kmem_start_pg; nuclear@23: pglist[MEM_USER]->next = 0; nuclear@47: nuclear@47: /* temporaroly map something into every 1024th page of the kernel address nuclear@47: * space to force pre-allocation of all the kernel page-tables nuclear@47: */ nuclear@47: for(i=kmem_start_pg; i 0 ? ADDR_TO_PAGE(paddr) : -1; nuclear@17: num_pages = ADDR_TO_PAGE(sz) + 1; nuclear@17: nuclear@23: return map_page_range(vpg_start, num_pages, ppg_start, attr); nuclear@17: } nuclear@17: nuclear@18: uint32_t virt_to_phys(uint32_t vaddr) nuclear@18: { nuclear@43: int pg; nuclear@43: uint32_t pgaddr; nuclear@43: nuclear@43: if((pg = virt_to_phys_page(ADDR_TO_PAGE(vaddr))) == -1) { nuclear@43: return 0; nuclear@43: } nuclear@43: pgaddr = PAGE_TO_ADDR(pg); nuclear@43: nuclear@43: return pgaddr | ADDR_TO_PGOFFS(vaddr); nuclear@43: } nuclear@43: nuclear@43: int virt_to_phys_page(int vpg) nuclear@43: { nuclear@18: uint32_t pgaddr, *pgtbl; nuclear@43: int diridx, pgidx; nuclear@43: nuclear@43: if(vpg < 0 || vpg >= PAGE_COUNT) { nuclear@43: return -1; nuclear@43: } nuclear@43: nuclear@43: diridx = PAGE_TO_PGTBL(vpg); nuclear@43: pgidx = PAGE_TO_PGTBL_PG(vpg); nuclear@18: nuclear@18: if(!(pgdir[diridx] & PG_PRESENT)) { nuclear@43: return -1; nuclear@18: } nuclear@26: pgtbl = PGTBL(diridx); nuclear@18: nuclear@18: if(!(pgtbl[pgidx] & PG_PRESENT)) { nuclear@43: return -1; nuclear@18: } nuclear@18: pgaddr = pgtbl[pgidx] & PGENT_ADDR_MASK; nuclear@43: return ADDR_TO_PAGE(pgaddr); nuclear@18: } nuclear@18: nuclear@22: /* allocate a contiguous block of virtual memory pages along with nuclear@22: * backing physical memory for them, and update the page table. nuclear@22: */ nuclear@22: int pgalloc(int num, int area) nuclear@22: { nuclear@25: int intr_state, ret = -1; nuclear@22: struct page_range *node, *prev, dummy; nuclear@22: nuclear@25: intr_state = get_intr_state(); nuclear@25: disable_intr(); nuclear@25: nuclear@22: dummy.next = pglist[area]; nuclear@22: node = pglist[area]; nuclear@22: prev = &dummy; nuclear@22: nuclear@22: while(node) { nuclear@22: if(node->end - node->start >= num) { nuclear@22: ret = node->start; nuclear@22: node->start += num; nuclear@22: nuclear@22: if(node->start == node->end) { nuclear@22: prev->next = node->next; nuclear@22: node->next = 0; nuclear@22: nuclear@22: if(node == pglist[area]) { nuclear@22: pglist[area] = 0; nuclear@22: } nuclear@22: free_node(node); nuclear@22: } nuclear@22: break; nuclear@22: } nuclear@22: nuclear@22: prev = node; nuclear@22: node = node->next; nuclear@22: } nuclear@22: nuclear@22: if(ret >= 0) { nuclear@55: /*unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : PG_GLOBAL;*/ nuclear@55: unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : 0; nuclear@55: nuclear@23: /* allocate physical storage and map */ nuclear@45: if(map_page_range(ret, num, -1, attr) == -1) { nuclear@45: ret = -1; nuclear@45: } nuclear@45: } nuclear@45: nuclear@45: set_intr_state(intr_state); nuclear@45: return ret; nuclear@45: } nuclear@45: nuclear@45: int pgalloc_vrange(int start, int num) nuclear@45: { nuclear@45: struct page_range *node, *prev, dummy; nuclear@45: int area, intr_state, ret = -1; nuclear@45: nuclear@45: area = (start >= ADDR_TO_PAGE(KMEM_START)) ? MEM_KERNEL : MEM_USER; nuclear@47: if(area == MEM_USER && start + num > ADDR_TO_PAGE(KMEM_START)) { nuclear@45: printf("pgalloc_vrange: invalid range request crossing user/kernel split\n"); nuclear@45: return -1; nuclear@45: } nuclear@45: nuclear@45: intr_state = get_intr_state(); nuclear@45: disable_intr(); nuclear@45: nuclear@45: dummy.next = pglist[area]; nuclear@45: node = pglist[area]; nuclear@45: prev = &dummy; nuclear@45: nuclear@45: /* check to see if the requested VM range is available */ nuclear@45: node = pglist[area]; nuclear@45: while(node) { nuclear@45: if(start >= node->start && start + num <= node->end) { nuclear@49: ret = start; /* can do .. */ nuclear@49: nuclear@49: if(start == node->start) { nuclear@49: /* adjacent to the start of the range */ nuclear@49: node->start += num; nuclear@49: } else if(start + num == node->end) { nuclear@49: /* adjacent to the end of the range */ nuclear@49: node->end = start; nuclear@49: } else { nuclear@49: /* somewhere in the middle, which means we need nuclear@49: * to allocate a new page_range nuclear@49: */ nuclear@49: struct page_range *newnode; nuclear@49: nuclear@49: if(!(newnode = alloc_node())) { nuclear@49: panic("pgalloc_vrange failed to allocate new page_range while splitting a range in half... bummer\n"); nuclear@49: } nuclear@49: newnode->start = start + num; nuclear@49: newnode->end = node->end; nuclear@49: newnode->next = node->next; nuclear@49: nuclear@49: node->end = start; nuclear@49: node->next = newnode; nuclear@49: /* no need to check for null nodes at this point, there's nuclear@49: * certainly stuff at the begining and the end, otherwise we nuclear@49: * wouldn't be here. so break out of it. nuclear@49: */ nuclear@49: break; nuclear@49: } nuclear@45: nuclear@45: if(node->start == node->end) { nuclear@45: prev->next = node->next; nuclear@45: node->next = 0; nuclear@45: nuclear@45: if(node == pglist[area]) { nuclear@45: pglist[area] = 0; nuclear@45: } nuclear@45: free_node(node); nuclear@45: } nuclear@45: break; nuclear@45: } nuclear@45: nuclear@45: prev = node; nuclear@45: node = node->next; nuclear@45: } nuclear@45: nuclear@45: if(ret >= 0) { nuclear@55: /*unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : PG_GLOBAL;*/ nuclear@55: unsigned int attr = (area == MEM_USER) ? (PG_USER | PG_WRITABLE) : 0; nuclear@55: nuclear@45: /* allocate physical storage and map */ nuclear@45: if(map_page_range(ret, num, -1, attr) == -1) { nuclear@23: ret = -1; nuclear@23: } nuclear@22: } nuclear@22: nuclear@25: set_intr_state(intr_state); nuclear@22: return ret; nuclear@22: } nuclear@22: nuclear@22: void pgfree(int start, int num) nuclear@22: { nuclear@33: int i, area, intr_state; nuclear@23: struct page_range *node, *new, *prev, *next; nuclear@23: nuclear@25: intr_state = get_intr_state(); nuclear@25: disable_intr(); nuclear@25: nuclear@26: for(i=0; istart = start; nuclear@33: new->end = start + num; nuclear@23: nuclear@23: area = PAGE_TO_ADDR(start) >= KMEM_START ? MEM_KERNEL : MEM_USER; nuclear@23: nuclear@23: if(!pglist[area] || pglist[area]->start > start) { nuclear@23: next = new->next = pglist[area]; nuclear@23: pglist[area] = new; nuclear@23: prev = 0; nuclear@23: nuclear@23: } else { nuclear@23: nuclear@23: prev = 0; nuclear@23: node = pglist[area]; nuclear@23: next = node ? node->next : 0; nuclear@23: nuclear@23: while(node) { nuclear@23: if(!next || next->start > start) { nuclear@23: /* place here, after node */ nuclear@23: new->next = next; nuclear@23: node->next = new; nuclear@23: prev = node; /* needed by coalesce after the loop */ nuclear@23: break; nuclear@23: } nuclear@23: nuclear@23: prev = node; nuclear@23: node = next; nuclear@23: next = node ? node->next : 0; nuclear@23: } nuclear@23: } nuclear@23: nuclear@23: coalesce(prev, new, next); nuclear@25: set_intr_state(intr_state); nuclear@23: } nuclear@23: nuclear@23: static void coalesce(struct page_range *low, struct page_range *mid, struct page_range *high) nuclear@23: { nuclear@23: if(high) { nuclear@23: if(mid->end == high->start) { nuclear@23: mid->end = high->end; nuclear@23: mid->next = high->next; nuclear@23: free_node(high); nuclear@23: } nuclear@23: } nuclear@23: nuclear@23: if(low) { nuclear@23: if(low->end == mid->start) { nuclear@23: low->end += mid->end; nuclear@23: low->next = mid->next; nuclear@23: free_node(mid); nuclear@23: } nuclear@23: } nuclear@22: } nuclear@22: nuclear@52: static void pgfault(int inum) nuclear@17: { nuclear@52: struct intr_frame *frm = get_intr_frame(); nuclear@52: uint32_t fault_addr = get_fault_addr(); nuclear@52: nuclear@52: /* the fault occured in user space */ nuclear@55: if(frm->err & PG_USER) { nuclear@52: int fault_page = ADDR_TO_PAGE(fault_addr); nuclear@52: struct process *proc = get_current_proc(); nuclear@55: printf("DBG: page fault in user space\n"); nuclear@52: assert(proc); nuclear@52: nuclear@52: if(frm->err & PG_PRESENT) { nuclear@52: /* it's not due to a missing page, just panic */ nuclear@52: goto unhandled; nuclear@52: } nuclear@52: nuclear@52: /* detect if it's an automatic stack growth deal */ nuclear@55: if(fault_page < proc->user_stack_pg && proc->user_stack_pg - fault_page < USTACK_MAXGROW) { nuclear@55: int num_pages = proc->user_stack_pg - fault_page; nuclear@52: printf("growing user (%d) stack by %d pages\n", proc->id, num_pages); nuclear@52: nuclear@52: if(pgalloc_vrange(fault_page, num_pages) != fault_page) { nuclear@52: printf("failed to allocate VM for stack growth\n"); nuclear@52: /* TODO: in the future we'd SIGSEGV the process here, for now just panic */ nuclear@52: goto unhandled; nuclear@52: } nuclear@55: proc->user_stack_pg = fault_page; nuclear@52: return; nuclear@52: } nuclear@52: } nuclear@52: nuclear@52: unhandled: nuclear@17: printf("~~~~ PAGE FAULT ~~~~\n"); nuclear@52: printf("fault address: %x\n", fault_addr); nuclear@17: nuclear@51: if(frm->err & PG_PRESENT) { nuclear@51: if(frm->err & 8) { nuclear@17: printf("reserved bit set in some paging structure\n"); nuclear@17: } else { nuclear@55: printf("%s protection violation ", (frm->err & PG_WRITABLE) ? "WRITE" : "READ"); nuclear@55: printf("in %s mode\n", (frm->err & PG_USER) ? "user" : "kernel"); nuclear@17: } nuclear@17: } else { nuclear@17: printf("page not present\n"); nuclear@17: } nuclear@19: nuclear@19: panic("unhandled page fault\n"); nuclear@17: } nuclear@22: nuclear@22: /* --- page range list node management --- */ nuclear@23: #define NODES_IN_PAGE (PGSIZE / sizeof(struct page_range)) nuclear@23: nuclear@22: static struct page_range *alloc_node(void) nuclear@22: { nuclear@22: struct page_range *node; nuclear@23: int pg, i; nuclear@22: nuclear@22: if(node_pool) { nuclear@22: node = node_pool; nuclear@22: node_pool = node_pool->next; nuclear@47: /*printf("alloc_node -> %x\n", (unsigned int)node);*/ nuclear@22: return node; nuclear@22: } nuclear@22: nuclear@23: /* no node structures in the pool, we need to allocate a new page, nuclear@23: * split it up into node structures, add them in the pool, and nuclear@23: * allocate one of them. nuclear@22: */ nuclear@23: if(!(pg = pgalloc(1, MEM_KERNEL))) { nuclear@22: panic("ran out of physical memory while allocating VM range structures\n"); nuclear@22: } nuclear@23: node_pool = (struct page_range*)PAGE_TO_ADDR(pg); nuclear@22: nuclear@23: /* link them up, skip the first as we'll just allocate it anyway */ nuclear@23: for(i=2; i %x\n", (unsigned int)node);*/ nuclear@23: return node; nuclear@22: } nuclear@22: nuclear@22: static void free_node(struct page_range *node) nuclear@22: { nuclear@22: node->next = node_pool; nuclear@22: node_pool = node; nuclear@47: /*printf("free_node\n");*/ nuclear@22: } nuclear@23: nuclear@55: #if 0 nuclear@47: /* clone_vm makes a copy of the current page tables, thus duplicating the nuclear@47: * virtual address space. nuclear@47: * nuclear@47: * For the kernel part of the address space (last 256 page directory entries) nuclear@47: * we don't want to diplicate the page tables, just point all page directory nuclear@47: * entries to the same set of page tables. nuclear@43: * nuclear@43: * Returns the physical address of the new page directory. nuclear@43: */ nuclear@47: uint32_t clone_vm(void) nuclear@43: { nuclear@48: int i, dirpg, tblpg, kstart_dirent; nuclear@43: uint32_t paddr; nuclear@43: uint32_t *ndir, *ntbl; nuclear@43: nuclear@47: /* allocate the new page directory */ nuclear@43: if((dirpg = pgalloc(1, MEM_KERNEL)) == -1) { nuclear@43: panic("clone_vmem: failed to allocate page directory page\n"); nuclear@43: } nuclear@43: ndir = (uint32_t*)PAGE_TO_ADDR(dirpg); nuclear@43: nuclear@47: /* allocate a virtual page for temporarily mapping all new nuclear@47: * page tables while we populate them. nuclear@47: */ nuclear@43: if((tblpg = pgalloc(1, MEM_KERNEL)) == -1) { nuclear@43: panic("clone_vmem: failed to allocate page table page\n"); nuclear@43: } nuclear@43: ntbl = (uint32_t*)PAGE_TO_ADDR(tblpg); nuclear@43: nuclear@43: /* we will allocate physical pages and map them to this virtual page nuclear@43: * as needed in the loop below. nuclear@43: */ nuclear@49: free_phys_page(virt_to_phys((uint32_t)ntbl)); nuclear@43: nuclear@48: kstart_dirent = ADDR_TO_PAGE(KMEM_START) / 1024; nuclear@47: nuclear@47: /* user space */ nuclear@48: for(i=0; istart > last) { nuclear@23: printf(" vm-used: %x -> %x\n", PAGE_TO_ADDR(last), PAGE_TO_ADDR(node->start)); nuclear@23: } nuclear@23: nuclear@23: printf(" vm-free: %x -> ", PAGE_TO_ADDR(node->start)); nuclear@23: if(node->end >= PAGE_COUNT) { nuclear@23: printf("END\n"); nuclear@23: } else { nuclear@23: printf("%x\n", PAGE_TO_ADDR(node->end)); nuclear@23: } nuclear@23: nuclear@23: last = node->end; nuclear@23: node = node->next; nuclear@23: } nuclear@25: nuclear@25: set_intr_state(intr_state); nuclear@23: }