linux內(nèi)存管理之非連續(xù)物理地址分配(vmalloc)
前面我們已經(jīng)分析了linux如何利用伙伴系統(tǒng),slab分配器分配內(nèi)存,用這些方法得到的內(nèi)存在物理地址上都是連續(xù)的,然而,有些時(shí)候,每次請(qǐng)求內(nèi)存時(shí),系統(tǒng)都分配物理地址連續(xù)的內(nèi)存塊是不合適的,可以利用小塊內(nèi)存“連接”成大塊可使用的內(nèi)存.這在操作系統(tǒng)設(shè)計(jì)中也被稱為 “內(nèi)存拼接”,顯然,內(nèi)存拼接在需要較大內(nèi)存,而內(nèi)存訪問相比之下不是很頻繁的情況下是比較有效的.
在linux內(nèi)核中用來管理內(nèi)存拼接的接口是vmalloc/vfree.用vmalloc分配得到的內(nèi)存在線性地址是平滑的,但是物理地址上是非連續(xù)的.
一:準(zhǔn)備知識(shí):
Linux用vm_struct結(jié)構(gòu)來表示vmalloc使用的線性地址.vmalloc所使用的線性地址區(qū)間為: VMALLOC_START VMALLOC_END.借用<
從上圖中我們可以看到每一個(gè)vmalloc_area用4KB隔開,這樣做是為了很容易就能捕捉到越界訪問,因?yàn)橹虚g是一個(gè) “空洞”.
二:相關(guān)的數(shù)據(jù)結(jié)構(gòu)
下面來分析一下vmalloc area的數(shù)據(jù)結(jié)構(gòu):
struct vm_struct {
void *addr; //虛擬地址
unsigned long size; //vm的大小
unsigned long flags;//vm的標(biāo)志
struct page**pages; //vm所映射的page
unsigned int nr_pages; //page個(gè)數(shù)
unsigned long phys_addr;//對(duì)應(yīng)的起始物理地址
struct vm_struct *next;//下一個(gè)vm.用來形成鏈表
}
全局變量vmlist用來管理vm構(gòu)成的鏈表
全局變量vmlist用于訪問vmlist所使用的信號(hào)量
對(duì)于vm_struct有兩個(gè)常用的操作: get_vm_area/remove_vm_area
get_vm_area:用來分配一個(gè)合適大小的vm結(jié)構(gòu),分配成功之后,將其鏈入到vmlist中,代碼在 mm/vmalloc.c中.如下示:
//size為vm的大小
struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
{
//在VMALLOC_START與VMALLOC_END找到一段合適的空間
return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
}
//參數(shù)說明:
//start:起始地址 end:結(jié)束地址 size 空間大小
struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
unsigned long start, unsigned long end)
{
struct vm_struct **p, *tmp, *area;
unsigned long align = 1;
unsigned long addr;
//如果指定了VM_IOREMAP.則調(diào)整對(duì)齊因子
if (flags & VM_IOREMAP) {
int bit = fls(size);
if (bit > IOREMAP_MAX_ORDER)
bit = IOREMAP_MAX_ORDER;
else if (bit < PAGE_SHIFT)
bit = PAGE_SHIFT;
align = 1ul << bit;
}
//將起始地址按照對(duì)齊因子對(duì)齊
addr = ALIGN(start, align);
#p#//分配一個(gè)vm_struct結(jié)構(gòu)空間
area = kmalloc(sizeof(*area), GFP_KERNEL);
if (unlikely(!area))
return NULL;
//PAGE_SIZE:在i32中為4KB,即上面所說的間隔空洞
size += PAGE_SIZE;
if (unlikely(!size)) {
kfree (area);
return NULL;
}
write_lock(&vmlist_lock);
//遍歷vmlist:找到合適大小的末使用空間
for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
//若起始地址落在某一個(gè)vm區(qū)間,則調(diào)整起始地址為vm區(qū)間的末尾
if ((unsigned long)tmp->addr < addr) {
if((unsigned long)tmp->addr + tmp->size >= addr)
addr = ALIGN(tmp->size +
(unsigned long)tmp->addr, align);
continue;
}
//size+addr < addr ?除非size == 0
if ((size + addr) < addr)
goto out;
//中間的空隙可以容納下size大小的vm.說明已經(jīng)找到了這樣的一個(gè)vm
if (size + addr <= (unsigned long)tmp->addr)
goto found;
//調(diào)整起始地址為vm的結(jié)束地址
addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
//如果超出了范圍
if (addr > end - size)
goto out;
}
found:
//找到了合適大小的空間,將area->addr賦值為addr,然后鏈入vmlist中
area->next = *p;
*p = area;
area->flags = flags;
area->addr = (void *)addr;
area->size = size;
area->pages = NULL;
area->nr_pages = 0;
area->phys_addr = 0;
write_unlock(&vmlist_lock);
return area;
out:
//沒有找到合適大小的空間,出錯(cuò)返回
write_unlock(&vmlist_lock);
kfree(area);
if (printk_ratelimit())
printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=
return NULL;
}
這段代碼不是很復(fù)雜,在此不詳細(xì)分析了.
remove_vm_area用來將相應(yīng)的vm從vmlist中斷開,使其表示的空間可以被利用
//addr:對(duì)應(yīng)vm的超始地址
struct vm_struct *remove_vm_area(void *addr)
{
struct vm_struct **p, *tmp;
write_lock(&vmlist_lock);
//遍歷vmlist.找到超始地址為addr的vm
for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
if (tmp->addr == addr)
goto found;
}
write_unlock(&vmlist_lock);
return NULL;
found:
//斷開tmp所對(duì)應(yīng)的映射關(guān)系
unmap_vm_area(tmp);
//找到了這個(gè)vm,將其從vmlist上斷開
*p = tmp->next;
write_unlock(&vmlist_lock);
return tmp;
}
unmap_vm_area用來斷開vm所在線性地址所對(duì)應(yīng)的映射關(guān)系.它的代碼如下:
void unmap_vm_area(struct vm_struct *area)
{
//vm所對(duì)應(yīng)的起始線性地址
unsigned long address = (unsigned long) area->addr;
//vm所對(duì)應(yīng)的結(jié)束線性地址
unsigned long end = (address + area->size);
pgd_t *dir;
//起始地址所在的內(nèi)核頁(yè)目錄項(xiàng)
dir = pgd_offset_k(address);
flush_cache_vunmap(address, end);
do {
//斷開地址所對(duì)應(yīng)的pmd映射
unmap_area_pmd(dir, address, end - address);
#p#//運(yùn)行到這里的時(shí)候,已經(jīng)斷開了一個(gè)頁(yè)目錄所表示的線性地址,而每個(gè)頁(yè)目錄表示的線性地址//大小為PGDIR_SIZE
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
//當(dāng)?shù)竭_(dá)末尾時(shí)結(jié)束循環(huán)
flush_tlb_kernel_range((unsigned long) area->addr, end);
}
//斷開線性地址區(qū)間所在的pmd的映射
static void unmap_area_pmd(pgd_t *dir, unsigned long address,
unsigned long size)
{
unsigned long end;
pmd_t *pmd;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
//斷開線性地址所在的pte的映射關(guān)系
unmap_area_pte(pmd, address, end - address);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
}
static void unmap_area_pte(pmd_t *pmd, unsigned long address,
unsigned long size)
{
unsigned long end;
pte_t *pte;
if (pmd_none(*pmd))
return;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
return;
}
pte = pte_offset_kernel(pmd, address);
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
end = PMD_SIZE;
do {
pte_t page;
//清除pte的對(duì)應(yīng)映射關(guān)系
page = ptep_get_and_clear(pte);
address += PAGE_SIZE;
pte++;
if (pte_none(page))
continue;
if (pte_present(page))
continue;
printk(KERN_CRIT "Whee.. Swapped out page in kernel page table\n");
} while (address < end);
}
經(jīng)過這幾個(gè)過程之后,實(shí)際上,它只是找到線性地址所對(duì)應(yīng)的pte,然后斷開pte的映射.值得注意的是:為了效率起見,這里只是斷開了pte的映射,即只是將pte置為none,表示pte末映射內(nèi)存.并末斷開pmd和pgd的映射
三:vmalloc的實(shí)現(xiàn):
void *vmalloc(unsigned long size)
{
return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
}
實(shí)際上調(diào)用__vmalloc:
void *__vmalloc(unsigned long size, int gfp_mask, pgprot_t prot)
{
struct vm_struct *area;
struct page **pages;
unsigned int nr_pages, array_size, i;
//使請(qǐng)求的大小與頁(yè)框?qū)R
size = PAGE_ALIGN(size);
//有效性檢查
if (!size || (size >> PAGE_SHIFT) > num_physpages)
return NULL;
#p#//取得一個(gè)有效的VM,這個(gè)函數(shù)我們?cè)谇懊嬉呀?jīng)詳細(xì)的分析過了
area = get_vm_area(size, VM_ALLOC);
if (!area)
return NULL;
//所要映射的頁(yè)面總數(shù)
nr_pages = size >> PAGE_SHIFT;
//頁(yè)面描述符所占的空間
array_size = (nr_pages * sizeof(struct page *));
area->nr_pages = nr_pages;
area->pages = pages = kmalloc(array_size, (gfp_mask & ~__GFP_HIGHMEM));
//如果空間分配失敗
if (!area->pages) {
remove_vm_area(area->addr);
kfree(area);
return NULL;
}
memset(area->pages, 0, array_size);
//為每一個(gè)頁(yè)面分配空間
for (i = 0; i < area->nr_pages; i++) {
area->pages[i] = alloc_page(gfp_mask);
if (unlikely(!area->pages[i])) {
/* Successfully allocated i pages, free them in __vunmap() */
area->nr_pages = i;
goto fail;
}
}
//為所分配的頁(yè)面建立映射關(guān)系
if (map_vm_area(area, prot, &pages))
goto fail;
return area->addr;
fail:
vfree(area->addr);
return NULL;
}
map_vm_area為所分配的內(nèi)存建立映射關(guān)系,它的程序流程與unmap_vm_area差不多,都是從pgd找到pte,如果同樣的映射關(guān)系不存在,則新建之.(如:pgd對(duì)應(yīng)的pmd不存在,則新建pmd項(xiàng),使pgd指向建好的pmd.同理,如果pmd所映射的pte項(xiàng)不存在,則新建pte,然后建立映射),然后將pte映射到相應(yīng)的頁(yè)表.代碼如下:
int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
{
unsigned long address = (unsigned long) area->addr;
unsigned long end = address + (area->size-PAGE_SIZE);
pgd_t *dir;
int err = 0;
//vm 起始地址所在的頁(yè)目錄
dir = pgd_offset_k(address);
spin_lock(&init_mm.page_table_lock);
do {
pmd_t *pmd = pmd_alloc(&init_mm, dir, address);
if (!pmd) {
err = -ENOMEM;
break;
}
//輪到pmd了 ^_^
if (map_area_pmd(pmd, address, end - address, prot, pages)) {
err = -ENOMEM;
break;
}
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
spin_unlock(&init_mm.page_table_lock);
flush_cache_vmap((unsigned long) area->addr, end);
return err;
}
static int map_area_pmd(pmd_t *pmd, unsigned long address,
unsigned long size, pgprot_t prot,
struct page ***pages)
{
unsigned long base, end;
base = address & PGDIR_MASK;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
pte_t * pte = pte_alloc_kernel(&init_mm, pmd, base + address);
if (!pte)
return -ENOMEM;
//輪到pte了 ^_^
if (map_area_pte(pte, address, end - address, prot, pages))
return -ENOMEM;
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return 0;
}
//為頁(yè)表頁(yè)建立映射關(guān)系
static int map_area_pte(pte_t *pte, unsigned long address,
unsigned long size, pgprot_t prot,
struct page ***pages)
{
unsigned long end;
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
end = PMD_SIZE;
do {
struct page *page = **pages;
WARN_ON(!pte_none(*pte));
if (!page)
return -ENOMEM;
//具體的映射在這里了 ^_^
set_pte(pte, mk_pte(page, prot));
address += PAGE_SIZE;
pte++;
(*pages)++;
} while (address < end);
return 0;
}
只要理解了斷開映射的過程,這段代碼是很好理解的.
總而言之:linux在建立映射的時(shí)候,從pgd 到pte相應(yīng)的建立映射關(guān)系,最后將pte映射到分配得到的物理內(nèi)存．而在斷開映射的時(shí)候，linux內(nèi)核從pgd找到pte,然后將pte置為none,表示pte末建立映射關(guān)系.
四:vfree的實(shí)現(xiàn):
代碼如下:
void vfree(void *addr)
{
BUG_ON(in_interrupt());
__vunmap(addr, 1);
}
跟蹤至__vunmap:
void __vunmap(void *addr, int deallocate_pages)
{
struct vm_struct *area;
//參數(shù)有效性檢查
if (!addr)
return;
//判斷addr是否是按頁(yè)框?qū)R的
if ((PAGE_SIZE-1) & (unsigned long)addr) {
printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
WARN_ON(1);
return;
}
//remove_vm_area:這個(gè)函數(shù)我們?cè)谥耙呀?jīng)分析過了 ^_^
area = remove_vm_area(addr);
if (unlikely(!area)) {
//沒有找到起始地址為addr的vm.則無效,退出
printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
addr);
WARN_ON(1);
return;
}
if (deallocate_pages) {
int i;
for (i = 0; i < area->nr_pages; i++) {
if (unlikely(!area->pages[i]))
BUG();
//釋放請(qǐng)求獲得的頁(yè)面
__free_page(area->pages[i]);
}
//釋放分配的page 描述符
kfree(area->pages);
}
//釋放內(nèi)核的vm 描述符
kfree(area);
return;
}
五:總結(jié)
經(jīng)過上面的分析，我們可以看到，vmalloc分配內(nèi)存的過程是十分低效的，不僅要從伙伴系統(tǒng)中取內(nèi)存而且要建立映射關(guān)系，顯然，用vmalloc分配較小的內(nèi)存是不合算的。此外。有個(gè)問題值得思考一下：為什么用__get_free_page不需要建立映射關(guān)系，而vmalloc就需要呢？
其實(shí)，不管使用何種方式。線性地址到物理地址的轉(zhuǎn)換最終都要經(jīng)過硬件的頁(yè)式管理去完成。所不同的是__get_free_page返回的線性地址是屬于（PAGE_OFFSET,HIGH_MEMORY）之間的，這段線性地址在內(nèi)核初始化的時(shí)候就完成了映射。而vmalloc使用的線性地址是屬于（VMALLOC_START VMALLOC_END）之間的，也就是說屬于一個(gè)臨時(shí)映射區(qū)，所以必須為其建立映射關(guān)系。
【編輯推薦】
