Lab8 Locks
Memory allocator
kalloctest会对Xv6的内存分配进行压力测试。目前的版本会产生大量的锁争用，其原因是kalloc()使用一个空闲链表，并由单个锁进行保护。
本实验的任务是重新设计内存分配器：为每个 CPU 维护一个单独的空闲链表，每个链表都有自己的锁。不同 CPU 上的分配和释放操作可以并行进行。主要挑战在于处理以下情况：当一个 CPU 的空闲链表为空，而另一个 CPU 的链表仍有可用内存时，该 CPU 必须“窃取”部分其他 CPU 的空闲链表。
首先修改kmem结构体，设置为NCPU大小的数组。同时为了符合initlock的初始化，为每个锁命名。
//kernel/kalloc.c
struct {
struct spinlock lock;
struct run *freelist;
} kmem[NCPU];//每个CPU都有自己的空闲内存链表
//NCPU值为8
char* kmem_lockname[] = {
"kmem_cpu0",
"kmem_cpu1",
"kmem_cpu2",
"kmem_cpu3",
"kmem_cpu4",
"kmem_cpu5",
"kmem_cpu6",
"kmem_cpu7",
};
初始化
void
kinit()
{
for(int i = 0; i < NCPU; ++i){
initlock(&kmem[i].lock, kmem_lockname[i]);
}
freerange(end, (void*)PHYSTOP);
}
释放内存
void
kfree(void *pa)
{
struct run *r;
if(((uint64)pa % PGSIZE) != 0 || (char*)pa < end || (uint64)pa >= PHYSTOP)
panic("kfree");
// Fill with junk to catch dangling refs.
memset(pa, 1, PGSIZE);
r = (struct run*)pa;
//先关中断在获取cpuid
//防止出现cpu不一致的问题
push_off();
int cpu = cpuid();
acquire(&kmem[cpu].lock);
r->next = kmem[cpu].freelist;
kmem[cpu].freelist = r;
release(&kmem[cpu].lock);
pop_off();//开中断
}
分配内存
void *
kalloc(void)
{
struct run *r;
push_off();
int cpu = cpuid();
acquire(&kmem[cpu].lock);
r = kmem[cpu].freelist;
if(r == 0){ //当前cpu无freelist
//这个地方比较坑，最开始用的是
//for( int newcpu = (cpu + 1) % NCPU; newcpu != cpu; newcpu++)
//这时Xv6启动时会卡死，后面发现是分配了两个cpu，然后在启动时互相偷页面导致死锁
for( int newcpu = 0; newcpu < NCPU; newcpu++){
if(newcpu == cpu)
continue;
acquire(&kmem[newcpu].lock);
r = kmem[newcpu].freelist;
if(r == 0){
release(&kmem[newcpu].lock);
continue;
}
kmem[newcpu].freelist = r->next;
r->next = kmem[cpu].freelist;
kmem[cpu].freelist = r;
release(&kmem[newcpu].lock);
break;
}
}
if(r){
kmem[cpu].freelist = r->next;
}
release(&kmem[cpu].lock);
pop_off();
if(r)
memset((char*)r, 5, PGSIZE); // fill with junk
return (void*)r;
}
运行kalloctest和usertest，顺利通过。
Buffer cache
如果多个进程密集使用文件系统，它们很可能会争夺保护内核中磁盘块缓存的 bcache.lock 锁(kernel/bio.c)。目的是不允许多个进程同时操作磁盘块缓存。目前块缓存的设计是使用双向链表存储，链表结点按LRU的思想组织。
//kernel/bio.c
struct {
struct spinlock lock;
struct buf buf[NBUF];//NBUF = 30
// Linked list of all buffers, through prev/next.
// Sorted by how recently the buffer was used.
// head.next is most recent, head.prev is least.
struct buf head;
} bcache;
//kernel/buf.h
struct buf {
int valid; // has data been read from disk?
int disk; // does disk "own" buf?
uint dev;
uint blockno;
struct sleeplock lock;
uint refcnt;
struct buf *prev; // LRU cache list
struct buf *next;
uchar data[BSIZE];//BSIZE = 1024
};
先来看看现在获取buf的代码
static struct buf*
bget(uint dev, uint blockno)
{
struct buf *b;
acquire(&bcache.lock);//锁住整个缓存区
// Is the block already cached?
//在缓冲区中根据dev和blockno查找对应的块是否已在缓存区中
for(b = bcache.head.next; b != &bcache.head; b = b->next){
//在缓冲区中
if(b->dev == dev && b->blockno == blockno){
b->refcnt++;//引用计数+1
release(&bcache.lock);//释放缓存区锁
acquiresleep(&b->lock);//对对应的块加睡眠锁
return b;//返回
}
}
// Not cached.
// Recycle the least recently used (LRU) unused buffer.
//不在
//从后往前遍历缓存区：即优先选择最近最久未被使用的那些buf
for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
//选引用计数为0的buf
if(b->refcnt == 0) {
//修改相关信息
b->dev = dev;
b->blockno = blockno;
b->valid = 0;
b->refcnt = 1;
release(&bcache.lock);
acquiresleep(&b->lock);
return b;
}
}
panic("bget: no buffers");
}
可以看到，每次有进程来尝试获取buf都会给整个缓存区上锁，因此多个进程会发生严重的锁竞争。
根据实验提示，采用哈希桶+时间戳的方式进行优化：
- 根据 dev 和 blockno 进行哈希映射到某一个哈希桶，每个哈希桶中维护独立的buf链表。这样就将一个锁变为了哈希桶个数的锁，降低了锁冲突；
- 当某一个桶中没有空闲buf时，全局遍历所有桶中的buf，寻找 refcnt为0 且 时间戳最小 的buf使用。将这个buf移除所在的那个桶并加入新桶
在写这部分实验的时候经常出现panic。通过搜索相关博客和问AI最后才解决，参考博客
https://blog.miigon.net/posts/s081-lab8-locks/#buffer-cache-hard
修改数据结构
//kernel/bio.c
#define NBUCKET 13
#define HASH2BUCKET(dev, blockno) ((blockno + dev) % NBUCKET)
struct {
// struct spinlock lock;
struct buf buf[NBUF];
struct spinlock eviction_lock;
struct buf bufmap[NBUCKET];//13个桶
struct spinlock bufmap_locks[NBUCKET];
} bcache;
//kernel/buf.h
struct buf {
...
//struct buf *prev; // LRU cache list
struct buf *next;
uint timestamp;//时间戳属性
uchar data[BSIZE];
};
相关操作
初始化
void
binit(void)
{
struct buf *b = 0;
for(int i = 0; i < NBUCKET; ++i){
initlock(&bcache.bufmap_locks[i], "bache");
bcache.bufmap[i].next = 0;
}
for(int i = 0; i < NBUF; ++i){
b = &bcache.buf[i];
initsleeplock(&b->lock, "buffer");
//初始时系统中所有buf都在0号桶中
b->next = bcache.bufmap[0].next;
bcache.bufmap[0].next = b;
}
initlock(&bcache.eviction_lock,"bache_eviction");
}
获取缓冲区块
static struct buf*
bget(uint dev, uint blockno)
{
struct buf *b;
uint bucket_no = HASH2BUCKET(dev, blockno);
//先查看block_n0的缓冲区是否在缓冲区中
acquire(&bcache.bufmap_locks[bucket_no]);
for(b = bcache.bufmap[bucket_no].next; b != 0; b = b->next){
if(b->dev == dev && b->blockno == blockno){
b->refcnt++;
release(&bcache.bufmap_locks[bucket_no]);
acquiresleep(&b->lock);
return b;
}
}
release(&bcache.bufmap_locks[bucket_no]);
//这里的问题在于当线程1查询发现不存在buf释放锁后，另一个线程在这期间可能创建了这个block_no的buf
//这就导致同一个block_no有两个缓存块
//解决方案是：加一个大锁 和 先检查在尝试
acquire(&bcache.eviction_lock);//加大锁
//先检查没有确保在释放block_no锁和加大锁期间没有创建buf
for(b = bcache.bufmap[bucket_no].next; b != 0; b = b->next){
if(b->dev == dev && b->blockno == blockno){
acquire(&bcache.bufmap_locks[bucket_no]);
b->refcnt++;
release(&bcache.bufmap_locks[bucket_no]);
release(&bcache.eviction_lock);
acquiresleep(&b->lock);
return b;
}
}
//确定没有创建buf，此时持有bcache.eviction_lock
struct buf* lru_buf = 0;
struct buf* pre_lru_buf = 0;
int lru_buf_bucket = -1;
uint min_timestamp = __UINT32_MAX__;
for (uint newbucket = 0; newbucket < NBUCKET; newbucket++){
acquire(&bcache.bufmap_locks[newbucket]);
int newflag = 0;
struct buf* prev = &bcache.bufmap[newbucket];
for(b = prev->next; b != 0; prev = b, b = b->next){
if(b->refcnt == 0 && b->timestamp < min_timestamp) {
lru_buf = b;
pre_lru_buf = prev;
newflag = 1;
}
}
if( !newflag )
release(&bcache.bufmap_locks[newbucket]);
else{
if( lru_buf_bucket != -1)
release(&bcache.bufmap_locks[lru_buf_bucket]);
lru_buf_bucket = newbucket;
}
}
if(!lru_buf)
panic("bget: no buffers");
if(lru_buf_bucket != bucket_no) {
//从原桶移除
pre_lru_buf->next = lru_buf->next;
release(&bcache.bufmap_locks[lru_buf_bucket]);
// 添加到目标桶
acquire(&bcache.bufmap_locks[bucket_no]);
lru_buf->next = bcache.bufmap[bucket_no].next;
bcache.bufmap[bucket_no].next = lru_buf;
}
lru_buf->dev = dev;
lru_buf->blockno = blockno;
lru_buf->refcnt = 1;
lru_buf->valid = 0;
release(&bcache.bufmap_locks[bucket_no]);
release(&bcache.eviction_lock);
acquiresleep(&lru_buf->lock);
return lru_buf;
}
释放
void
brelse(struct buf *b)
{
if(!holdingsleep(&b->lock))
panic("brelse");
releasesleep(&b->lock);
uint bucket_no = HASH2BUCKET(b->dev, b->blockno);
acquire(&bcache.bufmap_locks[bucket_no]);
b->refcnt--;
if (b->refcnt == 0) {
b->timestamp = ticks;
}
release(&bcache.bufmap_locks[bucket_no]);
}
pin与upin
void
bpin(struct buf *b) {
uint bucket_no = HASH2BUCKET(b->dev, b->blockno);
acquire(&bcache.bufmap_locks[bucket_no]);
b->refcnt++;
release(&bcache.bufmap_locks[bucket_no]);
}
void
bunpin(struct buf *b) {
uint bucket_no = HASH2BUCKET(b->dev, b->blockno);
acquire(&bcache.bufmap_locks[bucket_no]);
b->refcnt--;
release(&bcache.bufmap_locks[bucket_no]);
}
已能通过bcachetest和 usertests
$ bcachetest
start test0
test0 results:
--- lock kmem/bcache stats
lock: kmem_cpu0: #fetch-and-add 0 #acquire() 32959
lock: kmem_cpu1: #fetch-and-add 0 #acquire() 35
lock: kmem_cpu2: #fetch-and-add 0 #acquire() 104
--- top 5 contended locks:
lock: virtio_disk: #fetch-and-add 177766 #acquire() 2345
lock: proc: #fetch-and-add 83195 #acquire() 85478
lock: proc: #fetch-and-add 80024 #acquire() 85478
lock: proc: #fetch-and-add 68162 #acquire() 85500
lock: proc: #fetch-and-add 59295 #acquire() 86196
tot= 0
test0: OK
start test1
test1 OK
浙公网安备 33010602011771号