From 365c3523a258eb362ae8b81b19ba37eeee0e8954 Mon Sep 17 00:00:00 2001 From: yuchen Date: Wed, 13 May 2015 11:24:26 +0800 Subject: [PATCH] add lec21 spoc discuss: about simple file system --- related_info/lab8/sfs-homework.md | 112 +++++++++ related_info/lab8/sfs-homework.py | 470 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 582 insertions(+) create mode 100644 related_info/lab8/sfs-homework.md create mode 100755 related_info/lab8/sfs-homework.py diff --git a/related_info/lab8/sfs-homework.md b/related_info/lab8/sfs-homework.md new file mode 100644 index 0000000..7cee8ef --- /dev/null +++ b/related_info/lab8/sfs-homework.md @@ -0,0 +1,112 @@ +# 基于简单文件系统（Simple File System）模拟环境，理解文件系统的基本实现 + +## 面向文件系统的用户操作 +- mkdir() - 创建一个新目录 +- creat() - 创建一个空文件 +- open(), write(), close() - 对文件写一个数据buffer +- link() - 对文件创建一个硬链接（hard link） +- unlink() - 对文件取消一个硬链接 (如果文件的链接数为0，则删除文件 + + +## disk filesystem的内部组织和关键数据结构 + +### disk filesystem的内部组织 +- superblock : 可用inode数量，可用data block数量 +- inode bitmap ： inode的分配图（基于bitmap） +- inodes ： inode的存储区域 +- data bitmap ： data block的分配图（基于bitmap） +- data ： data block的存储区域 + +> bitmap: 0表示inode/data block是free， 1表示inode/data block是allocated + +### 关键数据结构 + +#### inode数据结构 + - inode : 包含3个fields, 用python list 表示 + - file type: f -> 常规文件：regular file, d -> 目录文件：directory + - data block addr of file content: -1 -> file is empty + - reference count: file/directory的引用计数 + +> 比如 刚创建的一个空文件inode： `[f a:-1 r:1]`， 一个有1个硬链接的文件inode `[f a:10 r:2]` + + +#### 数据块内容结构 + - 一般文件的内容的表示：只是包含单个字符的list，比如`['a']`, `['b']` ..... + - 目录内容的表示： 多个两元组`（name, inode_number）`形成的list，比如， 根目录 `[(.,0) (..,0)]`， 或者包含了一个`'f'`文件的根目录[(.,0) (..,0) (f,1)] + + +### 完整文件系统的例子 +``` +fs.ibitmap: inode bitmap 11110000 +fs.inodes: [d a:0 r:6] [f a:1 r:1] [f a:-1 r:1] [d a:2 r:2] [] ... +fs.dbitmap: data bitmap 11100000 +fs.data: [(.,0) (..,0) (y,1) (z,2) (x,3)] [u] [(.,3) (..,0)] [] ... +``` + +> 此文件系统有8个inode, 8个data blocks. 其中，根目录包含5个目录项，`”.“，”..“，”y“，”z“，”x“`, ”y“是常规文件,并有文件内容，包含一个data block，文件内容为”u“。”z“是一个空的常规文件。”x“是一个目录文件，是空目录。 + +### 辅助数据结构 +也可理解为内存中的文件系统相关数据结构 + +- fs.files :当前文件系统中的常规文件list +- fs.dirs : 当前文件系统中的目录文件list +- fs.nameToInum ： 文件名:inode_num的对应关系 + +## 文件系统执行流程 + +### 文件系统初始化 + +第一步：格式化sfs文件系统 +``` + self.numInodes = numInodes + self.numData = numData + + self.ibitmap = bitmap(self.numInodes) + self.inodes = [] + for i in range(self.numInodes): + self.inodes.append(inode()) + + self.dbitmap = bitmap(self.numData) + self.data = [] + for i in range(self.numData): + self.data.append(block('free')) +``` + +第二步：创建sfs文件系统的根目录 + +``` + self.ibitmap.markAllocated(self.ROOT) + self.inodes[self.ROOT].setAll('d', 0, 2) + self.dbitmap.markAllocated(self.ROOT) + self.data[0].setType('d') + self.data[0].addDirEntry('.', self.ROOT) + self.data[0].addDirEntry('..', self.ROOT) +``` + +第三步：在内存中保存相关数据 +``` + self.files = [] + self.dirs = ['/'] + self.nameToInum = {'/':self.ROOT} +``` + + +第四步：随机生成文件相关的操作，改变sfs文件系统的内容 + - doAppend: + - `fd=open(filename, O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd);` + - doDelete: + - `unlink()filename)` + - doLink + - `link()targetfile, sourcefile)` + - doCreate + - `create(filename)` OR `mkdir(dirname)` + + +## 问题1： +根据文件系统的状态变化信息，给出具体的文件相关操作内容 + +## 问题2： +实现 `writeFile, createFile, createLink, deleteFile`，使得你的实现能够达到与问题1的正确结果一致 + +## 问题3： +实现`soft link`机制，并设计测试用例说明你实现的正确性。 diff --git a/related_info/lab8/sfs-homework.py b/related_info/lab8/sfs-homework.py new file mode 100755 index 0000000..2836251 --- /dev/null +++ b/related_info/lab8/sfs-homework.py @@ -0,0 +1,470 @@ +#! /usr/bin/env python + +import random +from optparse import OptionParser + +DEBUG = False + +def dprint(str): + if DEBUG: + print str + +printOps = True +printState = True +printFinal = True + +class bitmap: + def __init__(self, size): + self.size = size + self.bmap = [] + for num in range(size): + self.bmap.append(0) + + def alloc(self): + for num in range(len(self.bmap)): + if self.bmap[num] == 0: + self.bmap[num] = 1 + return num + return -1 + + def free(self, num): + assert(self.bmap[num] == 1) + self.bmap[num] = 0 + + def markAllocated(self, num): + assert(self.bmap[num] == 0) + self.bmap[num] = 1 + + def dump(self): + s = '' + for i in range(len(self.bmap)): + s += str(self.bmap[i]) + return s + +class block: + def __init__(self, ftype): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + # only for directories, properly a subclass but who cares + self.dirUsed = 0 + self.maxUsed = 32 + self.dirList = [] + self.data = '' + + def dump(self): + if self.ftype == 'free': + return '[]' + elif self.ftype == 'd': + rc = '' + for d in self.dirList: + # d is of the form ('name', inum) + short = '(%s,%s)' % (d[0], d[1]) + if rc == '': + rc = short + else: + rc += ' ' + short + return '['+rc+']' + # return '%s' % self.dirList + else: + return '[%s]' % self.data + + def setType(self, ftype): + assert(self.ftype == 'free') + self.ftype = ftype + + def addData(self, data): + assert(self.ftype == 'f') + self.data = data + + def getNumEntries(self): + assert(self.ftype == 'd') + return self.dirUsed + + def getFreeEntries(self): + assert(self.ftype == 'd') + return self.maxUsed - self.dirUsed + + def getEntry(self, num): + assert(self.ftype == 'd') + assert(num < self.dirUsed) + return self.dirList[num] + + def addDirEntry(self, name, inum): + assert(self.ftype == 'd') + self.dirList.append((name, inum)) + self.dirUsed += 1 + assert(self.dirUsed <= self.maxUsed) + + def delDirEntry(self, name): + assert(self.ftype == 'd') + tname = name.split('/') + dname = tname[len(tname) - 1] + for i in range(len(self.dirList)): + if self.dirList[i][0] == dname: + self.dirList.pop(i) + self.dirUsed -= 1 + return + assert(1 == 0) + + def dirEntryExists(self, name): + assert(self.ftype == 'd') + for d in self.dirList: + if name == d[0]: + return True + return False + + def free(self): + assert(self.ftype != 'free') + if self.ftype == 'd': + # check for only dot, dotdot here + assert(self.dirUsed == 2) + self.dirUsed = 0 + self.data = '' + self.ftype = 'free' + +class inode: + def __init__(self, ftype='free', addr=-1, refCnt=1): + self.setAll(ftype, addr, refCnt) + + def setAll(self, ftype, addr, refCnt): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + self.addr = addr + self.refCnt = refCnt + + def incRefCnt(self): + self.refCnt += 1 + + def decRefCnt(self): + self.refCnt -= 1 + + def getRefCnt(self): + return self.refCnt + + def setType(self, ftype): + assert(ftype == 'd' or ftype == 'f' or ftype == 'free') + self.ftype = ftype + + def setAddr(self, block): + self.addr = block + + def getSize(self): + if self.addr == -1: + return 0 + else: + return 1 + + def getAddr(self): + return self.addr + + def getType(self): + return self.ftype + + def free(self): + self.ftype = 'free' + self.addr = -1 + + +class fs: + def __init__(self, numInodes, numData): + self.numInodes = numInodes + self.numData = numData + + self.ibitmap = bitmap(self.numInodes) + self.inodes = [] + for i in range(self.numInodes): + self.inodes.append(inode()) + + self.dbitmap = bitmap(self.numData) + self.data = [] + for i in range(self.numData): + self.data.append(block('free')) + + # root inode + self.ROOT = 0 + + # create root directory + self.ibitmap.markAllocated(self.ROOT) + self.inodes[self.ROOT].setAll('d', 0, 2) + self.dbitmap.markAllocated(self.ROOT) + self.data[0].setType('d') + self.data[0].addDirEntry('.', self.ROOT) + self.data[0].addDirEntry('..', self.ROOT) + + # these is just for the fake workload generator + self.files = [] + self.dirs = ['/'] + self.nameToInum = {'/':self.ROOT} + + def dump(self): + print 'inode bitmap ', self.ibitmap.dump() + print 'inodes ', + for i in range(0,self.numInodes): + ftype = self.inodes[i].getType() + if ftype == 'free': + print '[]', + else: + print '[%s a:%s r:%d]' % (ftype, self.inodes[i].getAddr(), self.inodes[i].getRefCnt()), + print '' + print 'data bitmap ', self.dbitmap.dump() + print 'data ', + for i in range(self.numData): + print self.data[i].dump(), + print '' + + def makeName(self): + p = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] + return p[int(random.random() * len(p))] + + def inodeAlloc(self): + return self.ibitmap.alloc() + + def inodeFree(self, inum): + self.ibitmap.free(inum) + self.inodes[inum].free() + + def dataAlloc(self): + return self.dbitmap.alloc() + + def dataFree(self, bnum): + self.dbitmap.free(bnum) + self.data[bnum].free() + + def getParent(self, name): + tmp = name.split('/') + if len(tmp) == 2: + return '/' + pname = '' + for i in range(1, len(tmp)-1): + pname = pname + '/' + tmp[i] + return pname + + def deleteFile(self, tfile): + if printOps: + print 'unlink("%s");' % tfile + + inum = self.nameToInum[tfile] + + # YOUR CODE, YOUR ID + # IF inode.refcnt ==1, THEN free data blocks first, then free inode, ELSE dec indoe.refcnt + # remove from parent directory: delete from parent inum, delete from parent addr + # DONE + + # finally, remove from files list + self.files.remove(tfile) + return 0 + + def createLink(self, target, newfile, parent): + # YOUR CODE, YOUR ID + # find info about parent + # is there room in the parent directory? + # if the newfile was already in parent dir? + # now, find inumber of target + # inc parent ref count + # now add to directory + # DONE + return tinum + + def createFile(self, parent, newfile, ftype): + # YOUR CODE, YOUR ID + # find info about parent + # is there room in the parent directory? + # have to make sure file name is unique + # find free inode + # if a directory, have to allocate directory block for basic (., ..) info + # now ok to init inode properly + # inc parent ref count + # and add to directory of parent + # DONE + return inum + + def writeFile(self, tfile, data): + inum = self.nameToInum[tfile] + curSize = self.inodes[inum].getSize() + dprint('writeFile: inum:%d cursize:%d refcnt:%d' % (inum, curSize, self.inodes[inum].getRefCnt())) + + # YOUR CODE, YOUR ID + # file is full? + # no data blocks left + # write file data + # DONE + + if printOps: + print 'fd=open("%s", O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd);' % tfile + return 0 + + def doDelete(self): + dprint('doDelete') + if len(self.files) == 0: + return -1 + dfile = self.files[int(random.random() * len(self.files))] + dprint('try delete(%s)' % dfile) + return self.deleteFile(dfile) + + def doLink(self): + dprint('doLink') + if len(self.files) == 0: + return -1 + parent = self.dirs[int(random.random() * len(self.dirs))] + nfile = self.makeName() + + # pick random target + target = self.files[int(random.random() * len(self.files))] + + # get full name of newfile + if parent == '/': + fullName = parent + nfile + else: + fullName = parent + '/' + nfile + + dprint('try createLink(%s %s %s)' % (target, nfile, parent)) + inum = self.createLink(target, nfile, parent) + if inum >= 0: + self.files.append(fullName) + self.nameToInum[fullName] = inum + if printOps: + print 'link("%s", "%s");' % (target, fullName) + return 0 + return -1 + + def doCreate(self, ftype): + dprint('doCreate') + parent = self.dirs[int(random.random() * len(self.dirs))] + nfile = self.makeName() + if ftype == 'd': + tlist = self.dirs + else: + tlist = self.files + + if parent == '/': + fullName = parent + nfile + else: + fullName = parent + '/' + nfile + + dprint('try createFile(%s %s %s)' % (parent, nfile, ftype)) + inum = self.createFile(parent, nfile, ftype) + if inum >= 0: + tlist.append(fullName) + self.nameToInum[fullName] = inum + if parent == '/': + parent = '' + if ftype == 'd': + if printOps: + print 'mkdir("%s/%s");' % (parent, nfile) + else: + if printOps: + print 'creat("%s/%s");' % (parent, nfile) + return 0 + return -1 + + def doAppend(self): + dprint('doAppend') + if len(self.files) == 0: + return -1 + afile = self.files[int(random.random() * len(self.files))] + dprint('try writeFile(%s)' % afile) + data = chr(ord('a') + int(random.random() * 26)) + rc = self.writeFile(afile, data) + return rc + + def run(self, numRequests): + self.percentMkdir = 0.40 + self.percentWrite = 0.40 + self.percentDelete = 0.20 + self.numRequests = numRequests + + print 'Initial state' + print '' + self.dump() + print '' + + for i in range(numRequests): + if printOps == False: + print 'Which operation took place?' + rc = -1 + while rc == -1: + r = random.random() + if r < 0.3: + rc = self.doAppend() + dprint('doAppend rc:%d' % rc) + elif r < 0.5: + rc = self.doDelete() + dprint('doDelete rc:%d' % rc) + elif r < 0.7: + rc = self.doLink() + dprint('doLink rc:%d' % rc) + else: + if random.random() < 0.75: + rc = self.doCreate('f') + dprint('doCreate(f) rc:%d' % rc) + else: + rc = self.doCreate('d') + dprint('doCreate(d) rc:%d' % rc) + if printState == True: + print '' + self.dump() + print '' + else: + print '' + print ' State of file system (inode bitmap, inodes, data bitmap, data)?' + print '' + + if printFinal: + print '' + print 'Summary of files, directories::' + print '' + print ' Files: ', self.files + print ' Directories:', self.dirs + print '' + +# +# main program +# +parser = OptionParser() + +parser.add_option('-s', '--seed', default=0, help='the random seed', action='store', type='int', dest='seed') +parser.add_option('-i', '--numInodes', default=8, help='number of inodes in file system', action='store', type='int', dest='numInodes') +parser.add_option('-d', '--numData', default=8, help='number of data blocks in file system', action='store', type='int', dest='numData') +parser.add_option('-n', '--numRequests', default=10, help='number of requests to simulate', action='store', type='int', dest='numRequests') +parser.add_option('-r', '--reverse', default=False, help='instead of printing state, print ops', action='store_true', dest='reverse') +parser.add_option('-p', '--printFinal', default=False, help='print the final set of files/dirs', action='store_true', dest='printFinal') + +(options, args) = parser.parse_args() + +print 'ARG seed', options.seed +print 'ARG numInodes', options.numInodes +print 'ARG numData', options.numData +print 'ARG numRequests', options.numRequests +print 'ARG reverse', options.reverse +print 'ARG printFinal', options.printFinal +print '' + +random.seed(options.seed) + +if options.reverse: + printState = False + printOps = True +else: + printState = True + printOps = False + + +printOps = True +printState = True + +printFinal = options.printFinal + +# +# have to generate RANDOM requests to the file system +# that are VALID! +# + +f = fs(options.numInodes, options.numData) + +# +# ops: mkdir rmdir : create delete : append write +# + +f.run(options.numRequests) + +