add lec21 spoc discuss: about simple file system

related_info/lab8/sfs-homework.md

@@ -0,0 +1,112 @@
+# 基于简单文件系统（Simple File System）模拟环境，理解文件系统的基本实现
+
+## 面向文件系统的用户操作
+- mkdir() - 创建一个新目录
+- creat() - 创建一个空文件
+- open(), write(), close() - 对文件写一个数据buffer
+- link()   - 对文件创建一个硬链接（hard link）
+- unlink() - 对文件取消一个硬链接 (如果文件的链接数为0，则删除文件
+
+
+## disk filesystem的内部组织和关键数据结构
+
+### disk filesystem的内部组织
+- superblock   : 可用inode数量，可用data block数量
+- inode bitmap ： inode的分配图（基于bitmap）
+- inodes       ： inode的存储区域
+- data bitmap  ： data block的分配图（基于bitmap）
+- data         ： data block的存储区域
+
+> bitmap: 0表示inode/data block是free， 1表示inode/data block是allocated
+
+### 关键数据结构 
+
+#### inode数据结构
+ - inode : 包含3个fields, 用python list 表示
+   - file type: f -> 常规文件：regular file, d -> 目录文件：directory
+   - data block addr of file content: -1 -> file is empty 
+   - reference count: file/directory的引用计数
+  
+> 比如 刚创建的一个空文件inode： `[f a:-1 r:1]`， 一个有1个硬链接的文件inode `[f a:10 r:2]`
+
+
+#### 数据块内容结构
+ - 一般文件的内容的表示：只是包含单个字符的list，比如`['a']`, `['b']` .....
+ - 目录内容的表示： 多个两元组`（name, inode_number）`形成的list，比如， 根目录 `[(.,0) (..,0)]`， 或者包含了一个`'f'`文件的根目录[(.,0) (..,0) (f,1)] 
+ 
+ 
+### 完整文件系统的例子 
+```
+fs.ibitmap: inode bitmap 11110000
+fs.inodes:       [d a:0 r:6] [f a:1 r:1] [f a:-1 r:1] [d a:2 r:2] [] ...
+fs.dbitmap: data bitmap  11100000
+fs.data:         [(.,0) (..,0) (y,1) (z,2) (x,3)] [u] [(.,3) (..,0)] [] ...
+```
+
+> 此文件系统有8个inode, 8个data blocks. 其中，根目录包含5个目录项，`”.“，”..“，”y“，”z“，”x“`, ”y“是常规文件,并有文件内容，包含一个data block，文件内容为”u“。”z“是一个空的常规文件。”x“是一个目录文件，是空目录。
+
+### 辅助数据结构 
+也可理解为内存中的文件系统相关数据结构
+
+- fs.files :当前文件系统中的常规文件list
+- fs.dirs : 当前文件系统中的目录文件list
+- fs.nameToInum ： 文件名:inode_num的对应关系
+
+## 文件系统执行流程
+
+### 文件系统初始化
+
+第一步：格式化sfs文件系统
+```
+        self.numInodes = numInodes
+        self.numData   = numData
+        
+        self.ibitmap = bitmap(self.numInodes)
+        self.inodes  = []
+        for i in range(self.numInodes):
+            self.inodes.append(inode())
+
+        self.dbitmap = bitmap(self.numData)
+        self.data    = []
+        for i in range(self.numData):
+            self.data.append(block('free'))
+```            
+
+第二步：创建sfs文件系统的根目录
+
+```
+        self.ibitmap.markAllocated(self.ROOT)
+        self.inodes[self.ROOT].setAll('d', 0, 2)
+        self.dbitmap.markAllocated(self.ROOT)
+        self.data[0].setType('d')
+        self.data[0].addDirEntry('.',  self.ROOT)
+        self.data[0].addDirEntry('..', self.ROOT)
+```
+
+第三步：在内存中保存相关数据
+```
+        self.files      = []
+        self.dirs       = ['/']
+        self.nameToInum = {'/':self.ROOT}         
+```
+
+
+第四步：随机生成文件相关的操作，改变sfs文件系统的内容        
+ - doAppend: 
+   - `fd=open(filename, O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd);`
+ - doDelete:
+   - `unlink()filename)`
+ - doLink
+   - `link()targetfile, sourcefile)`
+ - doCreate
+   - `create(filename)`  OR  `mkdir(dirname)`
+   
+
+## 问题1： 
+根据文件系统的状态变化信息，给出具体的文件相关操作内容
+
+## 问题2：
+实现 `writeFile, createFile, createLink, deleteFile`，使得你的实现能够达到与问题1的正确结果一致
+
+## 问题3：
+实现`soft link`机制，并设计测试用例说明你实现的正确性。   

related_info/lab8/sfs-homework.py

@@ -0,0 +1,470 @@
+#! /usr/bin/env python
+
+import random
+from optparse import OptionParser
+
+DEBUG = False
+
+def dprint(str):
+    if DEBUG:
+        print str
+
+printOps      = True
+printState    = True
+printFinal    = True
+
+class bitmap:
+    def __init__(self, size):
+        self.size = size
+        self.bmap = []
+        for num in range(size):
+            self.bmap.append(0)
+
+    def alloc(self):
+        for num in range(len(self.bmap)):
+            if self.bmap[num] == 0:
+                self.bmap[num] = 1
+                return num
+        return -1
+
+    def free(self, num):
+        assert(self.bmap[num] == 1)
+        self.bmap[num] = 0
+
+    def markAllocated(self, num):
+        assert(self.bmap[num] == 0)
+        self.bmap[num] = 1
+
+    def dump(self):
+        s = ''
+        for i in range(len(self.bmap)):
+            s += str(self.bmap[i])
+        return s
+
+class block:
+    def __init__(self, ftype):
+        assert(ftype == 'd' or ftype == 'f' or ftype == 'free')
+        self.ftype = ftype
+        # only for directories, properly a subclass but who cares
+        self.dirUsed = 0
+        self.maxUsed = 32
+        self.dirList = []
+        self.data    = ''
+
+    def dump(self):
+        if self.ftype == 'free':
+            return '[]'
+        elif self.ftype == 'd':
+            rc = ''
+            for d in self.dirList:
+                # d is of the form ('name', inum)
+                short = '(%s,%s)' % (d[0], d[1])
+                if rc == '':
+                    rc = short
+                else:
+                    rc += ' ' + short
+            return '['+rc+']'
+            # return '%s' % self.dirList
+        else:
+            return '[%s]' % self.data
+
+    def setType(self, ftype):
+        assert(self.ftype == 'free')
+        self.ftype = ftype
+
+    def addData(self, data):
+        assert(self.ftype == 'f')
+        self.data = data
+
+    def getNumEntries(self):
+        assert(self.ftype == 'd')
+        return self.dirUsed
+
+    def getFreeEntries(self):
+        assert(self.ftype == 'd')
+        return self.maxUsed - self.dirUsed
+
+    def getEntry(self, num):
+        assert(self.ftype == 'd')
+        assert(num < self.dirUsed)
+        return self.dirList[num]
+
+    def addDirEntry(self, name, inum):
+        assert(self.ftype == 'd')
+        self.dirList.append((name, inum))
+        self.dirUsed += 1
+        assert(self.dirUsed <= self.maxUsed)
+
+    def delDirEntry(self, name):
+        assert(self.ftype == 'd')
+        tname = name.split('/')
+        dname = tname[len(tname) - 1]
+        for i in range(len(self.dirList)):
+            if self.dirList[i][0] == dname:
+                self.dirList.pop(i)
+                self.dirUsed -= 1
+                return
+        assert(1 == 0)
+
+    def dirEntryExists(self, name):
+        assert(self.ftype == 'd')
+        for d in self.dirList:
+            if name == d[0]:
+                return True
+        return False
+
+    def free(self):
+        assert(self.ftype != 'free')
+        if self.ftype == 'd':
+            # check for only dot, dotdot here
+            assert(self.dirUsed == 2)
+            self.dirUsed = 0
+        self.data  = ''
+        self.ftype = 'free'
+
+class inode:
+    def __init__(self, ftype='free', addr=-1, refCnt=1):
+        self.setAll(ftype, addr, refCnt)
+
+    def setAll(self, ftype, addr, refCnt):
+        assert(ftype == 'd' or ftype == 'f' or ftype == 'free')
+        self.ftype  = ftype
+        self.addr   = addr
+        self.refCnt = refCnt
+
+    def incRefCnt(self):
+        self.refCnt += 1
+
+    def decRefCnt(self):
+        self.refCnt -= 1
+
+    def getRefCnt(self):
+        return self.refCnt
+
+    def setType(self, ftype):
+        assert(ftype == 'd' or ftype == 'f' or ftype == 'free')
+        self.ftype = ftype
+
+    def setAddr(self, block):
+        self.addr = block
+
+    def getSize(self):
+        if self.addr == -1:
+            return 0
+        else:
+            return 1
+
+    def getAddr(self):
+        return self.addr
+
+    def getType(self):
+        return self.ftype
+
+    def free(self):
+        self.ftype = 'free'
+        self.addr  = -1
+        
+
+class fs:
+    def __init__(self, numInodes, numData):
+        self.numInodes = numInodes
+        self.numData   = numData
+        
+        self.ibitmap = bitmap(self.numInodes)
+        self.inodes  = []
+        for i in range(self.numInodes):
+            self.inodes.append(inode())
+
+        self.dbitmap = bitmap(self.numData)
+        self.data    = []
+        for i in range(self.numData):
+            self.data.append(block('free'))
+    
+        # root inode
+        self.ROOT = 0
+
+        # create root directory
+        self.ibitmap.markAllocated(self.ROOT)
+        self.inodes[self.ROOT].setAll('d', 0, 2)
+        self.dbitmap.markAllocated(self.ROOT)
+        self.data[0].setType('d')
+        self.data[0].addDirEntry('.',  self.ROOT)
+        self.data[0].addDirEntry('..', self.ROOT)
+
+        # these is just for the fake workload generator
+        self.files      = []
+        self.dirs       = ['/']
+        self.nameToInum = {'/':self.ROOT}
+
+    def dump(self):
+        print 'inode bitmap ', self.ibitmap.dump()
+        print 'inodes       ',
+        for i in range(0,self.numInodes):
+            ftype = self.inodes[i].getType()
+            if ftype == 'free':
+                print '[]',
+            else:
+                print '[%s a:%s r:%d]' % (ftype, self.inodes[i].getAddr(), self.inodes[i].getRefCnt()),
+        print ''
+        print 'data bitmap  ', self.dbitmap.dump()
+        print 'data         ',
+        for i in range(self.numData):
+            print self.data[i].dump(),
+        print ''
+
+    def makeName(self):
+        p = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
+        return p[int(random.random() * len(p))]
+
+    def inodeAlloc(self):
+        return self.ibitmap.alloc()
+
+    def inodeFree(self, inum):
+        self.ibitmap.free(inum)
+        self.inodes[inum].free()
+
+    def dataAlloc(self):
+        return self.dbitmap.alloc()
+
+    def dataFree(self, bnum):
+        self.dbitmap.free(bnum)
+        self.data[bnum].free()
+        
+    def getParent(self, name):
+        tmp = name.split('/')
+        if len(tmp) == 2:
+            return '/'
+        pname = ''
+        for i in range(1, len(tmp)-1):
+            pname = pname + '/' + tmp[i]
+        return pname
+
+    def deleteFile(self, tfile):
+        if printOps:
+            print 'unlink("%s");' % tfile
+
+        inum = self.nameToInum[tfile]
+
+    # YOUR CODE, YOUR ID
+        # IF inode.refcnt ==1, THEN free data blocks first, then free inode, ELSE dec indoe.refcnt
+        # remove from parent directory: delete from parent inum, delete from parent addr
+    # DONE
+
+        # finally, remove from files list
+        self.files.remove(tfile)
+        return 0
+
+    def createLink(self, target, newfile, parent):
+    # YOUR CODE, YOUR ID
+        # find info about parent
+        # is there room in the parent directory?
+        # if the newfile was already in parent dir?
+        # now, find inumber of target
+        # inc parent ref count
+        # now add to directory
+    # DONE
+        return tinum
+
+    def createFile(self, parent, newfile, ftype):
+    # YOUR CODE, YOUR ID
+        # find info about parent
+        # is there room in the parent directory?
+        # have to make sure file name is unique
+        # find free inode
+        # if a directory, have to allocate directory block for basic (., ..) info
+        # now ok to init inode properly
+        # inc parent ref count
+        # and add to directory of parent
+    # DONE
+        return inum
+
+    def writeFile(self, tfile, data):
+        inum = self.nameToInum[tfile]
+        curSize = self.inodes[inum].getSize()
+        dprint('writeFile: inum:%d cursize:%d refcnt:%d' % (inum, curSize, self.inodes[inum].getRefCnt()))
+
+    # YOUR CODE, YOUR ID
+        # file is full?
+        # no data blocks left
+        # write file data
+    # DONE
+
+        if printOps:
+            print 'fd=open("%s", O_WRONLY|O_APPEND); write(fd, buf, BLOCKSIZE); close(fd);' % tfile
+        return 0
+            
+    def doDelete(self):
+        dprint('doDelete')
+        if len(self.files) == 0:
+            return -1
+        dfile = self.files[int(random.random() * len(self.files))]
+        dprint('try delete(%s)' % dfile)
+        return self.deleteFile(dfile)
+
+    def doLink(self):
+        dprint('doLink')
+        if len(self.files) == 0:
+            return -1
+        parent = self.dirs[int(random.random() * len(self.dirs))]
+        nfile = self.makeName()
+
+        # pick random target
+        target = self.files[int(random.random() * len(self.files))]
+
+        # get full name of newfile
+        if parent == '/':
+            fullName = parent + nfile
+        else:
+            fullName = parent + '/' + nfile
+
+        dprint('try createLink(%s %s %s)' % (target, nfile, parent))
+        inum = self.createLink(target, nfile, parent)
+        if inum >= 0:
+            self.files.append(fullName)
+            self.nameToInum[fullName] = inum
+            if printOps:
+                print 'link("%s", "%s");' % (target, fullName)
+            return 0
+        return -1
+    
+    def doCreate(self, ftype):
+        dprint('doCreate')
+        parent = self.dirs[int(random.random() * len(self.dirs))]
+        nfile = self.makeName()
+        if ftype == 'd':
+            tlist = self.dirs
+        else:
+            tlist = self.files
+
+        if parent == '/':
+            fullName = parent + nfile
+        else:
+            fullName = parent + '/' + nfile
+
+        dprint('try createFile(%s %s %s)' % (parent, nfile, ftype))
+        inum = self.createFile(parent, nfile, ftype)
+        if inum >= 0:
+            tlist.append(fullName)
+            self.nameToInum[fullName] = inum
+            if parent == '/':
+                parent = ''
+            if ftype == 'd':
+                if printOps:
+                    print 'mkdir("%s/%s");' % (parent, nfile)
+            else:
+                if printOps:
+                    print 'creat("%s/%s");' % (parent, nfile)
+            return 0
+        return -1
+
+    def doAppend(self):
+        dprint('doAppend')
+        if len(self.files) == 0:
+            return -1
+        afile = self.files[int(random.random() * len(self.files))]
+        dprint('try writeFile(%s)' % afile)
+        data = chr(ord('a') + int(random.random() * 26))
+        rc = self.writeFile(afile, data)
+        return rc
+
+    def run(self, numRequests):
+        self.percentMkdir  = 0.40
+        self.percentWrite  = 0.40
+        self.percentDelete = 0.20
+        self.numRequests   = numRequests
+
+        print 'Initial state'
+        print ''
+        self.dump()
+        print ''
+        
+        for i in range(numRequests):
+            if printOps == False:
+                print 'Which operation took place?'
+            rc = -1
+            while rc == -1:
+                r = random.random()
+                if r < 0.3:
+                    rc = self.doAppend()
+                    dprint('doAppend rc:%d' % rc)
+                elif r < 0.5:
+                    rc = self.doDelete()
+                    dprint('doDelete rc:%d' % rc)
+                elif r < 0.7:
+                    rc = self.doLink()
+                    dprint('doLink rc:%d' % rc)
+                else:
+                    if random.random() < 0.75:
+                        rc = self.doCreate('f')
+                        dprint('doCreate(f) rc:%d' % rc)
+                    else:
+                        rc = self.doCreate('d')
+                        dprint('doCreate(d) rc:%d' % rc)
+            if printState == True:
+                print ''
+                self.dump()
+                print ''
+            else:
+                print ''
+                print '  State of file system (inode bitmap, inodes, data bitmap, data)?'
+                print ''
+
+        if printFinal:
+            print ''
+            print 'Summary of files, directories::'
+            print ''
+            print '  Files:      ', self.files
+            print '  Directories:', self.dirs
+            print ''
+
+#
+# main program
+#
+parser = OptionParser()
+
+parser.add_option('-s', '--seed',        default=0,     help='the random seed',                      action='store', type='int', dest='seed')
+parser.add_option('-i', '--numInodes',   default=8,     help='number of inodes in file system',      action='store', type='int', dest='numInodes') 
+parser.add_option('-d', '--numData',     default=8,     help='number of data blocks in file system', action='store', type='int', dest='numData') 
+parser.add_option('-n', '--numRequests', default=10,    help='number of requests to simulate',       action='store', type='int', dest='numRequests')
+parser.add_option('-r', '--reverse',     default=False, help='instead of printing state, print ops', action='store_true',        dest='reverse')
+parser.add_option('-p', '--printFinal',  default=False, help='print the final set of files/dirs',    action='store_true',        dest='printFinal')
+
+(options, args) = parser.parse_args()
+
+print 'ARG seed',        options.seed
+print 'ARG numInodes',   options.numInodes
+print 'ARG numData',     options.numData
+print 'ARG numRequests', options.numRequests
+print 'ARG reverse',     options.reverse
+print 'ARG printFinal',  options.printFinal
+print ''
+
+random.seed(options.seed)
+
+if options.reverse:
+    printState = False
+    printOps   = True
+else:
+    printState = True
+    printOps   = False
+
+
+printOps   = True
+printState = True
+
+printFinal = options.printFinal
+
+#
+# have to generate RANDOM requests to the file system
+# that are VALID!
+#
+
+f = fs(options.numInodes, options.numData)
+
+#
+# ops: mkdir rmdir : create delete : append write
+#
+
+f.run(options.numRequests)
+
+