mirror of
https://github.com/deepseek-ai/3FS
synced 2025-05-13 09:00:37 +00:00
587 lines
18 KiB
Python
587 lines
18 KiB
Python
import struct
|
||
import os
|
||
import mmap
|
||
import array
|
||
import time
|
||
import threading
|
||
try:
|
||
import numpy as np
|
||
except ImportError:
|
||
np = None
|
||
|
||
|
||
file_modes = ['r', 'rb', 'r+', 'rb+', 'w', 'wb', 'w+', 'a', 'ab', 'a+', 'ab+']
|
||
path = ""
|
||
text_path = ""
|
||
binary_path = ""
|
||
|
||
def check_read(fd, path, val, desc):
|
||
if fd == None:
|
||
fd = open(path, 'r')
|
||
v = fd.read(len(val))
|
||
if v != val:
|
||
raise RuntimeError(v, val, 'Check Error: ', desc)
|
||
fd.close()
|
||
|
||
def write_file(fd, val, cls = False):
|
||
fd.write(val)
|
||
fd.flush()
|
||
if cls:
|
||
fd.close()
|
||
|
||
def print_file_data(fd, path, num):
|
||
if fd == None:
|
||
fd = open(path, 'r')
|
||
fd.read(num)
|
||
print('print_file_data', num, fd.read(num))
|
||
fd.close()
|
||
|
||
def open_test():
|
||
fd = open(text_path+'1', 'w+')
|
||
write_file(fd, 'abcdefghigk')
|
||
check_read(None, text_path+'1', 'abc', 'open w+ mode')
|
||
|
||
fd2 = open(text_path+'2', 'w+')
|
||
fd2.close()
|
||
|
||
fd = open(text_path+'1', 'r')
|
||
check_read(fd, '', 'abc', 'open r mode')
|
||
fd = open(text_path+'1', 'rb')
|
||
check_read(fd, '', b'abc', 'open rb mode')
|
||
fd = open(text_path+'1', 'r+')
|
||
check_read(fd, '', 'abc', 'open r+ mode')
|
||
fd = open(text_path+'1', 'rb+')
|
||
check_read(fd, '', b'abc', 'open rb+ mode')
|
||
|
||
#正常写文本文件
|
||
fd = open(text_path+'2_w', 'w')
|
||
write_file(fd, 'cba')
|
||
check_read(None, text_path+'2_w', 'cba', 'open w mode')
|
||
#将文本写入二进制文件
|
||
fd = open(binary_path+'2_w', 'wb')
|
||
#print_file_data(None, binary_path+'2', 6)
|
||
write_file(fd, b'efg')
|
||
check_read(None, binary_path+'2_w', 'efg', 'open wb mode')
|
||
#读写打开，用同一个fd读写，校验
|
||
fd = open(text_path+'2_w', 'w+')
|
||
write_file(fd, 'etc', True)
|
||
check_read(None, text_path+'2_w', 'etc', 'open w+ mode')
|
||
|
||
#追加文本文件
|
||
fd = open(text_path+'2_w', 'a')
|
||
write_file(fd, 'zzz', True)
|
||
check_read(None, text_path+'2_w', 'etczzz', 'open a mode')
|
||
#追加二进制文件
|
||
fd = open(binary_path+'2_w', 'ab')
|
||
write_file(fd, b'zzz', True)
|
||
check_read(None, binary_path+'2_w', 'efgzzz', 'open ab mode')
|
||
#读写文本文件，追加后再读取
|
||
fd = open(text_path+'2_w', 'a+')
|
||
write_file(fd, 'yyy', True)
|
||
check_read(None, text_path+'2_w', 'etczzzyyy', 'open a+ mode')
|
||
#读写二进制文件，追加后再读取
|
||
# fd = open(binary_path+'389', 'ab+')
|
||
# write_file(fd, b'xxx', True)
|
||
# check_read(None, binary_path+'389', 'xxx', 'open ab+ mode')
|
||
# fd.close()
|
||
#测试非法模式
|
||
is_err = True
|
||
try:
|
||
fd = open(binary_path+'389', 'aa')
|
||
except ValueError as err:
|
||
is_err = False
|
||
print('open_test 1: {0}'.format(err))
|
||
if is_err:
|
||
raise RuntimeError('open_test 1 Error')
|
||
|
||
os.unlink(text_path+'2_w')
|
||
print('open test over')
|
||
|
||
def seek_test():
|
||
fd = open(text_path+'1', 'r+')
|
||
#fd.seek(-1)
|
||
#获取文件长度
|
||
fd.seek(0, 2)
|
||
len = fd.tell()
|
||
print(text_path+'1', ' file_len = ', len)
|
||
#seek到文件末尾之后,是成功的
|
||
idx = fd.seek(len+2, 0)
|
||
fd.read(2)
|
||
#raise RuntimeError('seek_test len+1,', idx)
|
||
#直接seek到最后+1，异常
|
||
is_err = True
|
||
try:
|
||
fd.seek(1, 2)
|
||
except ValueError as err:
|
||
is_err = False
|
||
print('seek_test 2: {0}'.format(err))
|
||
if is_err:
|
||
raise RuntimeError('seek_test 2 Error')
|
||
fd.close()
|
||
|
||
print('seek test over')
|
||
|
||
def flush_test():
|
||
fd = open(text_path+'2', 'r')
|
||
fd.flush()
|
||
fd = open(text_path+'2_w', 'wb')
|
||
fd.flush()
|
||
|
||
print('flush test over')
|
||
|
||
def next_test():
|
||
fd = open(text_path+'2_w', 'w+')
|
||
#空文件获取行
|
||
is_err = True
|
||
try:
|
||
val = next(fd)
|
||
except StopIteration as err:
|
||
is_err = False
|
||
print('next_test 1: {0}'.format(err))
|
||
if is_err:
|
||
raise RuntimeError('next_test 1 Error')
|
||
#写入不同数据，顺序读取
|
||
fd.write('abc\nttt\nbbb\nccc\nggg')
|
||
fd.flush()
|
||
fd.seek(0)
|
||
val = next(fd)
|
||
if val.strip() != 'abc':
|
||
raise RuntimeError('next_test 2 Error')
|
||
val = next(fd)
|
||
if val.strip() != 'ttt':
|
||
raise RuntimeError('next_test 3 Error')
|
||
val = next(fd)
|
||
if val.strip() != 'bbb':
|
||
raise RuntimeError('next_test 4 Error')
|
||
|
||
fd.close()
|
||
os.remove(text_path+'2_w')
|
||
print('next test over')
|
||
|
||
def read_test():
|
||
fd = open(text_path+'1', 'r+')
|
||
#读取长度为0的数据
|
||
val = fd.read(0)
|
||
if len(val) != 0:
|
||
raise RuntimeError('read_test 1,', val)
|
||
#在文件最后位置读取数据
|
||
fd.seek(0, 2)
|
||
val = fd.read(1)
|
||
|
||
if len(val) != 0:
|
||
raise RuntimeError('read_test 2 Error')
|
||
|
||
fd.close()
|
||
|
||
#在写权限的文件读取
|
||
is_err = True
|
||
fd = open(text_path+'2_w', 'w')
|
||
try:
|
||
fd.read(1)
|
||
except IOError as err:
|
||
is_err = False
|
||
print('read_test 3: {0}'.format(err))
|
||
|
||
if is_err:
|
||
raise RuntimeError('read_test 3 Error')
|
||
print('read test over')
|
||
|
||
def truncate_test():
|
||
#读权限截取
|
||
fd = open(text_path+'2_w', 'r')
|
||
is_err = True
|
||
try:
|
||
fd.truncate()
|
||
fd.close()
|
||
except IOError as err:
|
||
is_err = False
|
||
print('truncate_test 1: {0}'.format(err))
|
||
if is_err:
|
||
raise RuntimeError('truncate_test 1 Error')
|
||
|
||
#正常截取
|
||
fd = open(text_path+'2_w', 'w+')
|
||
fd.truncate()
|
||
fd.write('abcd\nefg')
|
||
#当前位置截取
|
||
fd.seek(4, 0)
|
||
fd.truncate()
|
||
fd.seek(0, 0)
|
||
val = fd.read(4)
|
||
print('val = ', val)
|
||
if val != 'abcd':
|
||
raise RuntimeError('truncate_test 2 Error')
|
||
#非法位置截取
|
||
fd.truncate(10)
|
||
|
||
fd.close()
|
||
print('truncate test over')
|
||
|
||
def tell_test():
|
||
#正常获取
|
||
fd = open(text_path+'1', 'r')
|
||
fd.seek(0, 2)
|
||
len = fd.tell()
|
||
fd.seek(1, 0)
|
||
fd.read(5)
|
||
fd.seek(0, 1)
|
||
idx = fd.tell()
|
||
if idx != 6:
|
||
raise RuntimeError('tell_test 1 Error')
|
||
#超出文件末尾获取
|
||
fd.seek(len+1, 0)
|
||
idx = fd.tell()
|
||
if idx != len+1:
|
||
raise RuntimeError('tell_test 2 Error')
|
||
|
||
print('tell test over')
|
||
|
||
def readlines_test():
|
||
fd = open(text_path+'2_w', 'w+')
|
||
#空文件获取行
|
||
val = fd.readline(-1)
|
||
fd.write('abc\nttt\nbbb\nccc\nggg')
|
||
fd.flush()
|
||
fd.seek(0, 2)
|
||
f_len = fd.tell()
|
||
fd.seek(0)
|
||
val = fd.readline(-1)
|
||
if val.strip() != 'abc':
|
||
raise RuntimeError('readlines_test 1 Error')
|
||
val = fd.readline(0)
|
||
if len(val) != 0:
|
||
raise RuntimeError('readlines_test 2 Error')
|
||
|
||
val = fd.readline(2)
|
||
if val.strip() != 'tt':
|
||
raise RuntimeError('readlines_test 3 Error')
|
||
|
||
val = fd.readline(f_len + 5)
|
||
if val.strip() != 't':
|
||
raise RuntimeError('readlines_test 4 Error')
|
||
|
||
print('readlines test over')
|
||
|
||
def mmap_test():
|
||
fd = os.open(text_path+'1', os.O_RDWR | os.O_NONBLOCK)
|
||
mm = mmap.mmap(fd, 0)
|
||
if mm[:6] != b'abcdef':
|
||
raise RuntimeError('mmap_test 1 Error')
|
||
os.close(fd)
|
||
print('mmap test over')
|
||
|
||
def os_open_test():
|
||
fd = os.open(text_path+'2_rw', os.O_RDWR | os.O_CREAT | os.O_TRUNC)
|
||
b1 = bytearray(b'123')
|
||
b2 = bytearray(b'abc')
|
||
b3 = bytearray(b'456')
|
||
bufs = []
|
||
bufs.append(b1)
|
||
bufs.append(b2)
|
||
bufs.append(b3)
|
||
os.writev(fd, bufs)
|
||
os.lseek(fd, 0, 0)
|
||
val = os.read(fd, 5)
|
||
if val != b'123ab':
|
||
raise RuntimeError('os_open_test 1 Error')
|
||
os.close(fd)
|
||
|
||
#open文件，但bufsize超出长度
|
||
fd = os.open(text_path+'2_rw', os.O_RDONLY, 0)
|
||
val = os.read(fd, 20)
|
||
if val != b'123abc456':
|
||
raise RuntimeError('os_open_test 2 Error')
|
||
os.close(fd)
|
||
|
||
print('os_open test over')
|
||
|
||
def os_mkdir_test():
|
||
#创建文件夹
|
||
try:
|
||
os.mkdir(text_path+'test_file')
|
||
fd = open(text_path+'test_file/1_t', 'w')
|
||
fd.close()
|
||
#在文件夹里有文件的情况下删除
|
||
os.rmdir(text_path+'test_file')
|
||
except OSError as err:
|
||
print('os_mkdir_test 1: {0} 预期的结果'.format(err))
|
||
|
||
os.unlink(text_path+'test_file/1_t')
|
||
os.rmdir(text_path+'test_file')
|
||
if os.path.exists(text_path+'test_file2'):
|
||
os.rmdir(text_path+'test_file2')
|
||
|
||
print('os_mkdir test over')
|
||
|
||
def os_link_test():
|
||
try:
|
||
os.mkdir(text_path+'test_file')
|
||
os.mkdir(text_path+'test_file2')
|
||
fd = open(text_path+'test_file/1_t', 'w')
|
||
os.link(text_path+'test_file/1_t', text_path+'test_file2/1_t2')
|
||
except PermissionError as err:
|
||
print('os_link_test 1: {0}'.format(err))
|
||
|
||
print('os_link test over')
|
||
|
||
def os_unlink_test():
|
||
os.unlink(text_path+'test_file/1_t')
|
||
os.unlink(text_path+'test_file2/1_t2')
|
||
os.rmdir(text_path+'test_file')
|
||
os.rmdir(text_path+'test_file2')
|
||
|
||
print('os_unlink test over')
|
||
|
||
letter_ary = []
|
||
binary_ary = []
|
||
for ch in range(97, 123):
|
||
letter_ary.append(chr(ch))
|
||
binary_ary.append(ch)
|
||
|
||
def thread_check_text(start, offset, ary, text_data):
|
||
for i in range(offset):
|
||
t = (start + i) % 26
|
||
#print(t, text_data, text_data[i])
|
||
if ary[t] != text_data[i]:
|
||
raise RuntimeError('text_data check error: {0}, {1}, {2} right: {3}, real:{4}'.format( start, offset, i, ary[t], text_data[i]))
|
||
|
||
def thread_check_binary(start, offset, binary_data):
|
||
for i in range(offset):
|
||
t = (start + i + 1) % 126
|
||
if t == 0:
|
||
t = 126
|
||
if t != binary_data[i]:
|
||
raise RuntimeError('binary_data check error: start {0} offset {1} index {2} expected: {3} actual: {4}'.format( start, offset, i, t, binary_data[i]))
|
||
|
||
def thread_check_text_np(start, offset, ary, text_data):
|
||
expected = np.arange(start, start + offset, dtype=np.int32)
|
||
expected %= 26
|
||
expected += ary[0]
|
||
# actual = np.array(text_data, dtype=np.int32)
|
||
actual = np.frombuffer(text_data, dtype=np.uint8)
|
||
if not np.array_equal(expected, actual):
|
||
i = np.argmin(expected == actual)
|
||
raise RuntimeError('text_data check error: start {0} offset {1} index {2} expected: {3} actual: {4}'.format( start, offset, i, expected[i:i+16], actual[i:i+16]))
|
||
|
||
def thread_check_binary_np(start, offset, binary_data):
|
||
expected = np.arange(start + 1, start + offset + 1, dtype=np.int32)
|
||
expected %= 126
|
||
expected[expected == 0] = 126
|
||
# actual = np.array(binary_data, dtype=np.int32)
|
||
actual = np.frombuffer(binary_data, dtype=np.uint8)
|
||
if not np.array_equal(expected, actual):
|
||
i = np.argmin(expected == actual)
|
||
raise RuntimeError('binary_data check error: start {0} offset {1} index {2} expected: {3} actual: {4}'.format( start, offset, i, expected[i:i+16], actual[i:i+16]))
|
||
|
||
#文本校验
|
||
def check_text_data(start, offset, text_data):
|
||
if offset != len(text_data):
|
||
raise RuntimeError('check_text_data parameter error: offset != len(text_data), {0} {1} {2}'.format(offset, len(text_data), start))
|
||
|
||
return
|
||
|
||
if type(text_data).__name__ == 'bytes':
|
||
ary = binary_ary
|
||
else:
|
||
ary = letter_ary
|
||
|
||
if len(text_data) <= 1024:
|
||
thread_check_text(start, offset, ary, text_data)
|
||
else:
|
||
thread_check_text_np(start, offset, ary, text_data)
|
||
|
||
# p = threading.Thread(target=thread_check_text_np, args=(start, offset, ary, text_data,))
|
||
# p.start()
|
||
|
||
# for i in range(offset):
|
||
# t = (start + i) % 26
|
||
# #print(t, text_data, text_data[i])
|
||
# if ary[t] != text_data[i]:
|
||
# raise RuntimeError('text_data check error: {0}, {1}, {2} right: {3}, real:{4}'.format( start, offset, i, ary[t], text_data[i]))
|
||
|
||
#二进制校验
|
||
def check_binary_data(start, offset, binary_data):
|
||
if offset != len(binary_data):
|
||
raise RuntimeError('check_binary_data parameter error: offset != len(binary_data), {0} {1}'.format(offset, len(binary_data)))
|
||
|
||
if len(binary_data) <= 1024:
|
||
thread_check_binary(start, offset, binary_data)
|
||
else:
|
||
thread_check_binary_np(start, offset, binary_data)
|
||
|
||
# p = threading.Thread(target=thread_check_binary_np, args=(start, offset, binary_data,))
|
||
# p.start()
|
||
|
||
# for i in range(offset):
|
||
# t = (start + i + 1) % 126
|
||
# if t == 0:
|
||
# t = 126
|
||
# if t != binary_data[i]:
|
||
# raise RuntimeError('binary_data check error: {0}, {1}, {2} right: {3}, real:{4}'.format( start, offset, i, t, binary_data[i]))
|
||
|
||
def check_readv_results(file_path, read_offset, data_bufs, data_len):
|
||
index = read_offset
|
||
|
||
readbufsize = sum(len(buf) for buf in data_bufs)
|
||
filesize = os.path.getsize(file_path)
|
||
if read_offset > filesize :
|
||
assert data_len == 0, "read start exceeding file end, return size should be 0. now it's %d " % (data_len)
|
||
return
|
||
|
||
expected_return_size = readbufsize if (read_offset + readbufsize <= filesize) else (filesize - read_offset)
|
||
assert data_len == expected_return_size, "expected size %d, result return size %d" % (expected_return_size, data_len)
|
||
|
||
#print('data_bufs size = ', len(data_bufs[0]))
|
||
#print(data_bufs)
|
||
for buf in data_bufs:
|
||
l = len(buf)
|
||
if l <= data_len:
|
||
val = bytes(buf)
|
||
else:
|
||
val = bytes(buf[:data_len])
|
||
l = data_len
|
||
#print('val = ', val, data_len)
|
||
data_len -= l
|
||
#print(index, l, val[:128])
|
||
if file_path.find('binary') != -1:
|
||
check_binary_data(index, l, val)
|
||
else:
|
||
check_text_data(index, l, val)
|
||
index += l
|
||
if data_len <= 0:
|
||
break
|
||
|
||
def os_read_test_base(file_path, index, size, file_len = -1):
|
||
fd = os.open(file_path, os.O_RDONLY )
|
||
os.lseek(fd, index, 0)
|
||
val = os.read(fd, size)
|
||
os.close(fd)
|
||
|
||
readbufsize = size
|
||
filesize = os.path.getsize(file_path)
|
||
read_offset = index
|
||
data_len = len(val)
|
||
|
||
if read_offset > filesize :
|
||
assert data_len == 0, "read start exceeding file end, return size should be 0. now it's %d " % (data_len)
|
||
return
|
||
|
||
expected_return_size = readbufsize if (read_offset + readbufsize <= filesize) else (filesize - read_offset)
|
||
assert data_len == expected_return_size, "expected size %d, result return size %d" % (expected_return_size, data_len)
|
||
|
||
#print(index, l, val[:128])
|
||
if file_path.find('binary') != -1:
|
||
check_binary_data(index, data_len, val)
|
||
else:
|
||
check_text_data(index, data_len, val)
|
||
|
||
def os_readv_test_base(file_path, index, buffers_size, file_len = -1):
|
||
buffers = []
|
||
size = 0
|
||
for s in buffers_size:
|
||
buf = bytearray(s)
|
||
buffers.append(buf)
|
||
size += s
|
||
print('bufs = ', buffers_size)
|
||
fd = os.open(file_path, os.O_RDONLY ) #| os.O_DIRECT
|
||
os.lseek(fd, index, 0)
|
||
num_bytes = os.readv(fd, buffers)
|
||
os.close(fd)
|
||
|
||
check_readv_results(file_path, index, buffers, num_bytes)
|
||
|
||
def test_continuous_readv(fd, index, buffers_size):
|
||
buffers = []
|
||
size = 0
|
||
for s in buffers_size:
|
||
buf = bytearray(s)
|
||
buffers.append(buf)
|
||
size += s
|
||
print('bufs = ', buffers_size)
|
||
os.lseek(fd, index, 0)
|
||
num_bytes = os.readv(fd, buffers)
|
||
#print('buffers size = ', len(buffers), num_bytes, size, buffers)
|
||
if num_bytes == size:
|
||
for buf in buffers:
|
||
l = len(buf)
|
||
val = bytes(buf)
|
||
check_text_data(index, l, val)
|
||
index += l
|
||
else:
|
||
raise RuntimeError('os_readv_test_base error: num_bytes != size {0}, {1}'.format( num_bytes, size))
|
||
|
||
def posix_io_text():
|
||
print("=== [start test suite] posix_io_test basic funtions ===")
|
||
open_test()
|
||
seek_test()
|
||
flush_test()
|
||
next_test()
|
||
read_test()
|
||
truncate_test()
|
||
tell_test()
|
||
readlines_test()
|
||
mmap_test()
|
||
os_open_test()
|
||
os_mkdir_test()
|
||
os_link_test()
|
||
os_unlink_test()
|
||
print("=== [finish test suite] posix_io_test basic functions ===")
|
||
|
||
def os_write_run(path, open_mode, max_file_size, write_data):
|
||
fd = os.open(path, open_mode)
|
||
idx = 0
|
||
print("write started", path, max_file_size)
|
||
while idx < max_file_size:
|
||
os.write(fd, write_data)
|
||
idx += len(write_data)
|
||
os.close(fd)
|
||
print("write done", path, max_file_size)
|
||
|
||
def os_read_run(path, open_mode, max_file_size):
|
||
fd = os.open(path, open_mode)
|
||
idx = 0
|
||
data = array.array('B')
|
||
check_data = check_text_data if path.find('text') != -1 else check_binary_data
|
||
|
||
print("read started", path, max_file_size, flush=True)
|
||
while idx < max_file_size - 26:
|
||
val = os.read(fd, 26)
|
||
data.frombytes(val)
|
||
l = len(val)
|
||
idx += l
|
||
if l == 0:
|
||
print("wait for more data...")
|
||
time.sleep(0.5)
|
||
|
||
if idx % (max_file_size // 20) < 26:
|
||
print(idx * 100 // max_file_size, 'percent done', 'idx', idx, 'max file size', max_file_size, flush=True)
|
||
|
||
print("read done", path, flush=True)
|
||
os.close(fd)
|
||
print("check data", flush=True)
|
||
check_data(0, len(data), data)
|
||
|
||
if __name__ == "__main__":
|
||
import argparse
|
||
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("path", help="The path argument")
|
||
args = parser.parse_args()
|
||
path = args.path
|
||
text_path = os.path.join(path, "text")
|
||
binary_path = os.path.join(path, "binary")
|
||
|
||
print('Path', path)
|
||
print('posix test')
|
||
print('text_path = ', text_path)
|
||
print('binary_path = ', binary_path)
|
||
print('letter_ary = ', letter_ary)
|
||
print('binary_ary = ', binary_ary)
|
||
|
||
posix_io_text()
|
||
#
|
||
# os_read_test()
|
||
# os_readv_test()
|
||
|
||
|