mirror of
https://github.com/deepseek-ai/DeepSeek-Math
synced 2024-11-22 11:38:17 +00:00
31 lines
691 B
Python
31 lines
691 B
Python
|
import json
|
||
|
|
import random
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
def set_seed(seed):
|
||
|
|
if seed > 0:
|
||
|
|
random.seed(seed)
|
||
|
|
np.random.seed(seed)
|
||
|
|
|
||
|
|
def shuffle(data, seed):
|
||
|
|
if seed < 0:
|
||
|
|
return data
|
||
|
|
set_seed(seed)
|
||
|
|
indices = list(range(len(data)))
|
||
|
|
np.random.shuffle(indices)
|
||
|
|
data = [data[i] for i in indices]
|
||
|
|
return data
|
||
|
|
|
||
|
|
def read_data(path):
|
||
|
|
if path.endswith("json"):
|
||
|
|
data = json.load(open(path, "r"))
|
||
|
|
elif path.endswith("jsonl"):
|
||
|
|
data = []
|
||
|
|
with open(path, "r") as file:
|
||
|
|
for line in file:
|
||
|
|
line = json.loads(line)
|
||
|
|
data.append(line)
|
||
|
|
else:
|
||
|
|
raise NotImplementedError()
|
||
|
|
return data
|