110 lines
3.2 KiB
Python
110 lines
3.2 KiB
Python
|
"""检查JSONL文件是否符合API要求"""
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_jsonl_file(file_path):
|
|||
|
|
"""检查JSONL文件格式"""
|
|||
|
|
print("=" * 60)
|
|||
|
|
print("检查 JSONL 文件格式")
|
|||
|
|
print("=" * 60)
|
|||
|
|
print(f"文件: {file_path}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
if not os.path.exists(file_path):
|
|||
|
|
print("[ERROR] 文件不存在！")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
# 获取文件大小
|
|||
|
|
file_size = os.path.getsize(file_path)
|
|||
|
|
file_size_mb = file_size / (1024 * 1024)
|
|||
|
|
print(f"文件大小: {file_size} 字节 ({file_size_mb:.2f} MB)")
|
|||
|
|
|
|||
|
|
# 检查文件大小限制
|
|||
|
|
if file_size > 500 * 1024 * 1024:
|
|||
|
|
print("[ERROR] 文件大小超过500MB限制！")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
print()
|
|||
|
|
print("-" * 60)
|
|||
|
|
print("逐行检查:")
|
|||
|
|
print("-" * 60)
|
|||
|
|
|
|||
|
|
all_valid = True
|
|||
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|||
|
|
lines = f.readlines()
|
|||
|
|
|
|||
|
|
print(f"总行数: {len(lines)}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
for i, line in enumerate(lines, 1):
|
|||
|
|
line_size = len(line.encode("utf-8"))
|
|||
|
|
line_size_mb = line_size / (1024 * 1024)
|
|||
|
|
|
|||
|
|
print(f"第 {i} 行:")
|
|||
|
|
print(f" - 长度: {line_size} 字节 ({line_size_mb:.2f} MB)")
|
|||
|
|
|
|||
|
|
# 检查是否空行
|
|||
|
|
if len(line.strip()) == 0:
|
|||
|
|
print(f" - [ERROR] 空白行！API不允许空白行")
|
|||
|
|
all_valid = False
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 检查单行大小限制
|
|||
|
|
if line_size > 6 * 1024 * 1024:
|
|||
|
|
print(f" - [ERROR] 单行超过6MB限制！")
|
|||
|
|
all_valid = False
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 检查JSON格式
|
|||
|
|
try:
|
|||
|
|
obj = json.loads(line)
|
|||
|
|
print(f" - JSON格式: [OK]")
|
|||
|
|
|
|||
|
|
# 检查必需字段
|
|||
|
|
required_fields = ["custom_id", "method", "url", "body"]
|
|||
|
|
missing_fields = [field for field in required_fields if field not in obj]
|
|||
|
|
if missing_fields:
|
|||
|
|
print(f" - [ERROR] 缺少必需字段: {missing_fields}")
|
|||
|
|
all_valid = False
|
|||
|
|
else:
|
|||
|
|
print(f" - 必需字段: [OK]")
|
|||
|
|
print(f" - custom_id: {obj['custom_id']}")
|
|||
|
|
print(f" - method: {obj['method']}")
|
|||
|
|
print(f" - url: {obj['url']}")
|
|||
|
|
|
|||
|
|
# 检查body字段
|
|||
|
|
if "body" in obj:
|
|||
|
|
body = obj["body"]
|
|||
|
|
if "model" in body:
|
|||
|
|
print(f" - model: {body['model']}")
|
|||
|
|
else:
|
|||
|
|
print(f" - [ERROR] body中缺少model字段")
|
|||
|
|
all_valid = False
|
|||
|
|
|
|||
|
|
if "messages" in body:
|
|||
|
|
print(f" - messages: {len(body['messages'])} 条消息")
|
|||
|
|
else:
|
|||
|
|
print(f" - [ERROR] body中缺少messages字段")
|
|||
|
|
all_valid = False
|
|||
|
|
|
|||
|
|
except json.JSONDecodeError as e:
|
|||
|
|
print(f" - [ERROR] JSON解析失败: {e}")
|
|||
|
|
all_valid = False
|
|||
|
|
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
print("=" * 60)
|
|||
|
|
if all_valid:
|
|||
|
|
print("[SUCCESS] 文件格式检查通过！")
|
|||
|
|
else:
|
|||
|
|
print("[FAILED] 文件格式存在问题，请修复后再试")
|
|||
|
|
print("=" * 60)
|
|||
|
|
|
|||
|
|
return all_valid
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
check_jsonl_file("demo/test_model.jsonl")
|