修改了refer的加载机制
This commit is contained in:
parent
d01c5b13a7
commit
75bba5fdb3
@ -33,8 +33,8 @@
|
||||
{
|
||||
"type": "Refer",
|
||||
"file_path": [
|
||||
"./genPrompts/Refer/标题参考格式.txt",
|
||||
"./genPrompts/Refer/正文开头引入段落参考.txt"
|
||||
"./genPrompts/Refer/标题参考格式.json",
|
||||
"./genPrompts/Refer/正文开头引入段落参考.json"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
6
scripts/test_refer.py
Normal file
6
scripts/test_refer.py
Normal file
@ -0,0 +1,6 @@
|
||||
import json
|
||||
|
||||
with open("genPrompts/Refer/标题参考格式.json", "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
print(data)
|
||||
Binary file not shown.
Binary file not shown.
@ -72,6 +72,7 @@ class PromptManager:
|
||||
self._dateline_cache = ResourceLoader.load_file_content(dateline_path)
|
||||
logging.info(f"预加载日期线文件: {dateline_path}")
|
||||
|
||||
|
||||
# 加载prompts_config配置的文件
|
||||
if not self.prompts_config:
|
||||
return
|
||||
@ -92,9 +93,12 @@ class PromptManager:
|
||||
|
||||
elif prompt_type == "demand":
|
||||
for path in file_paths:
|
||||
# print(path)
|
||||
if os.path.exists(path):
|
||||
filename = os.path.basename(path)
|
||||
# print(filename)
|
||||
content = ResourceLoader.load_file_content(path)
|
||||
# print(content)
|
||||
if content:
|
||||
self._demand_cache[filename] = content
|
||||
name_without_ext = os.path.splitext(filename)[0]
|
||||
@ -102,11 +106,14 @@ class PromptManager:
|
||||
|
||||
elif prompt_type == "refer":
|
||||
for path in file_paths:
|
||||
# print(path)
|
||||
if os.path.exists(path):
|
||||
filename = os.path.basename(path)
|
||||
content = ResourceLoader.load_file_content(path)
|
||||
# print(filename)
|
||||
content = ResourceLoader.load_all_refer_files(path)
|
||||
if content:
|
||||
self._refer_cache[filename] = content
|
||||
# print(content)
|
||||
|
||||
def find_directory_fuzzy_match(self, name, directory=None, files=None):
|
||||
"""
|
||||
|
||||
@ -22,16 +22,14 @@ class ResourceLoader:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def load_all_refer_files(refer_dir, refer_content_length=50):
|
||||
"""加载Refer目录下的所有文件内容"""
|
||||
def load_all_refer_files(file_path, refer_content_rate=0.5):
|
||||
"""加载Refer目录下的指定文件内容"""
|
||||
refer_content = ""
|
||||
if not refer_dir or not os.path.isdir(refer_dir):
|
||||
print(f"Warning: Refer directory '{refer_dir}' not found or invalid.")
|
||||
if not file_path or not os.path.isfile(file_path):
|
||||
print(f"Warning: Refer directory '{file_path}' not found or invalid.")
|
||||
return ""
|
||||
try:
|
||||
files = os.listdir(refer_dir)
|
||||
for file in files:
|
||||
file_path = os.path.join(refer_dir, file)
|
||||
if True: # print(file_path)
|
||||
if os.path.isfile(file_path) and file_path.endswith(".txt"):
|
||||
# Use the updated load_file_content
|
||||
content = ResourceLoader.load_file_content(file_path)
|
||||
@ -39,18 +37,38 @@ class ResourceLoader:
|
||||
# 用\n分割content,取前length条
|
||||
content_lines = content.split("\n")
|
||||
# Ensure refer_content_length doesn't exceed available lines
|
||||
sample_size = min(refer_content_length, len(content_lines))
|
||||
sample_size = int(len(content_lines) * refer_content_rate)
|
||||
content_lines = random.sample(content_lines, sample_size)
|
||||
content = "\n".join(content_lines)
|
||||
refer_content += f"## {file}\n{content}\n\n"
|
||||
refer_content += f"## {file_path}\n{content}\n\n"
|
||||
elif os.path.isfile(file_path) and file_path.endswith(".json"):
|
||||
try:
|
||||
# 读取json文件
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = json.load(f)
|
||||
file_content = json.load(f)
|
||||
|
||||
## 随机进行多次抽样
|
||||
# 检查必要的键是否存在
|
||||
if "title" not in file_content or "description" not in file_content or "examples" not in file_content:
|
||||
print(f"Warning: JSON文件 '{file_path}' 缺少必要的键(title/description/examples)")
|
||||
|
||||
refer_content += f"## {file}\n{content}\n\n"
|
||||
title_content = file_content["title"]
|
||||
description_content = file_content["description"]
|
||||
examples = file_content["examples"]
|
||||
|
||||
# 对examples进行采样
|
||||
if examples and isinstance(examples, list):
|
||||
sample_size = max(1, int(len(examples) * refer_content_rate))
|
||||
sampled_examples = random.sample(examples, sample_size)
|
||||
|
||||
# 格式化内容
|
||||
examples_formatted = json.dumps(sampled_examples, ensure_ascii=False, indent=2)
|
||||
content = f"{title_content}\n{description_content}\n{examples_formatted}\n"
|
||||
|
||||
refer_content += f"## {file_path}\n{content}\n\n"
|
||||
else:
|
||||
print(f"Warning: JSON文件 '{file_path}' 的examples不是有效列表")
|
||||
except Exception as json_err:
|
||||
print(f"处理JSON文件 '{file_path}' 失败: {json_err}")
|
||||
return refer_content
|
||||
except Exception as e:
|
||||
print(f"加载Refer目录文件失败: {e}")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user