修改了refer的加载机制
This commit is contained in:
parent
d01c5b13a7
commit
75bba5fdb3
@ -33,8 +33,8 @@
|
|||||||
{
|
{
|
||||||
"type": "Refer",
|
"type": "Refer",
|
||||||
"file_path": [
|
"file_path": [
|
||||||
"./genPrompts/Refer/标题参考格式.txt",
|
"./genPrompts/Refer/标题参考格式.json",
|
||||||
"./genPrompts/Refer/正文开头引入段落参考.txt"
|
"./genPrompts/Refer/正文开头引入段落参考.json"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|||||||
6
scripts/test_refer.py
Normal file
6
scripts/test_refer.py
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
with open("genPrompts/Refer/标题参考格式.json", "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
print(data)
|
||||||
Binary file not shown.
Binary file not shown.
@ -71,6 +71,7 @@ class PromptManager:
|
|||||||
if os.path.exists(dateline_path):
|
if os.path.exists(dateline_path):
|
||||||
self._dateline_cache = ResourceLoader.load_file_content(dateline_path)
|
self._dateline_cache = ResourceLoader.load_file_content(dateline_path)
|
||||||
logging.info(f"预加载日期线文件: {dateline_path}")
|
logging.info(f"预加载日期线文件: {dateline_path}")
|
||||||
|
|
||||||
|
|
||||||
# 加载prompts_config配置的文件
|
# 加载prompts_config配置的文件
|
||||||
if not self.prompts_config:
|
if not self.prompts_config:
|
||||||
@ -92,9 +93,12 @@ class PromptManager:
|
|||||||
|
|
||||||
elif prompt_type == "demand":
|
elif prompt_type == "demand":
|
||||||
for path in file_paths:
|
for path in file_paths:
|
||||||
|
# print(path)
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
filename = os.path.basename(path)
|
filename = os.path.basename(path)
|
||||||
|
# print(filename)
|
||||||
content = ResourceLoader.load_file_content(path)
|
content = ResourceLoader.load_file_content(path)
|
||||||
|
# print(content)
|
||||||
if content:
|
if content:
|
||||||
self._demand_cache[filename] = content
|
self._demand_cache[filename] = content
|
||||||
name_without_ext = os.path.splitext(filename)[0]
|
name_without_ext = os.path.splitext(filename)[0]
|
||||||
@ -102,11 +106,14 @@ class PromptManager:
|
|||||||
|
|
||||||
elif prompt_type == "refer":
|
elif prompt_type == "refer":
|
||||||
for path in file_paths:
|
for path in file_paths:
|
||||||
|
# print(path)
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
filename = os.path.basename(path)
|
filename = os.path.basename(path)
|
||||||
content = ResourceLoader.load_file_content(path)
|
# print(filename)
|
||||||
|
content = ResourceLoader.load_all_refer_files(path)
|
||||||
if content:
|
if content:
|
||||||
self._refer_cache[filename] = content
|
self._refer_cache[filename] = content
|
||||||
|
# print(content)
|
||||||
|
|
||||||
def find_directory_fuzzy_match(self, name, directory=None, files=None):
|
def find_directory_fuzzy_match(self, name, directory=None, files=None):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@ -22,16 +22,14 @@ class ResourceLoader:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load_all_refer_files(refer_dir, refer_content_length=50):
|
def load_all_refer_files(file_path, refer_content_rate=0.5):
|
||||||
"""加载Refer目录下的所有文件内容"""
|
"""加载Refer目录下的指定文件内容"""
|
||||||
refer_content = ""
|
refer_content = ""
|
||||||
if not refer_dir or not os.path.isdir(refer_dir):
|
if not file_path or not os.path.isfile(file_path):
|
||||||
print(f"Warning: Refer directory '{refer_dir}' not found or invalid.")
|
print(f"Warning: Refer directory '{file_path}' not found or invalid.")
|
||||||
return ""
|
return ""
|
||||||
try:
|
try:
|
||||||
files = os.listdir(refer_dir)
|
if True: # print(file_path)
|
||||||
for file in files:
|
|
||||||
file_path = os.path.join(refer_dir, file)
|
|
||||||
if os.path.isfile(file_path) and file_path.endswith(".txt"):
|
if os.path.isfile(file_path) and file_path.endswith(".txt"):
|
||||||
# Use the updated load_file_content
|
# Use the updated load_file_content
|
||||||
content = ResourceLoader.load_file_content(file_path)
|
content = ResourceLoader.load_file_content(file_path)
|
||||||
@ -39,18 +37,38 @@ class ResourceLoader:
|
|||||||
# 用\n分割content,取前length条
|
# 用\n分割content,取前length条
|
||||||
content_lines = content.split("\n")
|
content_lines = content.split("\n")
|
||||||
# Ensure refer_content_length doesn't exceed available lines
|
# Ensure refer_content_length doesn't exceed available lines
|
||||||
sample_size = min(refer_content_length, len(content_lines))
|
sample_size = int(len(content_lines) * refer_content_rate)
|
||||||
content_lines = random.sample(content_lines, sample_size)
|
content_lines = random.sample(content_lines, sample_size)
|
||||||
content = "\n".join(content_lines)
|
content = "\n".join(content_lines)
|
||||||
refer_content += f"## {file}\n{content}\n\n"
|
refer_content += f"## {file_path}\n{content}\n\n"
|
||||||
elif os.path.isfile(file_path) and file_path.endswith(".json"):
|
elif os.path.isfile(file_path) and file_path.endswith(".json"):
|
||||||
# 读取json文件
|
try:
|
||||||
with open(file_path, 'r', encoding='utf-8') as f:
|
# 读取json文件
|
||||||
content = json.load(f)
|
with open(file_path, 'r', encoding='utf-8') as f:
|
||||||
|
file_content = json.load(f)
|
||||||
## 随机进行多次抽样
|
|
||||||
|
# 检查必要的键是否存在
|
||||||
refer_content += f"## {file}\n{content}\n\n"
|
if "title" not in file_content or "description" not in file_content or "examples" not in file_content:
|
||||||
|
print(f"Warning: JSON文件 '{file_path}' 缺少必要的键(title/description/examples)")
|
||||||
|
|
||||||
|
title_content = file_content["title"]
|
||||||
|
description_content = file_content["description"]
|
||||||
|
examples = file_content["examples"]
|
||||||
|
|
||||||
|
# 对examples进行采样
|
||||||
|
if examples and isinstance(examples, list):
|
||||||
|
sample_size = max(1, int(len(examples) * refer_content_rate))
|
||||||
|
sampled_examples = random.sample(examples, sample_size)
|
||||||
|
|
||||||
|
# 格式化内容
|
||||||
|
examples_formatted = json.dumps(sampled_examples, ensure_ascii=False, indent=2)
|
||||||
|
content = f"{title_content}\n{description_content}\n{examples_formatted}\n"
|
||||||
|
|
||||||
|
refer_content += f"## {file_path}\n{content}\n\n"
|
||||||
|
else:
|
||||||
|
print(f"Warning: JSON文件 '{file_path}' 的examples不是有效列表")
|
||||||
|
except Exception as json_err:
|
||||||
|
print(f"处理JSON文件 '{file_path}' 失败: {json_err}")
|
||||||
return refer_content
|
return refer_content
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"加载Refer目录文件失败: {e}")
|
print(f"加载Refer目录文件失败: {e}")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user