refer模块正确抽样
This commit is contained in:
parent
5c5d03d2a3
commit
bbaa6eee53
@ -305,232 +305,112 @@ class PromptManager:
|
||||
return None
|
||||
|
||||
def _get_all_refer_contents(self, random_sample=True):
|
||||
"""获取所有Refer文件内容，可选择随机抽样
|
||||
"""获取所有Refer文件内容，可选择随机抽样文件内容
|
||||
|
||||
Args:
|
||||
random_sample: 是否进行随机抽样，默认为True
|
||||
random_sample: 是否对文件内容进行随机抽样，默认为True
|
||||
|
||||
Returns:
|
||||
str: 组合后的refer内容
|
||||
"""
|
||||
# 如果缓存中有内容，先使用缓存
|
||||
if self._refer_cache:
|
||||
refer_content_all = ""
|
||||
|
||||
# 获取所有refer文件名列表
|
||||
refer_files = list(self._refer_cache.keys())
|
||||
|
||||
if random_sample and len(refer_files) > 1:
|
||||
# 随机决定要使用的refer文件数量，至少使用一个
|
||||
sample_size = max(1, int(len(refer_files) * self._sample_rate))
|
||||
# 随机选择要使用的refer文件
|
||||
sampled_files = random.sample(refer_files, sample_size)
|
||||
logging.info(f"随机抽样了{sample_size}/{len(refer_files)}个refer文件: {sampled_files}")
|
||||
|
||||
for filename in sampled_files:
|
||||
content = self._refer_cache[filename]
|
||||
|
||||
# 判断是否为JSON格式
|
||||
is_json = False
|
||||
json_data = None
|
||||
if filename.lower().endswith('.json'):
|
||||
try:
|
||||
json_data = json.loads(content)
|
||||
is_json = True
|
||||
logging.info(f"检测到JSON格式的refer文件: {filename}")
|
||||
except json.JSONDecodeError:
|
||||
logging.warning(f"文件{filename}扩展名为.json但内容不是有效的JSON格式")
|
||||
|
||||
if is_json and json_data:
|
||||
# 处理JSON格式的refer文件
|
||||
title = json_data.get("title", "未命名参考资料")
|
||||
description = json_data.get("description", "")
|
||||
examples = json_data.get("examples", [])
|
||||
|
||||
refer_content_all += f"--- Refer File: {filename} ---\n"
|
||||
refer_content_all += f"标题: {title}\n"
|
||||
refer_content_all += f"描述: {description}\n\n"
|
||||
|
||||
if examples and random_sample:
|
||||
# 对examples数组进行抽样
|
||||
sample_examples_count = max(5, int(len(examples) * self._sample_rate))
|
||||
sampled_examples = random.sample(examples, sample_examples_count)
|
||||
logging.info(f"从文件{filename}中随机抽样了{sample_examples_count}/{len(examples)}个示例")
|
||||
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(sampled_examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
elif examples:
|
||||
# 不抽样，使用所有examples
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
|
||||
refer_content_all += "\n"
|
||||
else:
|
||||
# 处理普通文本格式的refer文件
|
||||
# 对文件内容进行行级抽样
|
||||
if random_sample and content:
|
||||
lines = content.split('\n')
|
||||
if len(lines) > 5: # 只对较长的内容进行抽样
|
||||
sample_lines_count = max(5, int(len(lines) * self._sample_rate))
|
||||
sampled_lines = random.sample(lines, sample_lines_count)
|
||||
# 保持原有顺序
|
||||
sampled_lines.sort(key=lambda line: lines.index(line))
|
||||
content = '\n'.join(sampled_lines)
|
||||
logging.info(f"从文件{filename}中随机抽样了{sample_lines_count}/{len(lines)}行内容")
|
||||
|
||||
refer_content_all += f"--- Refer File: {filename} ---\n{content}\n\n"
|
||||
else:
|
||||
# 不进行抽样，使用所有文件
|
||||
for filename, content in self._refer_cache.items():
|
||||
# 判断是否为JSON格式
|
||||
is_json = False
|
||||
json_data = None
|
||||
if filename.lower().endswith('.json'):
|
||||
try:
|
||||
json_data = json.loads(content)
|
||||
is_json = True
|
||||
except json.JSONDecodeError:
|
||||
logging.warning(f"文件{filename}扩展名为.json但内容不是有效的JSON格式")
|
||||
|
||||
if is_json and json_data:
|
||||
# 处理JSON格式的refer文件
|
||||
title = json_data.get("title", "未命名参考资料")
|
||||
description = json_data.get("description", "")
|
||||
examples = json_data.get("examples", [])
|
||||
|
||||
refer_content_all += f"--- Refer File: {filename} ---\n"
|
||||
refer_content_all += f"标题: {title}\n"
|
||||
refer_content_all += f"描述: {description}\n\n"
|
||||
refer_content_all += "示例:\n"
|
||||
|
||||
for idx, example in enumerate(examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
|
||||
refer_content_all += "\n"
|
||||
else:
|
||||
# 处理普通文本格式的refer文件
|
||||
refer_content_all += f"--- Refer File: {filename} ---\n{content}\n\n"
|
||||
|
||||
return refer_content_all
|
||||
import json
|
||||
|
||||
# 如果缓存为空，尝试从prompts_dir加载（向后兼容）
|
||||
# 初始化结果字符串
|
||||
refer_content_all = ""
|
||||
|
||||
# 准备处理所有文件
|
||||
all_refer_files = {}
|
||||
|
||||
# 1. 从缓存中获取文件
|
||||
if self._refer_cache:
|
||||
all_refer_files.update(self._refer_cache)
|
||||
|
||||
# 2. 从本地目录获取其他文件（如果有的话）
|
||||
if self.prompts_dir:
|
||||
refer_dir = os.path.join(self.prompts_dir, "Refer")
|
||||
if os.path.isdir(refer_dir):
|
||||
refer_files = [f for f in os.listdir(refer_dir) if os.path.isfile(os.path.join(refer_dir, f))]
|
||||
dir_files = [f for f in os.listdir(refer_dir) if os.path.isfile(os.path.join(refer_dir, f))]
|
||||
for refer_file in dir_files:
|
||||
if refer_file not in all_refer_files:
|
||||
refer_path = os.path.join(refer_dir, refer_file)
|
||||
content = ResourceLoader.load_file_content(refer_path)
|
||||
if content:
|
||||
all_refer_files[refer_file] = content
|
||||
# 保存到缓存
|
||||
self._refer_cache[refer_file] = content
|
||||
|
||||
if not all_refer_files:
|
||||
logging.warning("没有找到任何Refer文件")
|
||||
return refer_content_all
|
||||
|
||||
logging.info(f"找到{len(all_refer_files)}个Refer文件")
|
||||
|
||||
# 3. 处理所有文件
|
||||
for filename, content in all_refer_files.items():
|
||||
# 检查是否为JSON格式，尝试解析
|
||||
is_json = False
|
||||
json_data = None
|
||||
|
||||
if filename.lower().endswith('.json'):
|
||||
try:
|
||||
json_data = json.loads(content)
|
||||
is_json = True
|
||||
logging.info(f"成功解析JSON格式的refer文件: {filename}")
|
||||
except json.JSONDecodeError as e:
|
||||
logging.warning(f"文件{filename}扩展名为.json但内容不是有效的JSON格式: {str(e)}")
|
||||
logging.info(f"将以文本格式处理文件: {filename}")
|
||||
|
||||
# 添加文件头部信息
|
||||
refer_content_all += f"--- Refer File: {filename} ---\n"
|
||||
|
||||
if is_json and json_data:
|
||||
# 处理JSON格式文件
|
||||
title = json_data.get("title", "未命名参考资料")
|
||||
description = json_data.get("description", "")
|
||||
examples = json_data.get("examples", [])
|
||||
|
||||
if random_sample and len(refer_files) > 1:
|
||||
# 随机决定要使用的refer文件数量，至少使用一个
|
||||
sample_size = max(1, int(len(refer_files) * self._sample_rate))
|
||||
# 随机选择要使用的refer文件
|
||||
sampled_files = random.sample(refer_files, sample_size)
|
||||
logging.info(f"从目录随机抽样了{sample_size}/{len(refer_files)}个refer文件: {sampled_files}")
|
||||
|
||||
for refer_file in sampled_files:
|
||||
refer_path = os.path.join(refer_dir, refer_file)
|
||||
content = ResourceLoader.load_file_content(refer_path)
|
||||
refer_content_all += f"标题: {title}\n"
|
||||
refer_content_all += f"描述: {description}\n\n"
|
||||
|
||||
if examples:
|
||||
# 处理examples数组
|
||||
if random_sample and len(examples) > 10:
|
||||
# 对examples进行随机抽样
|
||||
sample_size = max(10, int(len(examples) * self._sample_rate))
|
||||
sampled_examples = random.sample(examples, sample_size)
|
||||
logging.info(f"从文件{filename}的JSON中随机抽样了{sample_size}/{len(examples)}个示例")
|
||||
|
||||
if content:
|
||||
# 判断是否为JSON格式
|
||||
is_json = False
|
||||
json_data = None
|
||||
if refer_file.lower().endswith('.json'):
|
||||
try:
|
||||
json_data = json.loads(content)
|
||||
is_json = True
|
||||
logging.info(f"从目录检测到JSON格式的refer文件: {refer_file}")
|
||||
except json.JSONDecodeError:
|
||||
logging.warning(f"目录中的文件{refer_file}扩展名为.json但内容不是有效的JSON格式")
|
||||
|
||||
if is_json and json_data:
|
||||
# 处理JSON格式的refer文件
|
||||
title = json_data.get("title", "未命名参考资料")
|
||||
description = json_data.get("description", "")
|
||||
examples = json_data.get("examples", [])
|
||||
|
||||
refer_content_all += f"--- Refer File: {refer_file} ---\n"
|
||||
refer_content_all += f"标题: {title}\n"
|
||||
refer_content_all += f"描述: {description}\n\n"
|
||||
|
||||
if examples and random_sample:
|
||||
# 对examples数组进行抽样
|
||||
sample_examples_count = max(5, int(len(examples) * self._sample_rate))
|
||||
sampled_examples = random.sample(examples, sample_examples_count)
|
||||
logging.info(f"从文件{refer_file}中随机抽样了{sample_examples_count}/{len(examples)}个示例")
|
||||
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(sampled_examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
elif examples:
|
||||
# 不抽样，使用所有examples
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
|
||||
refer_content_all += "\n"
|
||||
else:
|
||||
# 对文件内容进行行级抽样
|
||||
if random_sample:
|
||||
lines = content.split('\n')
|
||||
if len(lines) > 5: # 只对较长的内容进行抽样
|
||||
sample_lines_count = max(5, int(len(lines) * self._sample_rate))
|
||||
sampled_lines = random.sample(lines, sample_lines_count)
|
||||
# 保持原有顺序
|
||||
sampled_lines.sort(key=lambda line: lines.index(line))
|
||||
content = '\n'.join(sampled_lines)
|
||||
logging.info(f"从目录文件{refer_file}中随机抽样了{sample_lines_count}/{len(lines)}行内容")
|
||||
|
||||
refer_content_all += f"--- Refer File: {refer_file} ---\n{content}\n\n"
|
||||
|
||||
# 保存到缓存
|
||||
self._refer_cache[refer_file] = content
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(sampled_examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
else:
|
||||
# 不进行抽样或examples数量较少，使用全部
|
||||
refer_content_all += "示例:\n"
|
||||
for idx, example in enumerate(examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
else:
|
||||
# 处理普通文本文件
|
||||
if random_sample:
|
||||
lines = content.split('\n')
|
||||
if len(lines) > 10: # 只对较长的内容进行抽样
|
||||
sample_size = max(10, int(len(lines) * self._sample_rate))
|
||||
sampled_lines = random.sample(lines, sample_size)
|
||||
# 保持原有顺序
|
||||
sampled_lines.sort(key=lambda line: lines.index(line))
|
||||
sampled_content = '\n'.join(sampled_lines)
|
||||
logging.info(f"从文件{filename}中随机抽样了{sample_size}/{len(lines)}行内容")
|
||||
refer_content_all += f"{sampled_content}\n"
|
||||
else:
|
||||
# 内容较短，不进行抽样
|
||||
refer_content_all += f"{content}\n"
|
||||
else:
|
||||
# 不进行抽样，使用所有文件
|
||||
for refer_file in refer_files:
|
||||
refer_path = os.path.join(refer_dir, refer_file)
|
||||
content = ResourceLoader.load_file_content(refer_path)
|
||||
|
||||
if content:
|
||||
# 判断是否为JSON格式
|
||||
is_json = False
|
||||
json_data = None
|
||||
if refer_file.lower().endswith('.json'):
|
||||
try:
|
||||
json_data = json.loads(content)
|
||||
is_json = True
|
||||
except json.JSONDecodeError:
|
||||
logging.warning(f"目录中的文件{refer_file}扩展名为.json但内容不是有效的JSON格式")
|
||||
|
||||
if is_json and json_data:
|
||||
# 处理JSON格式的refer文件
|
||||
title = json_data.get("title", "未命名参考资料")
|
||||
description = json_data.get("description", "")
|
||||
examples = json_data.get("examples", [])
|
||||
|
||||
refer_content_all += f"--- Refer File: {refer_file} ---\n"
|
||||
refer_content_all += f"标题: {title}\n"
|
||||
refer_content_all += f"描述: {description}\n\n"
|
||||
refer_content_all += "示例:\n"
|
||||
|
||||
for idx, example in enumerate(examples, 1):
|
||||
content_text = example.get("content", "")
|
||||
refer_content_all += f"{idx}. {content_text}\n"
|
||||
|
||||
refer_content_all += "\n"
|
||||
else:
|
||||
# 处理普通文本格式的refer文件
|
||||
refer_content_all += f"--- Refer File: {refer_file} ---\n{content}\n\n"
|
||||
|
||||
# 保存到缓存
|
||||
self._refer_cache[refer_file] = content
|
||||
# 不进行抽样
|
||||
refer_content_all += f"{content}\n"
|
||||
|
||||
# 添加文件之间的分隔
|
||||
refer_content_all += "\n"
|
||||
|
||||
return refer_content_all
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user