from vllm import LLM, SamplingParams from transformers import AutoTokenizer from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset from dataclasses import dataclass, field from typing import Optional, ClassVar, List, Tuple import numpy as np from numpy.typing import NDArray from PIL import Image import librosa import librosa.util import cv2 import os from openai import OpenAI import base64 # import ray # 注释掉Ray导入 # # 使用本地模式初始化Ray，避免分布式通信问题 # ray.init(local_mode=True, ignore_reinit_error=True) # 设置环境变量，禁用在线检查 os.environ["HF_DATASETS_OFFLINE"] = "1" os.environ["TRANSFORMERS_OFFLINE"] = "1" os.environ["HF_HUB_OFFLINE"] = "1" @dataclass(frozen=True) class VideoAsset: name: str num_frames: int = -1 _NAME_TO_FILE: ClassVar[dict[str, str]] = { "baby_reading": "sample_demo_1.mp4", } @property def filename(self) -> str: return self._NAME_TO_FILE[self.name] @property def pil_images(self) -> list[Image.Image]: video_path = download_video_asset(self.filename) return video_to_pil_images_list(video_path, self.num_frames) @property def np_ndarrays(self) -> NDArray: video_path = download_video_asset(self.filename) return video_to_ndarrays(video_path, self.num_frames) def get_audio(self, sampling_rate: Optional[float] = None) -> NDArray: video_path = download_video_asset(self.filename) return librosa.load(video_path, sr=sampling_rate)[0] @dataclass(frozen=True) class LocalVideoAsset: local_path: str name: str = "local_video" num_frames: int = -1 @property def filename(self) -> str: return self.local_path @property def pil_images(self) -> list[Image.Image]: return video_to_pil_images_list(self.filename, self.num_frames) @property def np_ndarrays(self) -> NDArray: return video_to_ndarrays(self.filename, self.num_frames) def get_audio(self, sampling_rate: Optional[float] = None) -> NDArray: try: if not os.path.exists(self.filename): print(f"音频文件不存在: {self.filename}") return np.zeros(1) # 返回空数组 return librosa.load(self.filename, sr=sampling_rate)[0] except Exception as e: print(f"加载音频时出错: {e}") return np.zeros(1) # 出错时返回空数组 # 辅助函数实现 def download_video_asset(filename: str) -> str: # 如果路径是绝对路径或相对路径，直接返回 if filename.startswith("/") or filename.startswith("./"): return filename # 否则执行下载逻辑（原实现） return f"/path/to/downloaded/{filename}" def video_to_pil_images_list(video_path: str, num_frames: int) -> list[Image.Image]: """将视频转换为PIL图像列表""" if not os.path.exists(video_path): print(f"视频文件不存在: {video_path}") return [] cap = cv2.VideoCapture(video_path) if not cap.isOpened(): print(f"无法打开视频: {video_path}") return [] # 获取视频帧数 total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fps = cap.get(cv2.CAP_PROP_FPS) duration = total_frames / fps if fps > 0 else 0 print(f"视频信息: 总帧数={total_frames}, FPS={fps:.2f}, 时长={duration:.2f}秒") # 如果指定了帧数，设置采样间隔；否则读取所有帧 if num_frames > 0 and num_frames < total_frames: frame_interval = total_frames / num_frames print(f"将提取 {num_frames} 帧，采样间隔为每 {frame_interval:.2f} 帧") else: frame_interval = 1 num_frames = total_frames print(f"将提取所有 {total_frames} 帧") pil_images = [] frame_count = 0 success = True last_progress = -1 while success and len(pil_images) < num_frames: # 读取下一帧 success, frame = cap.read() if not success: break # 按间隔采样帧 if frame_count % max(1, int(frame_interval)) == 0: # OpenCV使用BGR，转为RGB rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # 转为PIL图像 pil_image = Image.fromarray(rgb_frame) pil_images.append(pil_image) # 显示进度（每10%显示一次） progress = int(len(pil_images) / num_frames * 10) if progress > last_progress: print(f"提取进度: {len(pil_images)}/{num_frames} ({len(pil_images)/num_frames*100:.1f}%)") last_progress = progress frame_count += 1 cap.release() print(f"从视频中共提取了 {len(pil_images)} 帧") return pil_images def video_to_ndarrays(video_path: str, num_frames: int) -> NDArray: """将视频转换为NumPy数组""" pil_images = video_to_pil_images_list(video_path, num_frames) if not pil_images: print(f"未能从视频中提取帧: {video_path}") return np.zeros((1, 224, 224, 3)) # 将PIL图像列表转换为NumPy数组 arrays = [] for img in pil_images: # 调整图像大小为统一尺寸 img_resized = img.resize((224, 224)) # 转换为NumPy数组 arr = np.array(img_resized) arrays.append(arr) # 堆叠为单个NumPy数组，形状为[num_frames, height, width, channels] stacked_array = np.stack(arrays, axis=0) print(f"NumPy数组形状: {stacked_array.shape}") return stacked_array def encode_image(image_path): with open(image_path, "rb") as image_file: return base64.b64encode(image_file.read()).decode("utf-8") def encode_audio(audio_path): with open(audio_path, "rb") as audio_file: return base64.b64encode(audio_file.read()).decode("utf-8") # 使用示例 if __name__ == "__main__": # 使用本地视频资源 # local_video = LocalVideoAsset( # local_path="/root/autodl-tmp/hot_video_analyse/source/sample_demo_1.mp4", # num_frames = -1 # 限制帧数以加快测试速度 # ) # print("本地资源:", local_video.filename) # 获取PIL图像列表（实际会调用download_video_asset和转换函数） # pil_images = local_video.pil_images # print("PIL图像数量:", len(pil_images)) # print(pil_images[0]) # 获取NumPy数组 # print("\n=== 加载视频帧 ===") # np_arrays = local_video.np_ndarrays # print(f"视频数组形状: {np_arrays.shape}") # scene_change_arrays = np.load('/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_arrays.npy') # print(f"场景变化帧数组形状: {scene_change_arrays.shape}") # 获取音频数据 # audio = local_video.get_audio(sampling_rate=16000) # print("音频数据形状:", audio.shape) # print(type(audio)) base64_audio = encode_audio("/root/autodl-tmp/hot_video_analyse/source/transcription/sample_demo_1_audio.wav") client = OpenAI( # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", api_key="EMPTY", base_url="http://localhost:8000/v1", ) base64_image_0 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000034_t1.13s_ssim0.5673.jpg") base64_image_1 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000117_t3.90s_ssim0.1989.jpg") base64_image_2 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000119_t3.97s_ssim0.2138.jpg") base64_image_3 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000140_t4.67s_ssim0.5160.jpg") base64_image_4 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000160_t5.33s_ssim0.4934.jpg") base64_image_5 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000180_t6.00s_ssim0.3577.jpg") base64_image_6 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000201_t6.70s_ssim0.3738.jpg") base64_image_7 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000222_t7.40s_ssim0.6104.jpg") base64_image_8 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000243_t8.10s_ssim0.5099.jpg") base64_image_9 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000261_t8.70s_ssim0.4735.jpg") base64_image_10 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000281_t9.37s_ssim0.2703.jpg") base64_image_11 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000301_t10.03s_ssim0.2772.jpg") base64_image_12 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000321_t10.70s_ssim0.3721.jpg") base64_image_13 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000341_t11.37s_ssim0.3700.jpg") base64_image_14 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000361_t12.03s_ssim0.3494.jpg") base64_image_15 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000382_t12.73s_ssim0.3423.jpg") base64_image_16 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000402_t13.40s_ssim0.3252.jpg") base64_image_17 = encode_image("/root/autodl-tmp/hot_video_analyse/source/scene_change/scene_change_frames/scene_change_000421_t14.03s_ssim0.2086.jpg") completion = client.chat.completions.create( model="/root/autodl-tmp/llm", # 使用vLLM服务中实际注册的模型ID messages=[ { "role": "user", "content": [ # 将视频帧作为多个图像处理 { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_1}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_2}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_3}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_4}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_5}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_6}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_7}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_8}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_9}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_10}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_11}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_12}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_13}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_14}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_15}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_16}"} }, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_17}"} }, # 音频输入 { "type": "input_audio", "input_audio": { "data": base64_audio, "format": "wav" } }, # 文本提示 { "type": "text", "text": "描述一下关于这个视频的浅层特征" }, ], } ], # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] modalities=["text"], #audio={"voice": "Cherry", "format": "wav"}, # stream 必须设置为 True，否则会报错 stream=True, stream_options={"include_usage": True}, ) #"形容一下这段音频。这些图像是一个视频的连续帧，请结合音频描述这个视频的具体过程和内容。并解释一下这个视频吸引人眼球的原因" #"识别这段音频，并描述一下这段音频的内容。" # 收集完整的响应内容 full_response = "" usage_info = None print("正在生成回复...") for chunk in completion: if chunk.choices: delta = chunk.choices[0].delta if delta.content: # 拼接内容 full_response += delta.content # 实时显示（可选） print(delta.content, end='', flush=True) else: # 保存使用情况信息 usage_info = chunk.usage # 输出完整的响应 print("\n" + "="*50) print("完整回复:") print("="*50) print(full_response) # 输出使用情况信息 if usage_info: print("\n" + "="*50) print("使用情况:") print("="*50) print(usage_info)