diff --git a/SimSun.ttf b/SimSun.ttf new file mode 100644 index 00000000..e0115abe Binary files /dev/null and b/SimSun.ttf differ diff --git a/crazy_functional.py b/crazy_functional.py index 975f61b8..5c96dbb4 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -50,6 +50,7 @@ def get_crazy_functions(): from crazy_functions.SourceCode_Comment import 注释Python项目 from crazy_functions.SourceCode_Comment_Wrap import SourceCodeComment_Wrap from crazy_functions.VideoResource_GPT import 多媒体任务 + from crazy_functions.PDF_Convert import 解析PDF文档 function_plugins = { "多媒体智能体": { @@ -378,6 +379,14 @@ def get_crazy_functions(): "Info": "PDF翻译中文,并重新编译PDF | 输入参数为路径", "Function": HotReload(PDF翻译中文并重新编译PDF), # 当注册Class后,Function旧接口仅会在“虚空终端”中起作用 "Class": PDF_Localize # 新一代插件需要注册Class + }, + "解析PDF文档": { + "Group": "学术", + "Color": "stop", + "AsButton": False, + "AdvancedArgs": True, + "Info": "PDF解析", + "Function": HotReload(解析PDF文档), # 当注册Class后,Function旧接口仅会在“虚空终端”中起作用 } } diff --git a/crazy_functions/PDF_Convert.py b/crazy_functions/PDF_Convert.py new file mode 100644 index 00000000..d01fed82 --- /dev/null +++ b/crazy_functions/PDF_Convert.py @@ -0,0 +1,78 @@ +from toolbox import CatchException, report_exception, get_log_folder, gen_time_str +from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion +from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency +from crazy_functions.crazy_utils import read_and_clean_pdf_text +from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url, translate_pdf +from shared_utils.colorful import * +import copy +import os +import math +import logging + + +@CatchException +def 解析PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request): + + disable_auto_promotion(chatbot) + # 基本信息:功能、贡献者 + chatbot.append([ + "函数插件功能?", + "解析PDF文档。函数插件贡献者: Xunge-Jiang"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # 清空历史,以免输入溢出 + history = [] + + from crazy_functions.crazy_utils import get_files_from_everything + success, file_manifest, project_folder = get_files_from_everything(txt, type='.pdf') + + # success_md, file_manifest_md, _ = get_files_from_everything(txt, type='.md') + # success = success or success_md + # file_manifest += file_manifest_md + chatbot.append(["文件列表:", ", ".join([e.split('/')[-1] for e in file_manifest])]) + yield from update_ui(chatbot=chatbot, history=history) + # 检测输入参数,如没有给定输入参数,直接退出 + if not success: + if txt == "": txt = '空空如也的输入栏' + + # 如果没找到任何文件 + if len(file_manifest) == 0: + report_exception(chatbot, history, + a=f"解析项目: {txt}", b=f"找不到任何.pdf拓展名的文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + # 开始正式执行任务 + yield from 解析PDF_基于MinerU(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt) + + +def 解析PDF_基于MinerU(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): + import copy + import tiktoken + TOKEN_LIMIT_PER_FRAGMENT = 1024 + generated_conclusion_files = [] + generated_html_files = [] + DST_LANG = "中文" + from crazy_functions.crazy_utils import mineru_interface + from crazy_functions.pdf_fns.report_gen_html import construct_html + mineru_handle = mineru_interface() + for index, fp in enumerate(file_manifest): + if fp.endswith('pdf'): + chatbot.append(["当前进度:", f"正在解析论文,请稍候。"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + fpp = yield from mineru_handle.mineru_parse_pdf(fp, chatbot, history) + promote_file_to_downloadzone(fpp, rename_file=os.path.basename(fpp)+'.mineru.md', chatbot=chatbot) + else: + chatbot.append(["当前论文无需解析:", fp]); yield from update_ui(chatbot=chatbot, history=history) + fpp = fp + # with open(fpp, 'r', encoding='utf8') as f: + # article_content = f.readlines() + # article_dict = markdown_to_dict(article_content) + # logging.info(article_dict) + # yield from translate_pdf(article_dict, llm_kwargs, chatbot, fp, generated_conclusion_files, TOKEN_LIMIT_PER_FRAGMENT, DST_LANG) + + # chatbot.append(("给出输出文件清单", str(generated_conclusion_files))) + + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 5c8776ac..22eee624 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -632,6 +632,80 @@ class nougat_interface(): return res[0] +@Singleton +class mineru_interface(): + def __init__(self): + self.threadLock = threading.Lock() + + def mineru_with_timeout(self, command, cwd, timeout=3600, conda_env="mineru", conda_init="/home/jiang/anaconda3/etc/profile.d/conda.sh"): + import subprocess + from toolbox import ProxyNetworkActivate + logger.info(f"正在执行命令 {command} 在 Conda 环境 '{conda_env}' 中") + + import shlex + # 确保命令中的参数安全(转义空格等特殊字符) + safe_command = ' '.join([shlex.quote(arg) for arg in command]) + print('safe_command', safe_command) + + # 构造激活 Conda 环境的命令 + activate_command = ( + f"source {conda_init} && " + f"conda activate {conda_env} && " + f"CUDA_VISIBLE_DEVICES=1 {safe_command}" + ) + + try: + with ProxyNetworkActivate("MinerU"): + process = subprocess.Popen( + activate_command, + shell=True, + cwd=cwd, + env=os.environ, + executable="/bin/bash" # 指定使用 Bash 执行命令 + ) + stdout, stderr = process.communicate(timeout=timeout) + + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + logger.error("Process timed out!") + return False + + # 检查返回码 + if process.returncode != 0: + logger.error(f"Command failed with return code {process.returncode}: {stderr.decode()}") + return False + return True + + + def mineru_parse_pdf(self, fp, chatbot, history): + from toolbox import update_ui_lastest_msg + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度:正在排队, 等待线程锁...", + chatbot=chatbot, history=history, delay=0) + self.threadLock.acquire() + import glob, threading, os + from toolbox import get_log_folder, gen_time_str + dst = os.path.join(get_log_folder(plugin_name='mineru'), gen_time_str()) + os.makedirs(dst) + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度:正在加载MinerU... ", + chatbot=chatbot, history=history, delay=0) + command = ['magic-pdf', '-p', os.path.abspath(fp), '-o', os.path.abspath(dst), ] + self.mineru_with_timeout(command, cwd=os.getcwd(), timeout=3600) + + pdf_name = os.path.basename(fp) + # 去掉后缀 + name_without_ext = os.path.splitext(pdf_name)[0] + + res = glob.glob(os.path.join(dst, name_without_ext, 'auto', '*.md')) + + if len(res) == 0: + self.threadLock.release() + raise RuntimeError("MinerU解析论文失败。") + self.threadLock.release() + return res[0] + def try_install_deps(deps, reload_m=[]): diff --git a/shared_utils/convert_pdf.sh b/shared_utils/convert_pdf.sh new file mode 100644 index 00000000..b3a764b9 --- /dev/null +++ b/shared_utils/convert_pdf.sh @@ -0,0 +1,28 @@ +#!/bin/bash +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +FILE_PATH=$(realpath "$1") + +if [ ! -f "$FILE_PATH" ]; then + echo "File not found: $FILE_PATH" + exit 1 +fi + +conda activate mineru + +if ! command -v magic-pdf &> /dev/null; then + echo "magic-pdf could not be found in the current conda environment" + exit 1 +fi + +magic-pdf pdf-command --pdf "$FILE_PATH" --inside_model true + +if [ $? -eq 0 ]; then + echo "PDF conversion successful" +else + echo "PDF conversion failed" + exit 1 +fi