support_mineru
This commit is contained in:
xunge
parent
7eeab9e376
commit
a6de631d15
Binary file not shown.
|
|
@ -50,6 +50,7 @@ def get_crazy_functions():
|
|||
from crazy_functions.SourceCode_Comment import 注释Python项目
|
||||
from crazy_functions.SourceCode_Comment_Wrap import SourceCodeComment_Wrap
|
||||
from crazy_functions.VideoResource_GPT import 多媒体任务
|
||||
from crazy_functions.PDF_Convert import 解析PDF文档
|
||||
|
||||
function_plugins = {
|
||||
"多媒体智能体": {
|
||||
|
|
@ -378,6 +379,14 @@ def get_crazy_functions():
|
|||
"Info": "PDF翻译中文，并重新编译PDF | 输入参数为路径",
|
||||
"Function": HotReload(PDF翻译中文并重新编译PDF), # 当注册Class后，Function旧接口仅会在“虚空终端”中起作用
|
||||
"Class": PDF_Localize # 新一代插件需要注册Class
|
||||
},
|
||||
"解析PDF文档": {
|
||||
"Group": "学术",
|
||||
"Color": "stop",
|
||||
"AsButton": False,
|
||||
"AdvancedArgs": True,
|
||||
"Info": "PDF解析",
|
||||
"Function": HotReload(解析PDF文档), # 当注册Class后，Function旧接口仅会在“虚空终端”中起作用
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
from toolbox import CatchException, report_exception, get_log_folder, gen_time_str
|
||||
from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion
|
||||
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
from crazy_functions.crazy_utils import read_and_clean_pdf_text
|
||||
from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url, translate_pdf
|
||||
from shared_utils.colorful import *
|
||||
import copy
|
||||
import os
|
||||
import math
|
||||
import logging
|
||||
|
||||
|
||||
@CatchException
|
||||
def 解析PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
|
||||
|
||||
disable_auto_promotion(chatbot)
|
||||
# 基本信息：功能、贡献者
|
||||
chatbot.append([
|
||||
"函数插件功能？",
|
||||
"解析PDF文档。函数插件贡献者: Xunge-Jiang"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 清空历史，以免输入溢出
|
||||
history = []
|
||||
|
||||
from crazy_functions.crazy_utils import get_files_from_everything
|
||||
success, file_manifest, project_folder = get_files_from_everything(txt, type='.pdf')
|
||||
|
||||
# success_md, file_manifest_md, _ = get_files_from_everything(txt, type='.md')
|
||||
# success = success or success_md
|
||||
# file_manifest += file_manifest_md
|
||||
chatbot.append(["文件列表：", ", ".join([e.split('/')[-1] for e in file_manifest])])
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
# 检测输入参数，如没有给定输入参数，直接退出
|
||||
if not success:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
|
||||
# 如果没找到任何文件
|
||||
if len(file_manifest) == 0:
|
||||
report_exception(chatbot, history,
|
||||
a=f"解析项目: {txt}", b=f"找不到任何.pdf拓展名的文件: {txt}")
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
# 开始正式执行任务
|
||||
yield from 解析PDF_基于MinerU(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt)
|
||||
|
||||
|
||||
def 解析PDF_基于MinerU(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
||||
import copy
|
||||
import tiktoken
|
||||
TOKEN_LIMIT_PER_FRAGMENT = 1024
|
||||
generated_conclusion_files = []
|
||||
generated_html_files = []
|
||||
DST_LANG = "中文"
|
||||
from crazy_functions.crazy_utils import mineru_interface
|
||||
from crazy_functions.pdf_fns.report_gen_html import construct_html
|
||||
mineru_handle = mineru_interface()
|
||||
for index, fp in enumerate(file_manifest):
|
||||
if fp.endswith('pdf'):
|
||||
chatbot.append(["当前进度：", f"正在解析论文，请稍候。"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
fpp = yield from mineru_handle.mineru_parse_pdf(fp, chatbot, history)
|
||||
promote_file_to_downloadzone(fpp, rename_file=os.path.basename(fpp)+'.mineru.md', chatbot=chatbot)
|
||||
else:
|
||||
chatbot.append(["当前论文无需解析：", fp]); yield from update_ui(chatbot=chatbot, history=history)
|
||||
fpp = fp
|
||||
# with open(fpp, 'r', encoding='utf8') as f:
|
||||
# article_content = f.readlines()
|
||||
# article_dict = markdown_to_dict(article_content)
|
||||
# logging.info(article_dict)
|
||||
# yield from translate_pdf(article_dict, llm_kwargs, chatbot, fp, generated_conclusion_files, TOKEN_LIMIT_PER_FRAGMENT, DST_LANG)
|
||||
|
||||
# chatbot.append(("给出输出文件清单", str(generated_conclusion_files)))
|
||||
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
|
|
@ -632,6 +632,80 @@ class nougat_interface():
|
|||
return res[0]
|
||||
|
||||
|
||||
@Singleton
|
||||
class mineru_interface():
|
||||
def __init__(self):
|
||||
self.threadLock = threading.Lock()
|
||||
|
||||
def mineru_with_timeout(self, command, cwd, timeout=3600, conda_env="mineru", conda_init="/home/jiang/anaconda3/etc/profile.d/conda.sh"):
|
||||
import subprocess
|
||||
from toolbox import ProxyNetworkActivate
|
||||
logger.info(f"正在执行命令 {command} 在 Conda 环境 '{conda_env}' 中")
|
||||
|
||||
import shlex
|
||||
# 确保命令中的参数安全（转义空格等特殊字符）
|
||||
safe_command = ' '.join([shlex.quote(arg) for arg in command])
|
||||
print('safe_command', safe_command)
|
||||
|
||||
# 构造激活 Conda 环境的命令
|
||||
activate_command = (
|
||||
f"source {conda_init} && "
|
||||
f"conda activate {conda_env} && "
|
||||
f"CUDA_VISIBLE_DEVICES=1 {safe_command}"
|
||||
)
|
||||
|
||||
try:
|
||||
with ProxyNetworkActivate("MinerU"):
|
||||
process = subprocess.Popen(
|
||||
activate_command,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
env=os.environ,
|
||||
executable="/bin/bash" # 指定使用 Bash 执行命令
|
||||
)
|
||||
stdout, stderr = process.communicate(timeout=timeout)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
stdout, stderr = process.communicate()
|
||||
logger.error("Process timed out!")
|
||||
return False
|
||||
|
||||
# 检查返回码
|
||||
if process.returncode != 0:
|
||||
logger.error(f"Command failed with return code {process.returncode}: {stderr.decode()}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def mineru_parse_pdf(self, fp, chatbot, history):
|
||||
from toolbox import update_ui_lastest_msg
|
||||
|
||||
yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度：正在排队, 等待线程锁...",
|
||||
chatbot=chatbot, history=history, delay=0)
|
||||
self.threadLock.acquire()
|
||||
import glob, threading, os
|
||||
from toolbox import get_log_folder, gen_time_str
|
||||
dst = os.path.join(get_log_folder(plugin_name='mineru'), gen_time_str())
|
||||
os.makedirs(dst)
|
||||
|
||||
yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度：正在加载MinerU... ",
|
||||
chatbot=chatbot, history=history, delay=0)
|
||||
command = ['magic-pdf', '-p', os.path.abspath(fp), '-o', os.path.abspath(dst), ]
|
||||
self.mineru_with_timeout(command, cwd=os.getcwd(), timeout=3600)
|
||||
|
||||
pdf_name = os.path.basename(fp)
|
||||
# 去掉后缀
|
||||
name_without_ext = os.path.splitext(pdf_name)[0]
|
||||
|
||||
res = glob.glob(os.path.join(dst, name_without_ext, 'auto', '*.md'))
|
||||
|
||||
if len(res) == 0:
|
||||
self.threadLock.release()
|
||||
raise RuntimeError("MinerU解析论文失败。")
|
||||
self.threadLock.release()
|
||||
return res[0]
|
||||
|
||||
|
||||
|
||||
def try_install_deps(deps, reload_m=[]):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Usage: $0 <path-to-pdf-file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILE_PATH=$(realpath "$1")
|
||||
|
||||
if [ ! -f "$FILE_PATH" ]; then
|
||||
echo "File not found: $FILE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
conda activate mineru
|
||||
|
||||
if ! command -v magic-pdf &> /dev/null; then
|
||||
echo "magic-pdf could not be found in the current conda environment"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
magic-pdf pdf-command --pdf "$FILE_PATH" --inside_model true
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "PDF conversion successful"
|
||||
else
|
||||
echo "PDF conversion failed"
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Reference in New Issue