重构 LaTeX 处理和注释移除逻辑。
- 为部分函数添加类型提示 - 简化 LaTeX 环境处理 - 改进注释移除逻辑 - 修改空格缩减工具 - 修复小错误和拼写问题
This commit is contained in:
QianCY@aliyun
parent
1213ef19e5
commit
e974c6fb1d
|
|
@ -16,7 +16,7 @@ from crazy_functions.latex_fns.latex_pickle_io import objdump, objload
|
|||
pj = os.path.join
|
||||
|
||||
|
||||
def split_subprocess(txt, project_folder, return_dict, opts):
|
||||
def split_subprocess(txt: str, project_folder: str, return_dict, opts):
|
||||
"""
|
||||
break down latex file to a linked list,
|
||||
each node use a preserve flag to indicate whether it should
|
||||
|
|
@ -30,26 +30,24 @@ def split_subprocess(txt, project_folder, return_dict, opts):
|
|||
text, mask = set_forbidden_text(text, mask, r"^(.*?)\\begin{document}", re.DOTALL)
|
||||
# 吸收iffalse注释
|
||||
text, mask = set_forbidden_text(text, mask, r"\\iffalse(.*?)\\fi", re.DOTALL)
|
||||
# 吸收在42行以内的begin-end组合
|
||||
text, mask = set_forbidden_text_begin_end(text, mask, r"\\begin\{([a-z\*]*)\}(.*?)\\end\{\1\}", re.DOTALL, limit_n_lines=42)
|
||||
# 吸收在42行以内的begin-end组合 (为什么要吸收啊... 我的theorem基本都被剔除了)
|
||||
# text, mask = set_forbidden_text_begin_end(text, mask, r"\\begin\{([a-z\*]*)\}(.*?)\\end\{\1\}", re.DOTALL, limit_n_lines=42)
|
||||
# 吸收匿名公式
|
||||
text, mask = set_forbidden_text(text, mask, [ r"\$\$([^$]+)\$\$", r"\\\[.*?\\\]" ], re.DOTALL)
|
||||
# 吸收其他杂项
|
||||
text, mask = set_forbidden_text(text, mask, [ r"\\section\{(.*?)\}", r"\\section\*\{(.*?)\}", r"\\subsection\{(.*?)\}", r"\\subsubsection\{(.*?)\}" ])
|
||||
text, mask = set_forbidden_text(text, mask, [ r"\\section\*?\{(.*?)\}", r"\\subsection\{(.*?)\}", r"\\subsubsection\{(.*?)\}" ])
|
||||
text, mask = set_forbidden_text(text, mask, [ r"\\bibliography\{(.*?)\}", r"\\bibliographystyle\{(.*?)\}" ])
|
||||
text, mask = set_forbidden_text(text, mask, r"\\begin\{thebibliography\}.*?\\end\{thebibliography\}", re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, r"\\begin\{lstlisting\}(.*?)\\end\{lstlisting\}", re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, r"\\begin\{wraptable\}(.*?)\\end\{wraptable\}", re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, r"\\begin\{algorithm\}(.*?)\\end\{algorithm\}", re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{wrapfigure\}(.*?)\\end\{wrapfigure\}", r"\\begin\{wrapfigure\*\}(.*?)\\end\{wrapfigure\*\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{figure\}(.*?)\\end\{figure\}", r"\\begin\{figure\*\}(.*?)\\end\{figure\*\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{multline\}(.*?)\\end\{multline\}", r"\\begin\{multline\*\}(.*?)\\end\{multline\*\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{table\}(.*?)\\end\{table\}", r"\\begin\{table\*\}(.*?)\\end\{table\*\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{minipage\}(.*?)\\end\{minipage\}", r"\\begin\{minipage\*\}(.*?)\\end\{minipage\*\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{align\*\}(.*?)\\end\{align\*\}", r"\\begin\{align\}(.*?)\\end\{align\}"], re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\begin\{equation\}(.*?)\\end\{equation\}", r"\\begin\{equation\*\}(.*?)\\end\{equation\*\}"], re.DOTALL)
|
||||
for environment in [
|
||||
"thebibliography",
|
||||
"wraptable", "table", "tabular",
|
||||
"wrapfigure", "figure", "tikzpicture",
|
||||
"lstlisting", "algorithm", "algorithmic",
|
||||
"multline", "align", "equation",
|
||||
"minipage",
|
||||
]:
|
||||
text, mask = set_forbidden_text(text, mask, rf"\\begin\{{({environment}\*?)\}}(.*?)\\end\{{\1\}}", re.DOTALL)
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\includepdf\[(.*?)\]\{(.*?)\}", r"\\clearpage", r"\\newpage", r"\\appendix", r"\\tableofcontents", r"\\include\{(.*?)\}"])
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\vspace\{(.*?)\}", r"\\hspace\{(.*?)\}", r"\\label\{(.*?)\}", r"\\begin\{(.*?)\}", r"\\end\{(.*?)\}", r"\\item "])
|
||||
text, mask = set_forbidden_text(text, mask, [r"\\vspace\{(.*?)\}", r"\\hspace\{(.*?)\}", r"\\label\{(.*?)\}", r"\\begin\{(.*?)\}", r"\\end\{(.*?)\}", r"\\item\s?"])
|
||||
text, mask = set_forbidden_text_careful_brace(text, mask, r"\\hl\{(.*?)\}", re.DOTALL)
|
||||
# reverse 操作必须放在最后
|
||||
text, mask = reverse_forbidden_text_careful_brace(text, mask, r"\\caption\{(.*?)\}", re.DOTALL, forbid_wrapper=True)
|
||||
|
|
@ -92,18 +90,25 @@ class LatexPaperSplit():
|
|||
self.msg = "*{\\scriptsize\\textbf{警告：该PDF由GPT-Academic开源项目调用大语言模型+Latex翻译插件一键生成，" + \
|
||||
"版权归原文作者所有。翻译内容可靠性无保障，请仔细鉴别并以原文为准。" + \
|
||||
"项目Github地址 \\url{https://github.com/binary-husky/gpt_academic/}。"
|
||||
# 请您不要删除或修改这行警告，除非您是论文的原作者（如果您是论文原作者，欢迎加REAME中的QQ联系开发者）
|
||||
# 请您不要删除或修改这行警告，除非您是论文的原作者（如果您是论文原作者，欢迎加README中的QQ联系开发者）
|
||||
self.msg_declare = "为了防止大语言模型的意外谬误产生扩散影响，禁止移除或修改此警告。}}\\\\"
|
||||
self.title = "unknown"
|
||||
self.abstract = "unknown"
|
||||
|
||||
def read_title_and_abstract(self, txt):
|
||||
def shrink_spaces(self, text: str):
|
||||
"""删除空白字符 包含\\n, \\\\, \\t, 空格
|
||||
|
||||
残存BUG: 未考虑\\\\[length]"""
|
||||
text = text.replace('\\\\', ' ')
|
||||
return re.sub(r'\s+', ' ', text)
|
||||
|
||||
def read_title_and_abstract(self, txt: str):
|
||||
try:
|
||||
title, abstract = find_title_and_abs(txt)
|
||||
if title is not None:
|
||||
self.title = title.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '')
|
||||
self.title = self.shrink_spaces(title)
|
||||
if abstract is not None:
|
||||
self.abstract = abstract.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '')
|
||||
self.abstract = self.shrink_spaces(abstract)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
|
@ -147,7 +152,7 @@ class LatexPaperSplit():
|
|||
return result_string
|
||||
|
||||
|
||||
def split(self, txt, project_folder, opts):
|
||||
def split(self, txt: str, project_folder: str, opts: list):
|
||||
"""
|
||||
break down latex file to a linked list,
|
||||
each node use a preserve flag to indicate whether it should
|
||||
|
|
@ -215,7 +220,7 @@ class LatexPaperFileGroup():
|
|||
return manifest
|
||||
|
||||
|
||||
def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None, opts=[]):
|
||||
def Latex精细分解与转化(file_manifest: list[str], project_folder: str, llm_kwargs: dict[str,str], plugin_kwargs: dict[str,str], chatbot, history:list[str], system_prompt, mode='proofread', switch_prompt=None, opts=[]):
|
||||
import time, os, re
|
||||
from ..crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
from .latex_actions import LatexPaperFileGroup, LatexPaperSplit
|
||||
|
|
|
|||
|
|
@ -153,9 +153,9 @@ Latex segmentation with a binary mask (PRESERVE=0, TRANSFORM=1)
|
|||
def set_forbidden_text(text, mask, pattern, flags=0):
|
||||
"""
|
||||
Add a preserve text area in this paper
|
||||
e.g. with pattern = r"\\begin\{algorithm\}(.*?)\\end\{algorithm\}"
|
||||
e.g. with pattern = r"\\begin\\{algorithm\\}(.*?)\\end\\{algorithm\\}"
|
||||
you can mask out (mask = PRESERVE so that text become untouchable for GPT)
|
||||
everything between "\begin{equation}" and "\end{equation}"
|
||||
everything between "\\begin{equation}" and "\\end{equation}"
|
||||
"""
|
||||
if isinstance(pattern, list):
|
||||
pattern = "|".join(pattern)
|
||||
|
|
@ -331,18 +331,58 @@ def find_main_tex_file(file_manifest, mode):
|
|||
return canidates[select]
|
||||
|
||||
|
||||
def rm_comments(main_file):
|
||||
new_file_remove_comment_lines = []
|
||||
for l in main_file.splitlines():
|
||||
# 删除整行的空注释
|
||||
if l.lstrip().startswith("%"):
|
||||
pass
|
||||
else:
|
||||
new_file_remove_comment_lines.append(l)
|
||||
main_file = "\n".join(new_file_remove_comment_lines)
|
||||
# main_file = re.sub(r"\\include{(.*?)}", r"\\input{\1}", main_file) # 将 \include 命令转换为 \input 命令
|
||||
main_file = re.sub(r"(?<!\\)%.*", "", main_file) # 使用正则表达式查找半行注释, 并替换为空字符串
|
||||
return main_file
|
||||
def rm_comments_inline(content: str):
|
||||
"""删除掉所有行内注释"""
|
||||
return re.sub(r"(?<!\\)%.*\n[ \t\r\f\v]*", "", content)
|
||||
|
||||
|
||||
def rm_comments_block(content: str):
|
||||
"""删除掉所有块注释"""
|
||||
stack: list[int] = []
|
||||
output: list[str] = []
|
||||
pos = 0
|
||||
len_text = len(content)
|
||||
pattern = re.compile(r"\\begin\{comment\}|\\end\{comment\}")
|
||||
|
||||
for match in pattern.finditer(content):
|
||||
start, end = match.start(), match.end()
|
||||
tag = match.group()
|
||||
|
||||
# 记录非注释内容
|
||||
if not stack:
|
||||
output.append(content[pos:start])
|
||||
|
||||
# 处理标签
|
||||
if tag == r"\begin{comment}":
|
||||
stack.append(end) # 记录起始位置
|
||||
else: # \end{comment}
|
||||
if stack:
|
||||
stack.pop() # 弹出匹配的\begin{comment}
|
||||
else:
|
||||
raise ValueError("\\end{comment}没有匹配的\\begin{comment}")
|
||||
|
||||
pos = end
|
||||
|
||||
# 添加最后一段内容（如果存在）
|
||||
if stack:
|
||||
raise ValueError("\\begin{comment}没有匹配的\\end{comment}")
|
||||
elif pos < len_text:
|
||||
output.append(content[pos:])
|
||||
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def rm_comments(main_file: str):
|
||||
"""删除掉所有注释
|
||||
|
||||
TeX文件有数种注释:
|
||||
- % 注释: 删除本行%后所有内容, 删除回车, 删除下一行缩进
|
||||
- \\iffalse ... \\fi 注释: 呃, 删不动... (主要是可能的嵌套问题)
|
||||
- \\begin{comment} ... \\end{comment} 注释: 删除所有内容
|
||||
|
||||
残存BUG: 未考虑 % 在verbatim环境中出现的情况
|
||||
"""
|
||||
return rm_comments_block(rm_comments_inline(main_file))
|
||||
|
||||
|
||||
def find_tex_file_ignore_case(fp):
|
||||
|
|
|
|||
Loading…
Reference in New Issue