diff --git a/.gitignore b/.gitignore index 267b4b1..791ee7c 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,6 @@ patches/ .pdm-python .vscode + +linux-stable/ +kernel-euler/ diff --git a/pdm.lock b/pdm.lock index 007a13f..2f030ff 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:838eb6c6f611048bd6715fbdf796cbb900f404bfe053b4b8ac5d784265850b95" +content_hash = "sha256:b2a865c729a9a1851ef91c1f81da1996e532e115bcd4afaf64895c51469e6ef7" [[metadata.targets]] requires_python = ">=3.10" @@ -525,6 +525,61 @@ files = [ {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, ] +[[package]] +name = "tree-sitter" +version = "0.24.0" +requires_python = ">=3.10" +summary = "Python bindings to the Tree-sitter parsing library" +groups = ["default"] +files = [ + {file = "tree-sitter-0.24.0.tar.gz", hash = "sha256:abd95af65ca2f4f7eca356343391ed669e764f37748b5352946f00f7fc78e734"}, + {file = "tree_sitter-0.24.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f3f00feff1fc47a8e4863561b8da8f5e023d382dd31ed3e43cd11d4cae445445"}, + {file = "tree_sitter-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f9691be48d98c49ef8f498460278884c666b44129222ed6217477dffad5d4831"}, + {file = "tree_sitter-0.24.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:098a81df9f89cf254d92c1cd0660a838593f85d7505b28249216661d87adde4a"}, + {file = "tree_sitter-0.24.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b26bf9e958da6eb7e74a081aab9d9c7d05f9baeaa830dbb67481898fd16f1f5"}, + {file = "tree_sitter-0.24.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2a84ff87a2f2a008867a1064aba510ab3bd608e3e0cd6e8fef0379efee266c73"}, + {file = "tree_sitter-0.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:c012e4c345c57a95d92ab5a890c637aaa51ab3b7ff25ed7069834b1087361c95"}, + {file = "tree_sitter-0.24.0-cp310-cp310-win_arm64.whl", hash = "sha256:033506c1bc2ba7bd559b23a6bdbeaf1127cee3c68a094b82396718596dfe98bc"}, + {file = "tree_sitter-0.24.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de0fb7c18c6068cacff46250c0a0473e8fc74d673e3e86555f131c2c1346fb13"}, + {file = "tree_sitter-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7c9c89666dea2ce2b2bf98e75f429d2876c569fab966afefdcd71974c6d8538"}, + {file = "tree_sitter-0.24.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddb113e6b8b3e3b199695b1492a47d87d06c538e63050823d90ef13cac585fd"}, + {file = "tree_sitter-0.24.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01ea01a7003b88b92f7f875da6ba9d5d741e0c84bb1bd92c503c0eecd0ee6409"}, + {file = "tree_sitter-0.24.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:464fa5b2cac63608915a9de8a6efd67a4da1929e603ea86abaeae2cb1fe89921"}, + {file = "tree_sitter-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:3b1f3cbd9700e1fba0be2e7d801527e37c49fc02dc140714669144ef6ab58dce"}, + {file = "tree_sitter-0.24.0-cp311-cp311-win_arm64.whl", hash = "sha256:f3f08a2ca9f600b3758792ba2406971665ffbad810847398d180c48cee174ee2"}, + {file = "tree_sitter-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14beeff5f11e223c37be7d5d119819880601a80d0399abe8c738ae2288804afc"}, + {file = "tree_sitter-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26a5b130f70d5925d67b47db314da209063664585a2fd36fa69e0717738efaf4"}, + {file = "tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fc5c3c26d83c9d0ecb4fc4304fba35f034b7761d35286b936c1db1217558b4e"}, + {file = "tree_sitter-0.24.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:772e1bd8c0931c866b848d0369b32218ac97c24b04790ec4b0e409901945dd8e"}, + {file = "tree_sitter-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:24a8dd03b0d6b8812425f3b84d2f4763322684e38baf74e5bb766128b5633dc7"}, + {file = "tree_sitter-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9e8b1605ab60ed43803100f067eed71b0b0e6c1fb9860a262727dbfbbb74751"}, + {file = "tree_sitter-0.24.0-cp312-cp312-win_arm64.whl", hash = "sha256:f733a83d8355fc95561582b66bbea92ffd365c5d7a665bc9ebd25e049c2b2abb"}, + {file = "tree_sitter-0.24.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d4a6416ed421c4210f0ca405a4834d5ccfbb8ad6692d4d74f7773ef68f92071"}, + {file = "tree_sitter-0.24.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0992d483677e71d5c5d37f30dfb2e3afec2f932a9c53eec4fca13869b788c6c"}, + {file = "tree_sitter-0.24.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57277a12fbcefb1c8b206186068d456c600dbfbc3fd6c76968ee22614c5cd5ad"}, + {file = "tree_sitter-0.24.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25fa22766d63f73716c6fec1a31ee5cf904aa429484256bd5fdf5259051ed74"}, + {file = "tree_sitter-0.24.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d5d9537507e1c8c5fa9935b34f320bfec4114d675e028f3ad94f11cf9db37b9"}, + {file = "tree_sitter-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:f58bb4956917715ec4d5a28681829a8dad5c342cafd4aea269f9132a83ca9b34"}, + {file = "tree_sitter-0.24.0-cp313-cp313-win_arm64.whl", hash = "sha256:23641bd25dcd4bb0b6fa91b8fb3f46cc9f1c9f475efe4d536d3f1f688d1b84c8"}, +] + +[[package]] +name = "tree-sitter-c" +version = "0.23.2" +requires_python = ">=3.9" +summary = "C grammar for tree-sitter" +groups = ["default"] +files = [ + {file = "tree_sitter_c-0.23.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:0e8c60630a34e6fa0680b10df984c6f9dd48a5ef0ea8a564d3c5f0e0345d4489"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffcaab20e65a4d29a635c978b22a12e4d8ffe9dc5adc35860e3982a929e6c438"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b40205ef5919d32083ad7eea13882c7f8bc85281c1a6672ed2b3543f18b7aad6"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47ba5b66cfab7f38549d571170069ef2d877e9468177fa5bea9b5fe1fd52badc"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:926c6da41f3943a08eebf96d1d3bc71c9c0181a2c72d859a4e52f49ace691ff1"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-win_amd64.whl", hash = "sha256:82167c253832e23e47413fbb06d572d53494b4d134e20eee372551da8fa4e931"}, + {file = "tree_sitter_c-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:5a5d54b27665f6596e6c10c6b2c7bc4c181428e3a235ccfe4e9237e27751c5c9"}, + {file = "tree_sitter_c-0.23.2.tar.gz", hash = "sha256:ef7ecc95c170747aceccf13ae0e37e101f9ba25a639729be0f70222f2a02f99c"}, +] + [[package]] name = "typer" version = "0.15.1" diff --git a/pyproject.toml b/pyproject.toml index e7772df..271456f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ppatch" -version = "0.0.5" +version = "0.0.6b2.post1" description = "Universal tool to analyze patches" authors = [ {name = "jingfelix", email = "jingfelix@outlook.com"}, @@ -11,6 +11,8 @@ dependencies = [ "pydantic-settings>=2.2.1", "cscopy>=0.0.3", "whatthepatch-pydantic==1.0.6a2", + "tree-sitter>=0.24.0", + "tree-sitter-c==0.23.2", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index 34f2f21..4eedb90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,8 @@ shellingham==1.5.4 stack-data==0.6.3; python_version > "3.6" tomli==2.2.1; python_version > "3.6" and python_version < "3.11" traitlets==5.14.3; python_version > "3.6" +tree-sitter==0.24.0 +tree-sitter-c==0.23.2 typer[all]==0.15.1 typing-extensions==4.12.2 viztracer==1.0.0 diff --git a/scripts/run_one_ppatch.sh b/scripts/run_one_ppatch.sh index 85f7446..7980d20 100755 --- a/scripts/run_one_ppatch.sh +++ b/scripts/run_one_ppatch.sh @@ -8,13 +8,15 @@ use_config=$2 # 定义三个命令，使用该变量 if [[ $use_config == "true" ]]; then echo "Use config = True" - command1="ppatch auto /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch -c /home/laboratory/workspace/exps/ppatch/example_extra_config.json" + # command1="ppatch --verbose auto /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch -c /home/laboratory/workspace/exps/ppatch/example_extra_config.json" + command1="ppatch auto -O /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch -c /home/laboratory/workspace/exps/ppatch/example_extra_config.json" else - command1="ppatch auto /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch" + # command1="ppatch --verbose auto /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch" + command1="ppatch auto -O /home/laboratory/workspace/archive/patches/poc_$variable.patch -o ../6.1-result/$variable.patch" fi command2="patch -p 1 -F 3 -i ../6.1-result/$variable.patch" -command3=" patch -R -p 1 -F 3 -f -i /home/laboratory/workspace/archive/patches/poc_$variable.patch" +command3="patch -R -p 1 -F 3 -f -i /home/laboratory/workspace/archive/patches/poc_$variable.patch" # 执行第一个命令并检查返回值 $command1 @@ -42,6 +44,11 @@ read -p "命令3执行成功，按任意键继续..." git diff > $variable.patch -scp pride:/mnt/cd7/ppatch_autoset/auto_linux_6.9.5_back/exps/poc_$variable/linux_6_lts/x86_64/linux_6_lts_kernel/upper_dir/.config .config.$variable +# scp pride:/mnt/cd7/ppatch_autoset/auto_linux_6.9.5_back/exps/poc_$variable/linux_6_lts/x86_64/linux_6_lts_kernel/upper_dir/.config .config.$variable +command4="scp pride:/mnt/cd7/ppatch_autoset/ppatch_partner/src/exps/poc_$variable/longterm/x86_64/longterm_kernel/upper_dir/.config .config.$variable" +$command4 +if [ $? -ne 0 ]; then + scp pride:/mnt/cd7/ppatch_autoset/ppatch_partner/src/configs/x86_64_common_config .config.$variable +fi read -p "patch 生成与 config 下载结束，按任意键继续..." diff --git a/scripts/test_find_changed_funcs.py b/scripts/test_find_changed_funcs.py new file mode 100644 index 0000000..67c2dfe --- /dev/null +++ b/scripts/test_find_changed_funcs.py @@ -0,0 +1,86 @@ +# input 1: current file +# input 2: current patch +# input 3: original file +# input 4: original patch + +from ppatch.model import File, Patch +from ppatch.utils.ast import File as FileAST +from ppatch.utils.ast import Func +from ppatch.utils.parse import parse_patch + +file_path = "" + +original_file = "" +original_patch = "" + +current_file = "" +current_patch = "" + + +def find_changed_funcs(file: str, patch: str) -> list[Func]: + + original_lines = File(content=file).line_list + current_patch: Patch = parse_patch(patch) + + # 从 Patch 确定要产生修改的行 + # 前提在于选定的 patch 是正确的（如何保证呢，需要使用 Diff） + current_diff = None + + for diff in current_patch.diff: + if diff.header.old_path == file_path: + current_diff = diff + break + + changed_lines = [] + for change in current_diff.changes: + if change.old is not None: + original_lines[change.old].changed = True + changed_lines.append(change.old) + + ast_file = FileAST("".join([str(line) + "\n" for line in original_lines])) + + changed_funcs: list[Func] = [] + for line in changed_lines: + func = ast_file.line_func_map[line] + if func and func not in changed_funcs: + changed_funcs.append(func) + + return changed_funcs + + +def check( + original_file: str, original_patch: str, current_file: str, current_patch: str +): + + old_changed_funcs = find_changed_funcs(original_file, original_patch) + new_changed_funcs = find_changed_funcs(current_file, current_patch) + + old_funcs = FileAST(original_file).funcs + # new_funcs = FileAST(current_file).funcs + + old_changed_func_names = [func["name"] for func in old_changed_funcs] + old_func_names = [func["name"] for func in old_funcs] + + new_changed_func_names = [func["name"] for func in new_changed_funcs] + new_func_names = [func["name"] for func in FileAST(current_file).funcs] + + for func in new_changed_funcs: + if func["name"] not in old_changed_func_names: + print(f"Warning: found new changed function {func['name']}") + # Check if the function actually exists in the old file + if func["name"] in old_func_names: + print(f"Warning: found inconsistent changed function {func['name']}") + + # for func in old_changed_funcs: + # if func["name"] not in new_changed_func_names: + # print(f"Warning: found missing changed function {func['name']}") + # # Check if the function actually exists in the new file + # if func["name"] not in new_func_names: + # print(f"Warning: found inconsistent changed function {func['name']}") + + # for func in old_funcs: + # if func["name"] == "hci_get_route" or func["name"] == "hci_conn_enter_active_mode": + # print(f"Warning: found changed function {func['name']}") + + +check(original_file, original_patch, current_file, current_patch) diff --git a/src/ppatch/__version__.py b/src/ppatch/__version__.py index 7d719a3..ed71fc2 100644 --- a/src/ppatch/__version__.py +++ b/src/ppatch/__version__.py @@ -6,4 +6,7 @@ import importlib_metadata -__version__ = importlib_metadata.version("ppatch") +try: + __version__ = importlib_metadata.version("ppatch") +except importlib_metadata.PackageNotFoundError: + __version__ = "0.0.0" diff --git a/src/ppatch/commands/apply.py b/src/ppatch/commands/apply.py index 04fffcb..8d66bd6 100644 --- a/src/ppatch/commands/apply.py +++ b/src/ppatch/commands/apply.py @@ -1,11 +1,15 @@ import os +import subprocess from typing import Annotated import typer -import whatthepatch from ppatch.app import app, logger -from ppatch.model import File +from ppatch.commands.get import getpatches +from ppatch.config import settings +from ppatch.model import SHA, File +from ppatch.utils.ast import File as FileAST +from ppatch.utils.common import get_changed_funcs, process_title from ppatch.utils.parse import parse_patch from ppatch.utils.resolve import apply_change @@ -16,6 +20,7 @@ def apply( patch_path: str, reverse: Annotated[bool, typer.Option("-R", "--reverse")] = False, fuzz: Annotated[int, typer.Option("-F", "--fuzz")] = 0, + function_check: bool = typer.Option(False, "--function-check", "-f"), ): """ Apply a patch to a file. @@ -31,12 +36,49 @@ def apply( has_failed = False with open(patch_path, mode="r", encoding="utf-8") as (f): - diffes = parse_patch(f.read()).diff + parser = parse_patch(f.read()) + diffes = parser.diff + sha_list: list[SHA] = [] + + if function_check: + file_name = diffes[0].header.old_path # No need to distinguish reverse here + + subject = parser.subject + hit_list, sha_list = getpatches(file_name, subject, save=True) + sha_for_sure = None + + for sha in hit_list: + with open( + os.path.join( + settings.base_dir, + settings.patch_store_dir, + f"{sha}-{process_title(file_name)}.patch", + ), + mode="r", + encoding="utf-8", + ) as (f): + text = f.read() + + # logger.debug(f"1st: {parse_patch(text).subject}") + # logger.debug(f"2nd: {subject}") + + target_subject = parse_patch(text).subject + if target_subject in subject or subject in target_subject: + sha_for_sure = sha + break + + if sha_for_sure is None: + logger.error( + f"function_check: Cannot find a patch with subject '{subject}' for file '{file_name}'." + ) + raise typer.Exit(code=1) for diff in diffes: old_filename = diff.header.old_path new_filename = diff.header.new_path + if reverse: + old_filename, new_filename = new_filename, old_filename if os.path.exists(old_filename): @@ -54,13 +96,62 @@ def apply( logger.error(f"Failed hunk: {failed_hunk.index}") else: logger.error(f"{old_filename} not found!") - raise typer.Exit(code=1) - # 写入文件 - if not has_failed: - with open(new_filename, mode="w+", encoding="utf-8") as f: - for line in new_line_list: - if line.status: - f.write(line.content + "\n") + # git log --oneline --diff-filter=R -- + output: str = subprocess.run( + ["git", "log", "--oneline", "--diff-filter=R", "--", old_filename], + capture_output=True, + ).stdout.decode("utf-8", errors="ignore") + if len(output) > 0: + logger.warning(f"File {old_filename} has been renamed.") + + raise typer.Exit(code=2) + + # 写入文件：与 GNU patch 一致，成功 hunk 即使存在失败 hunk 也会落盘 + if function_check: + # Note that file won't be saved if function_check is True + + # 记录当前修改所属的函数列表和所有函数列表 + patched_text = "\n".join([line.content for line in new_line_list]) + file_ast = FileAST(patched_text) + + patched_all_funcs = file_ast.funcs + patched_changed_funcs = get_changed_funcs(new_line_list, file_ast) + + # Get to the original file + # git show sha:filename + # 利用 sha_for_sure 获取指定 sha 的前一个 sha + sha_for_sure_index = sha_list.index(sha_for_sure) + sha_before_sure = sha_list[sha_for_sure_index + 1] # TODO: check this + + before_original_file: str = subprocess.run( + [ + "git", + "show", + f"{sha_before_sure}:{diff.header.old_path}", + ], # No need to distinguish reverse here + capture_output=True, + ).stdout.decode("utf-8", errors="ignore") + + # 尝试 apply 原补丁来确定原始修改所属的函数 + original_file_line_list = apply_change( + diff.hunks, + File(content=before_original_file).line_list, + reverse=reverse, + fuzz=fuzz, + ).new_line_list + + original_file_ast = FileAST(before_original_file) + original_changed_funcs = get_changed_funcs( + original_file_line_list, original_file_ast + ) + + logger.info(f"Patched changed funcs: {patched_changed_funcs}") + logger.info(f"Original changed funcs: {original_changed_funcs}") + + with open(new_filename, mode="w+", encoding="utf-8") as f: + for line in new_line_list: + if line.status: + f.write(line.content + "\n") raise typer.Exit(code=1 if has_failed else 0) diff --git a/src/ppatch/commands/auto.py b/src/ppatch/commands/auto.py index 593d5a7..44f543c 100644 --- a/src/ppatch/commands/auto.py +++ b/src/ppatch/commands/auto.py @@ -17,17 +17,29 @@ Diff, File, ) -from ppatch.utils.common import process_json_config, process_title +from ppatch.utils.ast import File as FileAST +from ppatch.utils.ast import Func +from ppatch.utils.common import ( + get_changed_funcs, + match_file_patterns, + process_json_config, + process_title, +) from ppatch.utils.parse import changes_to_hunks, parse_patch from ppatch.utils.resolve import apply_change +def process_str(s: str) -> str: + return "".join(c if c.isalnum() or c == "_" else "_" for c in s) + + @app.command() def auto( filename: str, output: str = typer.Option("", "--output", "-o"), extra_config: str = typer.Option("", "--extra-config", "-c"), use_multi_file: bool = typer.Option(False, "--multi-file", "-m"), + oracle: bool = typer.Option(False, "--oracle", "-O"), ): """Automatic do ANYTHING""" if not os.path.exists(filename): @@ -47,7 +59,7 @@ def auto( output = os.path.join(output, "auto.patch") content = "" - with open(filename, mode="r", encoding="utf-8") as (f): + with open(filename, mode="r", encoding="utf-8", errors="ignore") as (f): content = f.read() parser = parse_patch(content) @@ -56,6 +68,13 @@ def auto( for diff in diffes: target_file = diff.header.new_path # 这里注意是 new_path 还是 old_path + # 检查文件是否符合 include_file_list 中的通配符 + if not match_file_patterns(target_file, settings.include_file_list): + logger.info( + f"Skipping file {target_file} as it does not match include patterns" + ) + continue + if not os.path.exists(target_file): logger.error(f"File {target_file} not found!") return CommandResult( @@ -66,7 +85,10 @@ def auto( # 执行 Reverse，确定失败的 Hunk apply_result = apply_change( - diff.hunks, origin_file.line_list, reverse=True, fuzz=3 + diff.hunks, + origin_file.line_list, + reverse=True, + fuzz=2, # TODO: 调整为 fuzz=2，还需要改 trace 里的 fuzz 参数 ) if len(apply_result.failed_hunk_list) != 0: @@ -84,7 +106,7 @@ def auto( subject = parser.subject diffes: list = [] - filename_with_conflict_list: dict[FILENAME, dict[SHA, ApplyResult]] = {} + filename_with_conflict_list: dict[FILENAME, list[tuple[SHA, ApplyResult]]] = {} symbols: list[str] = None if extra_config != "": @@ -246,7 +268,7 @@ def auto( planned_hunks_count += 1 _apply_result = apply_change( - changes_to_hunks(changes), line_list, reverse=True, fuzz=3 + changes_to_hunks(changes), line_list, reverse=True, fuzz=3, flag=True ) # TODO: 错误处理 try: @@ -263,6 +285,31 @@ def auto( patched_text = "\n".join([line.content for line in line_list]) origin_text = "\n".join([line.content for line in origin_file.line_list]) + # 在 patched_text 上进行修改 + # 1. 获取所有发生变更的行的行号和 hunk.index + # 2. 搜索这些行在在 patched_text 中属于哪些函数的范围，如果不是函数则不处理 + # 3. 在这些函数的起始位置添加 printk + if oracle: + file_ast = FileAST(content=patched_text) + + changed_funcs: list[Func] = get_changed_funcs(line_list, file_ast) + + for func in changed_funcs: + # 从 start_line 开始读取，读取到的第一个 '{' 之后插入一行 printk + patched_lines = patched_text.splitlines() + # convert to 0-based index + start_idx = func["start_line"] - 1 + # 从函数定义开始位置往后找第一个 '{' + for i in range(start_idx, len(patched_lines)): + if "{" in patched_lines[i]: + patched_lines.insert( + i + 1, + f"""\tprintk(KERN_NOTICE "PPATCH {process_str(func["name"])} {filename.split("/")[-1]}\\n");""", + ) + break + # 重新构建 patched_text + patched_text = "\n".join(patched_lines) + import difflib diffes_ = difflib.unified_diff( diff --git a/src/ppatch/commands/trace.py b/src/ppatch/commands/trace.py index 88287ae..3bd0d2b 100644 --- a/src/ppatch/commands/trace.py +++ b/src/ppatch/commands/trace.py @@ -11,7 +11,7 @@ from ppatch.utils.resolve import apply_change -@app.command() +@app.command(name="trace") def trace_command( filename: str, from_commit: str = "", flag_hunk_list: list[int] = None ) -> dict[str, ApplyResult]: diff --git a/src/ppatch/config.py b/src/ppatch/config.py index 4e9b572..c03c32d 100644 --- a/src/ppatch/config.py +++ b/src/ppatch/config.py @@ -23,6 +23,7 @@ class Settings(BaseSettings): patch_store_dir: str = "_patches" max_diff_lines: int = 3 work_dir: str = os.path.abspath(os.getcwd()) + include_file_list: list[str] = ["*.c", "*.h", "*.cpp", "*.hpp", "*.cc", "*.hh"] class Config: env_file = os.path.join(os.environ.get("HOME"), ".ppatch.env") diff --git a/src/ppatch/model.py b/src/ppatch/model.py index 5ef8373..2dcff0d 100644 --- a/src/ppatch/model.py +++ b/src/ppatch/model.py @@ -24,12 +24,22 @@ def __str__(self) -> str: class File(object): - def __init__(self, file_path: str) -> None: + def __init__(self, file_path: str = "", content: str = "") -> None: self.line_list: list[Line] = [] - with open(file_path, mode="r", encoding="utf-8", errors="ignore") as f: - for i, line in enumerate(f): - self.line_list.append(Line(index=i, content=line.rstrip("\n"))) + if not file_path and not content: + raise ValueError("Either file_path or content must be provided") + + if content: + self.line_list = [ + Line(index=i, content=line.rstrip("\n")) + for i, line in enumerate(content.splitlines()) + ] + + elif file_path: + with open(file_path, mode="r", encoding="utf-8", errors="ignore") as f: + for i, line in enumerate(f): + self.line_list.append(Line(index=i, content=line.rstrip("\n"))) def __str__(self) -> str: return "".join([str(line) for line in self.line_list]) diff --git a/src/ppatch/utils/ast.py b/src/ppatch/utils/ast.py new file mode 100644 index 0000000..7015dcb --- /dev/null +++ b/src/ppatch/utils/ast.py @@ -0,0 +1,87 @@ +from typing import TypedDict + +import tree_sitter_c as tsc +from tree_sitter import Language, Parser + +C_LANGUAGE = Language(tsc.language()) + +from tree_sitter import Language, Parser + + +class Func(TypedDict): + name: str + start_line: int + end_line: int + + +class File(object): + + def __init__(self, content: str) -> None: + if content is None: + raise ValueError("Content cannot be None") + + self.content = content + self.parser = Parser(C_LANGUAGE) + self.tree = self.parser.parse(bytes(content, "utf8")) + self.root_node = self.tree.root_node + + self._funcs: list[Func] | None = None + self._line_func_map: list[Func | None] | None = None + + @property + def funcs(self) -> list[Func]: + if self._funcs is None: + self._funcs = self.find_funcs() + return self._funcs + + @property + def line_func_map(self) -> list[Func | None]: + if self._line_func_map is None: + self._line_func_map = self.make_line_func_map() + return self._line_func_map + + def find_funcs(self, node=None) -> list[Func]: + """ + Find all function definitions recursively and return a list of function information. + """ + if node is None: + node = self.root_node + functions = [] + if node.type == "function_definition": + function_declarator = node.child_by_field_name("declarator") + if function_declarator: + function_name_node = function_declarator.child_by_field_name( + "declarator" + ) + if function_name_node: + function_name = function_name_node.text.decode("utf8") + start_line = node.start_point[0] + 1 + end_line = node.end_point[0] + 1 + functions.append( + Func( + name=function_name, start_line=start_line, end_line=end_line + ) + ) + for child in node.children: + functions.extend(self.find_funcs(child)) + return functions + + def make_line_func_map(self) -> list[Func | None]: + + total_lines = len(self.content.splitlines()) + line_map = [None] * total_lines + for func in self.funcs: + for i in range(func["start_line"] - 1, func["end_line"]): + if 0 <= i < total_lines: + line_map[i] = func + + return line_map + + def locate_line(self, line: int) -> Func | None: + """ + Locate the function that contains the specified line number. + """ + + if 0 <= line - 1 < len(self.line_func_map): + return self.line_func_map[line - 1] + return None diff --git a/src/ppatch/utils/common.py b/src/ppatch/utils/common.py index e2d0b8f..56b1e72 100644 --- a/src/ppatch/utils/common.py +++ b/src/ppatch/utils/common.py @@ -1,3 +1,4 @@ +import fnmatch import json import re import subprocess @@ -5,7 +6,9 @@ from typing import Any from ppatch.app import logger -from ppatch.model import CommandResult, CommandType +from ppatch.model import CommandResult, CommandType, Line +from ppatch.utils.ast import File as FileAST +from ppatch.utils.ast import Func def clean_repo(): @@ -37,12 +40,12 @@ def process_file_path(file_path: str, reverse: bool = False) -> str: return file_path.replace("/", "&#") -def find_list_positions(main_list: list[str], sublist: list[str]) -> list[int]: - sublist_length = len(sublist) +def find_list_positions(main_list: list[str], sub_list: list[str]) -> list[int]: + sublist_length = len(sub_list) positions = [] for i in range(len(main_list) - sublist_length + 1): - if main_list[i : i + sublist_length] == sublist: + if main_list[i : i + sublist_length] == sub_list: positions.append(i) return positions @@ -106,6 +109,8 @@ def process_json_config(input_file: str) -> dict[str:list]: if kind == "error" and message and file_name: # 使用正则表达式提取变量 symbols = re.findall(r"‘(.*?)’", message) + symbols.extend(re.findall(r"'(.*?)'", message)) # When to use ``? + new_data.append( { "kind": kind, @@ -129,3 +134,44 @@ def process_json_config(input_file: str) -> dict[str:list]: result = {file: list(symbols) for file, symbols in merged_data.items()} return result + + +def match_file_patterns(filename: str, patterns: list[str]) -> bool: + """ + Check if the filename matches any of the patterns + + Args: + filename: file name + patterns: list of wildcard patterns + + Returns: + Returns True if the filename matches any pattern, otherwise returns False + """ + return any(fnmatch.fnmatch(filename, pattern) for pattern in patterns) + + +def get_changed_funcs(line_list: list[Line], file_ast: FileAST) -> list[Func]: + """ + line_list: list of changed lines, each line is a Line object with .changed attribute + file_ast: FileAST object representing the file's AST + """ + + changed_lines: list[Line] = [line for line in line_list if line.changed] + changed_funcs: list[Func] = [] + + for line in changed_lines: + line_number = line.index + 1 + + func: Func | None = file_ast.locate_line(line_number) + if func: + changed_funcs.append(func) + + # TODO: 简化 + sorted_changed_funcs = [] + for func in changed_funcs: + if func not in sorted_changed_funcs: + sorted_changed_funcs.append(func) + + changed_funcs = sorted_changed_funcs + + return changed_funcs diff --git a/src/ppatch/utils/resolve.py b/src/ppatch/utils/resolve.py index a0c2902..e5b4534 100644 --- a/src/ppatch/utils/resolve.py +++ b/src/ppatch/utils/resolve.py @@ -26,8 +26,8 @@ def apply_change( # TODO: 注意，修改了该函数后，需要将此处修改为对 hunk 内的 change 进行修改 if reverse: - if flag: - raise Exception("flag is not supported with reverse") + # if flag: + # raise Exception("flag is not supported with reverse") for hunk in hunk_list: for change in hunk.context + hunk.middle + hunk.post: @@ -37,11 +37,18 @@ def apply_change( changes: list[Change] = [] failed_hunk_list: list[Hunk] = [] last_pos = None - - last_offset = 0 - line_count_diff = 0 + # 上一个成功 hunk 的定位偏移，加上该 hunk 的净行数变化。 + # 供后续无法用上下文定位的 hunk（fuzz 后只剩新增行）使用，对齐 GNU patch 的 last_offset。 + carry_offset = 0 for hunk in hunk_list: + orig_context_len = len(hunk.context) + orig_start = None + for change in list(hunk.context) + hunk.middle + list(hunk.post): + if change.old is not None: + orig_start = change.old + break + current_hunk_fuzz = 0 while current_hunk_fuzz <= fuzz: @@ -76,28 +83,27 @@ def apply_change( pos_origin = change.old break - # TODO: 这里不太对，要想一下怎么处理，不应该是加入 failed hunk list - # 仅在 -F 3 且只有添加行 的情况下出现（指与 GNU patch 行为不一致） - # 也可以看一下这样的情况有多少 - if current_hunk_fuzz == fuzz and not pos_origin: - # failed_hunk_list.append(hunk) - # logger.debug(f"Could not determine pos_origin") - # logger.warning(f"Apply failed with hunk {hunk.index}") - # continue - for change in changes_to_search: - if change.new is not None: - pos_origin = change.new - break - - # 使用上一次偏移加上行数变化差值 - min_offset = last_offset - else: - if len(pos_list) == 0: - failed_hunk_list.append(hunk) - logger.debug(f"Could not determine proper position") - logger.warning(f"Apply failed with hunk {hunk.index}") - continue + # fuzz 把上下文剥光后只剩新增行时，没有 old 行可搜索。 + # GNU patch 仍按 hunk 起始行 + 此前偏移插入，插入点在前置上下文之后。 + anchorless = pos_origin is None + if anchorless and not (reverse and orig_start is not None): + failed_hunk_list.append(hunk) + logger.debug(f"Could not determine pos_origin") + logger.warning(f"Apply failed with hunk {hunk.index}") + continue + + if not anchorless and len(pos_list) == 0: + failed_hunk_list.append(hunk) + logger.debug(f"Could not determine proper position") + logger.warning(f"Apply failed with hunk {hunk.index}") + continue + if anchorless: + min_offset = carry_offset + pos_new = orig_start + carry_offset + orig_context_len - 1 + if pos_new < 0: + pos_new = 0 + else: offset_list = [ pos + 1 - pos_origin for pos in pos_list ] # 确认这里是否需要 1？ @@ -108,26 +114,12 @@ def apply_change( if min_offset is None or abs(offset) < abs(min_offset): min_offset = offset - if reverse: - min_offset += line_count_diff - pos_origin -= line_count_diff - - last_offset = min_offset - - # 更新行数变化差值 - hunk_add_count = sum( - 1 for c in changes_to_search if c.old is None and c.new is not None - ) - hunk_del_count = sum( - 1 for c in changes_to_search if c.new is None and c.old is not None - ) - line_count_diff += hunk_del_count - hunk_add_count + pos_new = pos_origin + min_offset - 1 logger.info( - f"Apply hunk {hunk.index} with offset {min_offset} fuzz {current_hunk_fuzz} line_diff {line_count_diff}" + f"Apply hunk {hunk.index} with offset {min_offset} fuzz {current_hunk_fuzz}" ) - pos_new = pos_origin + min_offset - 1 # 处理 pos_new 小于 last_pos 的情况 logger.debug(f"pos_origin: {pos_origin}, last_pos: {last_pos}") if last_pos is None: @@ -169,11 +161,18 @@ def apply_change( target = ( target[:pos_new] + [ - Line(index=pos_new + i, content=new_lines[i]) + Line( + index=pos_new + i, + content=new_lines[i], + changed=True, + flag=flag, + hunk=hunk.index, + ) for i in range(len(new_lines)) ] + target[pos_new + len(old_lines) :] ) + carry_offset = min_offset + (len(new_lines) - len(old_lines)) else: for change in hunk.middle: