diff --git a/graph_net/paddle/backend/cinn_backend.py b/graph_net/paddle/backend/cinn_backend.py index bd19a6f80..4955f41ab 100644 --- a/graph_net/paddle/backend/cinn_backend.py +++ b/graph_net/paddle/backend/cinn_backend.py @@ -12,7 +12,6 @@ def __call__(self, model, input_spec=None): full_graph=True, ) compiled_model.eval() - program = compiled_model.forward.concrete_program.main_program return compiled_model def synchronize(self): diff --git a/graph_net/paddle/check_redundant_incrementally.py b/graph_net/paddle/check_redundant_incrementally.py index c5b59839a..81b75a3f6 100644 --- a/graph_net/paddle/check_redundant_incrementally.py +++ b/graph_net/paddle/check_redundant_incrementally.py @@ -1,16 +1,6 @@ -from . import utils import argparse -import importlib.util -import inspect -from pathlib import Path -from typing import Type, Any -import sys import os import os.path -from dataclasses import dataclass -from contextlib import contextmanager -import time -import glob def get_recursively_model_pathes(root_dir): diff --git a/graph_net/paddle/remove_redundant_incrementally.py b/graph_net/paddle/remove_redundant_incrementally.py index dcbd455b0..b55176ba6 100644 --- a/graph_net/paddle/remove_redundant_incrementally.py +++ b/graph_net/paddle/remove_redundant_incrementally.py @@ -1,16 +1,6 @@ -from . import utils import argparse -import importlib.util -import inspect -from pathlib import Path -from typing import Type, Any -import sys import os import os.path -from dataclasses import dataclass -from contextlib import contextmanager -import time -import glob import shutil diff --git a/graph_net/paddle/validate.py b/graph_net/paddle/validate.py index c1122cbd7..2942a2589 100644 --- a/graph_net/paddle/validate.py +++ b/graph_net/paddle/validate.py @@ -1,14 +1,8 @@ from . import utils import argparse import importlib.util -import inspect -from pathlib import Path -from typing import Type, Any import sys import hashlib -from contextlib import contextmanager -from collections import ChainMap -import numpy as np import graph_net import os import ast @@ -36,7 +30,6 @@ def _extract_forward_source(model_path, class_name): source = f.read() tree = ast.parse(source) - forward_code = None for node in tree.body: if isinstance(node, ast.ClassDef) and node.name == class_name: diff --git a/graph_net/test/bert_model_test.py b/graph_net/test/bert_model_test.py index c85a2d927..7a9a5f14d 100644 --- a/graph_net/test/bert_model_test.py +++ b/graph_net/test/bert_model_test.py @@ -1,7 +1,6 @@ import torch from transformers import AutoModel, AutoTokenizer import graph_net.torch -import os def get_model_name(): diff --git a/graph_net/torch/check_redundant_incrementally.py b/graph_net/torch/check_redundant_incrementally.py index 0c0f70534..6495a1d71 100644 --- a/graph_net/torch/check_redundant_incrementally.py +++ b/graph_net/torch/check_redundant_incrementally.py @@ -1,17 +1,6 @@ -from . import utils import argparse -import importlib.util -import inspect -import torch -from pathlib import Path -from typing import Type, Any -import sys import os import os.path -from dataclasses import dataclass -from contextlib import contextmanager -import time -import glob def get_recursively_model_pathes(root_dir): diff --git a/graph_net/torch/dim_gen_passes/__init__.py b/graph_net/torch/dim_gen_passes/__init__.py index aa1adce34..888fd5e1b 100644 --- a/graph_net/torch/dim_gen_passes/__init__.py +++ b/graph_net/torch/dim_gen_passes/__init__.py @@ -1 +1,3 @@ -from graph_net.torch.dim_gen_passes.pass_base import DimensionGeneralizationPass +from graph_net.torch.dim_gen_passes.pass_base import ( + DimensionGeneralizationPass as DimensionGeneralizationPass, +) diff --git a/graph_net/torch/dim_gen_passes/naive_call_method_reshape_pass.py b/graph_net/torch/dim_gen_passes/naive_call_method_reshape_pass.py index b9dade0cc..39c50da15 100644 --- a/graph_net/torch/dim_gen_passes/naive_call_method_reshape_pass.py +++ b/graph_net/torch/dim_gen_passes/naive_call_method_reshape_pass.py @@ -1,4 +1,3 @@ -import torch import torch.fx as fx from graph_net.torch.dim_gen_passes import DimensionGeneralizationPass import os diff --git a/graph_net/torch/dim_gen_passes/non_batch_call_function_getitem_slice_pass.py b/graph_net/torch/dim_gen_passes/non_batch_call_function_getitem_slice_pass.py index 75634cba2..90e683c85 100644 --- a/graph_net/torch/dim_gen_passes/non_batch_call_function_getitem_slice_pass.py +++ b/graph_net/torch/dim_gen_passes/non_batch_call_function_getitem_slice_pass.py @@ -1,4 +1,3 @@ -import torch import torch.fx as fx from graph_net.torch.dim_gen_passes import DimensionGeneralizationPass from collections import namedtuple diff --git a/graph_net/torch/dim_gen_passes/pass_base.py b/graph_net/torch/dim_gen_passes/pass_base.py index d780c5c3e..ce8098288 100644 --- a/graph_net/torch/dim_gen_passes/pass_base.py +++ b/graph_net/torch/dim_gen_passes/pass_base.py @@ -1,7 +1,4 @@ -import torch import torch.fx as fx -import inspect -import os class DimensionGeneralizationPass: diff --git a/graph_net/torch/dim_gen_passes/pass_mgr.py b/graph_net/torch/dim_gen_passes/pass_mgr.py index 2934da776..5b4ea96a4 100644 --- a/graph_net/torch/dim_gen_passes/pass_mgr.py +++ b/graph_net/torch/dim_gen_passes/pass_mgr.py @@ -2,7 +2,7 @@ import os -def get_dim_gen_pass(pass_name) -> "DimensionGeneralizationPass": +def get_dim_gen_pass(pass_name) -> "DimensionGeneralizationPass": # noqa: F821 import graph_net.torch.dim_gen_passes as dgpass py_module = load_module(f"{os.path.dirname(dgpass.__file__)}/{pass_name}.py") diff --git a/graph_net/torch/dump_graph_hash.py b/graph_net/torch/dump_graph_hash.py index 01cba0162..668da8293 100644 --- a/graph_net/torch/dump_graph_hash.py +++ b/graph_net/torch/dump_graph_hash.py @@ -3,7 +3,6 @@ import tempfile import sys import contextlib -import graph_net @contextlib.contextmanager @@ -17,7 +16,7 @@ def temp_workspace(): def main(args): model_path = args.model_path - with temp_workspace() as tmp_dir_name: + with temp_workspace(): print("dump-graph-hash ...") extract_name = "temp" cmd = f"{sys.executable} -m graph_net.torch.single_device_runner --model-path {model_path} --enable-extract True --extract-name {extract_name} --dump-graph-hash-key" diff --git a/graph_net/torch/remove_redundant_incrementally.py b/graph_net/torch/remove_redundant_incrementally.py index df046b504..b55176ba6 100644 --- a/graph_net/torch/remove_redundant_incrementally.py +++ b/graph_net/torch/remove_redundant_incrementally.py @@ -1,17 +1,6 @@ -from . import utils import argparse -import importlib.util -import inspect -import torch -from pathlib import Path -from typing import Type, Any -import sys import os import os.path -from dataclasses import dataclass -from contextlib import contextmanager -import time -import glob import shutil diff --git a/graph_net/torch/rp_expr/longest_rp_expr_parser.py b/graph_net/torch/rp_expr/longest_rp_expr_parser.py index 42e3bf832..c8f1c926d 100644 --- a/graph_net/torch/rp_expr/longest_rp_expr_parser.py +++ b/graph_net/torch/rp_expr/longest_rp_expr_parser.py @@ -2,7 +2,6 @@ from graph_net.torch.rp_expr.rp_expr_parser import RpExprParser from graph_net.torch.rp_expr.rp_expr import PrimitiveId, LetsListTokenRpExpr import numpy as np -import sys class LongestRpExprParser: diff --git a/graph_net/torch/rp_expr/rp_expr.py b/graph_net/torch/rp_expr/rp_expr.py index 7fcd7a1f6..c16a32a48 100644 --- a/graph_net/torch/rp_expr/rp_expr.py +++ b/graph_net/torch/rp_expr/rp_expr.py @@ -3,7 +3,6 @@ import numpy as np import torch from collections import defaultdict -import functools PrimitiveId = t.TypeVar("PrimitiveId") @@ -428,7 +427,7 @@ def ValueToString(token_id): ] ) yield from [ - f"def main():", + "def main():", *[ f" {SymbolToString(int(t[0]))}(){end_of_line}" for t in self.body_rp_expr diff --git a/graph_net/torch/rp_expr/rp_expr_parser.py b/graph_net/torch/rp_expr/rp_expr_parser.py index 0791622f8..b5344af7e 100644 --- a/graph_net/torch/rp_expr/rp_expr_parser.py +++ b/graph_net/torch/rp_expr/rp_expr_parser.py @@ -3,9 +3,7 @@ from graph_net.torch.rp_expr.rp_expr import Tokenize, PrimitiveId, LetsListTokenRpExpr from graph_net.torch.rp_expr.rp_expr_passes import ( FlattenTokenListPass, - FoldTokensPass, RecursiveFoldTokensPass, - FoldIfTokenIdGreatEqualPass, UnflattenAndSubThresholdPass, ) diff --git a/graph_net/torch/rp_expr/rp_expr_passes.py b/graph_net/torch/rp_expr/rp_expr_passes.py index b904eb90c..2d57af455 100644 --- a/graph_net/torch/rp_expr/rp_expr_passes.py +++ b/graph_net/torch/rp_expr/rp_expr_passes.py @@ -1,11 +1,8 @@ -from dataclasses import dataclass import typing as t import numpy as np -import re import itertools import torch import torch.nn.functional as F -import math from graph_net.torch.rp_expr.rp_expr import ( TokenIdAllocator, NaiveTokenListRpExpr, @@ -14,7 +11,6 @@ LetsTokenRpExpr, LetsListTokenRpExpr, ) -import itertools import sys @@ -167,7 +163,7 @@ def GetWeight(): conv_weight = torch.cat( [GetWeight() for _ in range(self.random_feature_size)], dim=1 ) - conv = lambda input: F.conv1d(input, conv_weight, padding=0) + conv = lambda input: F.conv1d(input, conv_weight, padding=0) # noqa: E731 return conv, windows_size def GetDisjoint(self, gap, indexes): diff --git a/graph_net/torch/rp_expr/rp_expr_util.py b/graph_net/torch/rp_expr/rp_expr_util.py index dbe20c5ed..43bd35e7d 100644 --- a/graph_net/torch/rp_expr/rp_expr_util.py +++ b/graph_net/torch/rp_expr/rp_expr_util.py @@ -2,7 +2,6 @@ from graph_net.torch.rp_expr.rp_expr import LetsListTokenRpExpr from graph_net.torch.rp_expr.nested_range import Range, Tree from collections import defaultdict -from dataclasses import dataclass def MakeNestedIndexRangeFromLetsListTokenRpExpr( diff --git a/graph_net/torch/shape_prop.py b/graph_net/torch/shape_prop.py index d1111098b..14bb1f461 100644 --- a/graph_net/torch/shape_prop.py +++ b/graph_net/torch/shape_prop.py @@ -1,5 +1,4 @@ import torch -from typing import Union, Callable from torch.fx.passes.shape_prop import ShapeProp import inspect import os @@ -33,4 +32,4 @@ def forward(self, *args, **kwargs): inputs = [ kwargs[name] for name in inspect.signature(self.module.forward).parameters ] - propagated_model = ShapeProp(traced_model).propagate(*inputs) + ShapeProp(traced_model).propagate(*inputs) diff --git a/graph_net/torch/single_device_runner.py b/graph_net/torch/single_device_runner.py index 98dbd5eec..a12ece963 100644 --- a/graph_net/torch/single_device_runner.py +++ b/graph_net/torch/single_device_runner.py @@ -1,12 +1,9 @@ from . import utils import argparse import importlib.util -import inspect import torch import logging -from pathlib import Path -from typing import Type, Any -import sys +from typing import Type from graph_net.torch.extractor import extract import hashlib import json diff --git a/graph_net/torch/typical_sequence_split_points.py b/graph_net/torch/typical_sequence_split_points.py index f85e9f228..86207aec5 100644 --- a/graph_net/torch/typical_sequence_split_points.py +++ b/graph_net/torch/typical_sequence_split_points.py @@ -1,17 +1,12 @@ import argparse import json -import os from pathlib import Path -from typing import Any, Dict, List -import torch -import torch.nn as nn +from typing import Dict, List from graph_net.torch.rp_expr.rp_expr_parser import RpExprParser from graph_net.torch.rp_expr.rp_expr_util import ( MakeNestedIndexRangeFromLetsListTokenRpExpr, ) -from graph_net.torch.fx_graph_module_util import get_torch_module_and_inputs -from graph_net.torch.fx_graph_parse_util import parse_sole_graph_module_without_varify class SplitAnalyzer: @@ -365,4 +360,4 @@ def _get_model_subgraph_ranges( help="maximum number of sequence operators", ) args = parser.parse_args() - main(args) \ No newline at end of file + main(args) diff --git a/pyproject.toml b/pyproject.toml index 610c946d0..08e8a8992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,4 +34,7 @@ include = [ "LICENSE", ] - +[tool.ruff.lint.per-file-ignores] +# F821: numpy string annotations like np.ndarray["N", np.int64] trigger false positives +"graph_net/torch/rp_expr/rp_expr.py" = ["F821"] +"graph_net/torch/rp_expr/rp_expr_passes.py" = ["F821"]