diff --git a/mypyc/emitmodule.py b/mypyc/emitmodule.py index cb7cd82d9f2f..39b786fccfe7 100644 --- a/mypyc/emitmodule.py +++ b/mypyc/emitmodule.py @@ -19,8 +19,9 @@ from mypy.fscache import FileSystemCache from mypy.util import hash_digest -from mypyc import genops +from mypyc import genopsmain from mypyc.genopsprepare import load_type_map +from mypyc.genopsmapper import Mapper from mypyc.common import ( PREFIX, TOP_LEVEL_NAME, INT_PREFIX, MODULE_PREFIX, shared_lib_name, ) @@ -178,7 +179,7 @@ def parse_and_typecheck( def compile_scc_to_ir( scc: List[MypyFile], result: BuildResult, - mapper: genops.Mapper, + mapper: Mapper, compiler_options: CompilerOptions, errors: Errors, ) -> ModuleIRs: @@ -201,7 +202,7 @@ def compile_scc_to_ir( print("Compiling {}".format(", ".join(x.name for x in scc))) # Generate basic IR, with missing exception and refcount handling. - modules = genops.build_ir( + modules = genopsmain.build_ir( scc, result.graph, result.types, mapper, compiler_options, errors ) if errors.num_errors > 0: @@ -225,7 +226,7 @@ def compile_scc_to_ir( def compile_modules_to_ir( result: BuildResult, - mapper: genops.Mapper, + mapper: Mapper, compiler_options: CompilerOptions, errors: Errors, ) -> ModuleIRs: @@ -260,7 +261,7 @@ def compile_ir_to_c( groups: Groups, modules: ModuleIRs, result: BuildResult, - mapper: genops.Mapper, + mapper: Mapper, compiler_options: CompilerOptions, ) -> Dict[Optional[str], List[Tuple[str, str]]]: """Compile a collection of ModuleIRs to C source text. @@ -358,7 +359,7 @@ def write_cache( def load_scc_from_cache( scc: List[MypyFile], result: BuildResult, - mapper: genops.Mapper, + mapper: Mapper, ctx: DeserMaps, ) -> ModuleIRs: """Load IR for an SCC of modules from the cache. @@ -401,7 +402,7 @@ def compile_modules_to_c( """ # Construct a map from modules to what group they belong to group_map = {source.module: lib_name for group, lib_name in groups for source in group} - mapper = genops.Mapper(group_map) + mapper = Mapper(group_map) modules = compile_modules_to_ir(result, mapper, compiler_options, errors) ctext = compile_ir_to_c(groups, modules, result, mapper, compiler_options) diff --git a/mypyc/genclass.py b/mypyc/genclass.py index 009c1009fa0f..76fc4dd4a607 100644 --- a/mypyc/genclass.py +++ b/mypyc/genclass.py @@ -1,5 +1,5 @@ from typing import List, Optional, Union -from typing_extensions import overload, TYPE_CHECKING +from typing_extensions import overload from mypy.nodes import ( ClassDef, FuncDef, OverloadedFuncDef, PassStmt, AssignmentStmt, NameExpr, StrExpr, @@ -23,14 +23,13 @@ from mypyc.genopsutil import ( is_dataclass_decorator, get_func_def, is_dataclass, is_constant, add_self_to_env ) +from mypyc.genfunc import BuildFuncIR from mypyc.common import SELF_NAME - -if TYPE_CHECKING: - from mypyc.genops import IRBuilder +from mypyc.genops import IRBuilder class BuildClassIR: - def __init__(self, builder: 'IRBuilder') -> None: + def __init__(self, builder: IRBuilder) -> None: self.builder = builder self.mapper = builder.mapper self.module_name = builder.module_name @@ -82,7 +81,7 @@ def visit_class_def(self, cdef: ClassDef) -> None: stmt.line) for item in stmt.items: with self.builder.catch_errors(stmt.line): - self.builder.visit_method(cdef, non_ext, get_func_def(item)) + BuildFuncIR(self.builder).visit_method(cdef, non_ext, get_func_def(item)) elif isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef)): # Ignore plugin generated methods (since they have no # bodies to compile and will need to have the bodies @@ -90,7 +89,7 @@ def visit_class_def(self, cdef: ClassDef) -> None: if cdef.info.names[stmt.name].plugin_generated: continue with self.builder.catch_errors(stmt.line): - self.builder.visit_method(cdef, non_ext, get_func_def(stmt)) + BuildFuncIR(self.builder).visit_method(cdef, non_ext, get_func_def(stmt)) elif isinstance(stmt, PassStmt): continue elif isinstance(stmt, AssignmentStmt): @@ -417,7 +416,7 @@ def load_decorated_class(self, cdef: ClassDef, type_obj: Value) -> Value: decorators = cdef.decorators dec_class = type_obj for d in reversed(decorators): - decorator = d.accept(self.builder) + decorator = d.accept(self.builder.visitor) assert isinstance(decorator, Value) dec_class = self.builder.py_call(decorator, [dec_class], dec_class.line) return dec_class diff --git a/mypyc/genexpr.py b/mypyc/genexpr.py index 3f827556c9ac..6f1302b4b667 100644 --- a/mypyc/genexpr.py +++ b/mypyc/genexpr.py @@ -4,7 +4,6 @@ """ from typing import List, Optional, Union -from typing_extensions import TYPE_CHECKING from mypy.nodes import ( Expression, NameExpr, MemberExpr, SuperExpr, CallExpr, UnaryExpr, OpExpr, IndexExpr, @@ -25,13 +24,11 @@ from mypyc.ops_dict import new_dict_op, dict_set_item_op from mypyc.ops_set import new_set_op, set_add_op, set_update_op from mypyc.specialize import specializers - -if TYPE_CHECKING: - from mypyc.genops import IRBuilder +from mypyc.genops import IRBuilder class BuildExpressionIR: - def __init__(self, builder: 'IRBuilder') -> None: + def __init__(self, builder: IRBuilder) -> None: self.builder = builder # Name and attribute references diff --git a/mypyc/genfunc.py b/mypyc/genfunc.py index bfb629ebfbef..3ed76382a0aa 100644 --- a/mypyc/genfunc.py +++ b/mypyc/genfunc.py @@ -4,7 +4,6 @@ """ from typing import Optional, List, Tuple, Union -from typing_extensions import TYPE_CHECKING from mypy.nodes import ( ClassDef, FuncDef, OverloadedFuncDef, Decorator, Var, YieldFromExpr, AwaitExpr, YieldExpr, @@ -31,13 +30,11 @@ from mypyc.genopsutil import concrete_arg_kind, is_constant, add_self_to_env from mypyc.genopscontext import FuncInfo, GeneratorClass, ImplicitClass from mypyc.genstatement import BuildStatementIR - -if TYPE_CHECKING: - from mypyc.genops import IRBuilder +from mypyc.genops import IRBuilder class BuildFuncIR: - def __init__(self, builder: 'IRBuilder') -> None: + def __init__(self, builder: IRBuilder) -> None: self.builder = builder self.module_name = builder.module_name self.functions = builder.functions @@ -878,7 +875,7 @@ def load_decorated_func(self, fdef: FuncDef, orig_func_reg: Value) -> Value: decorators = self.builder.fdefs_to_decorators[fdef] func_reg = orig_func_reg for d in reversed(decorators): - decorator = d.accept(self.builder) + decorator = d.accept(self.builder.visitor) assert isinstance(decorator, Value) func_reg = self.builder.py_call(decorator, [func_reg], func_reg.line) return func_reg diff --git a/mypyc/genops.py b/mypyc/genops.py index 99e9d71d1faa..85c88c2b29c3 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -1,48 +1,32 @@ -"""Transform a mypy AST to the IR form (Intermediate Representation). +"""Builder class used to transform a mypy AST to the IR form. -For example, consider a function like this: +The IRBuilder class maintains transformation state and provides access +to various helpers used to implement the transform. - def f(x: int) -> int: - return x * 2 + 1 +The top-level transform control logic is in mypyc.genopsmain. -It would be translated to something that conceptually looks like this: - - r0 = 2 - r1 = 1 - r2 = x * r0 :: int - r3 = r2 + r1 :: int - return r3 - -The IR is implemented in mypyc.ops. +mypyc.genopsvisitor.IRBuilderVisitor is used to dispatch based on mypy +AST node type to code that actually does the bulk of the work. For +example, expressions are transformed in mypyc.genexpr and functions are +transformed in mypyc.genfunc. """ -from typing import ( - TypeVar, Callable, Dict, List, Tuple, Optional, Union, Sequence, Set, Any, cast -) -from typing_extensions import overload, NoReturn +from typing import Callable, Dict, List, Tuple, Optional, Union, Sequence, Set, Any +from typing_extensions import overload from collections import OrderedDict import importlib.util from mypy.build import Graph from mypy.nodes import ( - MypyFile, SymbolNode, Statement, FuncDef, ReturnStmt, AssignmentStmt, OpExpr, - IntExpr, NameExpr, LDEF, Var, IfStmt, UnaryExpr, ComparisonExpr, WhileStmt, CallExpr, - IndexExpr, Block, Expression, ListExpr, ExpressionStmt, MemberExpr, ForStmt, RefExpr, Lvalue, - BreakStmt, ContinueStmt, ConditionalExpr, OperatorAssignmentStmt, TupleExpr, ClassDef, - TypeInfo, Import, ImportFrom, ImportAll, DictExpr, StrExpr, CastExpr, TempNode, - PassStmt, PromoteExpr, AssignmentExpr, AwaitExpr, BackquoteExpr, AssertStmt, BytesExpr, - ComplexExpr, Decorator, DelStmt, DictionaryComprehension, EllipsisExpr, EnumCallExpr, ExecStmt, - FloatExpr, GeneratorExpr, GlobalDecl, LambdaExpr, ListComprehension, SetComprehension, - NamedTupleExpr, NewTypeExpr, NonlocalDecl, OverloadedFuncDef, PrintStmt, RaiseStmt, - RevealExpr, SetExpr, SliceExpr, StarExpr, SuperExpr, TryStmt, TypeAliasExpr, TypeApplication, - TypeVarExpr, TypedDictExpr, UnicodeExpr, WithStmt, YieldFromExpr, YieldExpr, GDEF, ARG_POS, - ARG_NAMED, + MypyFile, SymbolNode, Statement, OpExpr, IntExpr, NameExpr, LDEF, Var, UnaryExpr, + CallExpr, IndexExpr, Expression, MemberExpr, RefExpr, Lvalue, TupleExpr, ClassDef, + TypeInfo, Import, ImportFrom, ImportAll, Decorator, GeneratorExpr, OverloadedFuncDef, + StarExpr, GDEF, ARG_POS, ARG_NAMED ) from mypy.types import ( Type, Instance, TupleType, AnyType, TypeOfAny, UninhabitedType, get_proper_type ) from mypy.visitor import ExpressionVisitor, StatementVisitor -from mypy.state import strict_optional_set from mypy.util import split_target from mypyc.common import TEMP_ATTR_NAME, TOP_LEVEL_NAME @@ -51,7 +35,7 @@ def f(x: int) -> int: BasicBlock, AssignmentTarget, AssignmentTargetRegister, AssignmentTargetIndex, AssignmentTargetAttr, AssignmentTargetTuple, Environment, LoadInt, RType, Value, Register, Op, FuncIR, Assign, Branch, RTuple, Unreachable, - TupleGet, ClassIR, NonExtClassInfo, RInstance, ModuleIR, ModuleIRs, GetAttr, SetAttr, + TupleGet, ClassIR, NonExtClassInfo, RInstance, GetAttr, SetAttr, LoadStatic, InitStatic, INVALID_FUNC_DEF, int_rprimitive, bool_rprimitive, list_rprimitive, is_list_rprimitive, dict_rprimitive, set_rprimitive, str_rprimitive, none_rprimitive, is_none_rprimitive, object_rprimitive, @@ -71,74 +55,23 @@ def f(x: int) -> int: from mypyc.nonlocalcontrol import ( NonlocalControl, BaseNonlocalControl, LoopNonlocalControl, GeneratorNonlocalControl ) -from mypyc.genclass import BuildClassIR -from mypyc.genfunc import BuildFuncIR -from mypyc.genstatement import BuildStatementIR -from mypyc.genexpr import BuildExpressionIR from mypyc.genopscontext import FuncInfo, ImplicitClass from mypyc.genopsmapper import Mapper -from mypyc.genopsvtable import compute_vtable -from mypyc.genopsprepare import build_type_map from mypyc.ir_builder import LowLevelIRBuilder from mypyc.specialize import specialize_function GenFunc = Callable[[], None] -class UnsupportedException(Exception): +class IRVisitor(ExpressionVisitor[Value], StatementVisitor[None]): pass -# The stubs for callable contextmanagers are busted so cast it to the -# right type... -F = TypeVar('F', bound=Callable[..., Any]) -strict_optional_dec = cast(Callable[[F], F], strict_optional_set(True)) - - -@strict_optional_dec # Turn on strict optional for any type manipulations we do -def build_ir(modules: List[MypyFile], - graph: Graph, - types: Dict[Expression, Type], - mapper: 'Mapper', - options: CompilerOptions, - errors: Errors) -> ModuleIRs: - - build_type_map(mapper, modules, graph, types, options, errors) - - result = OrderedDict() # type: ModuleIRs - - # Generate IR for all modules. - class_irs = [] - - for module in modules: - # First pass to determine free symbols. - pbv = PreBuildVisitor() - module.accept(pbv) - - # Second pass. - builder = IRBuilder( - module.fullname, types, graph, errors, mapper, pbv, options - ) - builder.visit_mypy_file(module) - module_ir = ModuleIR( - module.fullname, - list(builder.imports), - builder.functions, - builder.classes, - builder.final_names - ) - result[module.fullname] = module_ir - class_irs.extend(builder.classes) - - # Compute vtables. - for cir in class_irs: - if cir.is_ext_class: - compute_vtable(cir) - - return result +class UnsupportedException(Exception): + pass -class IRBuilder(ExpressionVisitor[Value], StatementVisitor[None]): +class IRBuilder: def __init__(self, current_module: str, types: Dict[Expression, Type], @@ -146,6 +79,7 @@ def __init__(self, errors: Errors, mapper: Mapper, pbv: PreBuildVisitor, + visitor: IRVisitor, options: CompilerOptions) -> None: self.builder = LowLevelIRBuilder(current_module, mapper) self.builders = [self.builder] @@ -174,6 +108,8 @@ def __init__(self, self.nested_fitems = pbv.nested_funcs.keys() self.fdefs_to_decorators = pbv.funcs_to_decorators + self.visitor = visitor + # This list operates similarly to a function call stack for nested functions. Whenever a # function definition begins to be generated, a FuncInfo instance is added to the stack, # and information about that function (e.g. whether it is nested, its environment class to @@ -298,13 +234,6 @@ def visit_mypy_file(self, mypyfile: MypyFile) -> None: traceback_name="") self.functions.append(func_ir) - def visit_method( - self, cdef: ClassDef, non_ext: Optional[NonExtClassInfo], fdef: FuncDef) -> None: - BuildFuncIR(self).visit_method(cdef, non_ext, fdef) - - def visit_class_def(self, cdef: ClassDef) -> None: - BuildClassIR(self).visit_class_def(cdef) - def add_to_non_ext_dict(self, non_ext: NonExtClassInfo, key: str, val: Value, line: int) -> None: # Add an attribute entry into the class dict of a non-extension class. @@ -408,12 +337,6 @@ def maybe_add_implicit_return(self) -> None: else: self.add_implicit_unreachable() - def visit_func_def(self, fdef: FuncDef) -> None: - BuildFuncIR(self).visit_func_def(fdef) - - def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: - BuildFuncIR(self).visit_overloaded_func_def(o) - def add_implicit_return(self) -> None: block = self.builder.blocks[-1] if not block.terminated: @@ -1106,150 +1029,6 @@ def loop_contents( handle_loop(loop_params) - def visit_decorator(self, dec: Decorator) -> None: - BuildFuncIR(self).visit_decorator(dec) - - def visit_block(self, block: Block) -> None: - BuildStatementIR(self).visit_block(block) - - # Statements - - def visit_expression_stmt(self, stmt: ExpressionStmt) -> None: - BuildStatementIR(self).visit_expression_stmt(stmt) - - def visit_return_stmt(self, stmt: ReturnStmt) -> None: - BuildStatementIR(self).visit_return_stmt(stmt) - - def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None: - BuildStatementIR(self).visit_assignment_stmt(stmt) - - def visit_operator_assignment_stmt(self, stmt: OperatorAssignmentStmt) -> None: - BuildStatementIR(self).visit_operator_assignment_stmt(stmt) - - def visit_if_stmt(self, stmt: IfStmt) -> None: - BuildStatementIR(self).visit_if_stmt(stmt) - - def visit_while_stmt(self, stmt: WhileStmt) -> None: - BuildStatementIR(self).visit_while_stmt(stmt) - - def visit_for_stmt(self, stmt: ForStmt) -> None: - BuildStatementIR(self).visit_for_stmt(stmt) - - def visit_break_stmt(self, stmt: BreakStmt) -> None: - BuildStatementIR(self).visit_break_stmt(stmt) - - def visit_continue_stmt(self, stmt: ContinueStmt) -> None: - BuildStatementIR(self).visit_continue_stmt(stmt) - - def visit_raise_stmt(self, stmt: RaiseStmt) -> None: - BuildStatementIR(self).visit_raise_stmt(stmt) - - def visit_try_stmt(self, stmt: TryStmt) -> None: - BuildStatementIR(self).visit_try_stmt(stmt) - - def visit_with_stmt(self, stmt: WithStmt) -> None: - BuildStatementIR(self).visit_with_stmt(stmt) - - def visit_pass_stmt(self, stmt: PassStmt) -> None: - pass - - def visit_assert_stmt(self, stmt: AssertStmt) -> None: - BuildStatementIR(self).visit_assert_stmt(stmt) - - def visit_del_stmt(self, stmt: DelStmt) -> None: - BuildStatementIR(self).visit_del_stmt(stmt) - - def visit_global_decl(self, stmt: GlobalDecl) -> None: - # Pure declaration -- no runtime effect - pass - - def visit_nonlocal_decl(self, stmt: NonlocalDecl) -> None: - # Pure declaration -- no runtime effect - pass - - # Expressions - - def visit_name_expr(self, expr: NameExpr) -> Value: - return BuildExpressionIR(self).visit_name_expr(expr) - - def visit_member_expr(self, expr: MemberExpr) -> Value: - return BuildExpressionIR(self).visit_member_expr(expr) - - def visit_super_expr(self, expr: SuperExpr) -> Value: - return BuildExpressionIR(self).visit_super_expr(expr) - - def visit_call_expr(self, expr: CallExpr) -> Value: - return BuildExpressionIR(self).visit_call_expr(expr) - - def visit_unary_expr(self, expr: UnaryExpr) -> Value: - return BuildExpressionIR(self).visit_unary_expr(expr) - - def visit_op_expr(self, expr: OpExpr) -> Value: - return BuildExpressionIR(self).visit_op_expr(expr) - - def visit_index_expr(self, expr: IndexExpr) -> Value: - return BuildExpressionIR(self).visit_index_expr(expr) - - def visit_conditional_expr(self, expr: ConditionalExpr) -> Value: - return BuildExpressionIR(self).visit_conditional_expr(expr) - - def visit_comparison_expr(self, expr: ComparisonExpr) -> Value: - return BuildExpressionIR(self).visit_comparison_expr(expr) - - def visit_int_expr(self, expr: IntExpr) -> Value: - return BuildExpressionIR(self).visit_int_expr(expr) - - def visit_float_expr(self, expr: FloatExpr) -> Value: - return BuildExpressionIR(self).visit_float_expr(expr) - - def visit_complex_expr(self, expr: ComplexExpr) -> Value: - return BuildExpressionIR(self).visit_complex_expr(expr) - - def visit_str_expr(self, expr: StrExpr) -> Value: - return BuildExpressionIR(self).visit_str_expr(expr) - - def visit_bytes_expr(self, expr: BytesExpr) -> Value: - return BuildExpressionIR(self).visit_bytes_expr(expr) - - def visit_ellipsis(self, expr: EllipsisExpr) -> Value: - return BuildExpressionIR(self).visit_ellipsis(expr) - - def visit_list_expr(self, expr: ListExpr) -> Value: - return BuildExpressionIR(self).visit_list_expr(expr) - - def visit_tuple_expr(self, expr: TupleExpr) -> Value: - return BuildExpressionIR(self).visit_tuple_expr(expr) - - def visit_dict_expr(self, expr: DictExpr) -> Value: - return BuildExpressionIR(self).visit_dict_expr(expr) - - def visit_set_expr(self, expr: SetExpr) -> Value: - return BuildExpressionIR(self).visit_set_expr(expr) - - def visit_list_comprehension(self, expr: ListComprehension) -> Value: - return BuildExpressionIR(self).visit_list_comprehension(expr) - - def visit_set_comprehension(self, expr: SetComprehension) -> Value: - return BuildExpressionIR(self).visit_set_comprehension(expr) - - def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> Value: - return BuildExpressionIR(self).visit_dictionary_comprehension(expr) - - def visit_slice_expr(self, expr: SliceExpr) -> Value: - return BuildExpressionIR(self).visit_slice_expr(expr) - - def visit_generator_expr(self, expr: GeneratorExpr) -> Value: - return BuildExpressionIR(self).visit_generator_expr(expr) - - def visit_lambda_expr(self, expr: LambdaExpr) -> Value: - return BuildFuncIR(self).visit_lambda_expr(expr) - - def visit_yield_expr(self, expr: YieldExpr) -> Value: - return BuildFuncIR(self).visit_yield_expr(expr) - - def visit_yield_from_expr(self, o: YieldFromExpr) -> Value: - return BuildFuncIR(self).visit_yield_from_expr(o) - # Builtin function special cases @specialize_function('builtins.globals') @@ -1434,66 +1213,6 @@ def flatten_classes(self, arg: Union[RefExpr, TupleExpr]) -> Optional[List[Class return None return res - def visit_await_expr(self, o: AwaitExpr) -> Value: - return BuildFuncIR(self).visit_await_expr(o) - - # Unimplemented constructs - def visit_assignment_expr(self, o: AssignmentExpr) -> Value: - self.bail("I Am The Walrus (unimplemented)", o.line) - - # Unimplemented constructs that shouldn't come up because they are py2 only - def visit_backquote_expr(self, o: BackquoteExpr) -> Value: - self.bail("Python 2 features are unsupported", o.line) - - def visit_exec_stmt(self, o: ExecStmt) -> None: - self.bail("Python 2 features are unsupported", o.line) - - def visit_print_stmt(self, o: PrintStmt) -> None: - self.bail("Python 2 features are unsupported", o.line) - - def visit_unicode_expr(self, o: UnicodeExpr) -> Value: - self.bail("Python 2 features are unsupported", o.line) - - # Constructs that shouldn't ever show up - def visit_enum_call_expr(self, o: EnumCallExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit__promote_expr(self, o: PromoteExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_namedtuple_expr(self, o: NamedTupleExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_newtype_expr(self, o: NewTypeExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_temp_node(self, o: TempNode) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_type_alias_expr(self, o: TypeAliasExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_type_application(self, o: TypeApplication) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_type_var_expr(self, o: TypeVarExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_typeddict_expr(self, o: TypedDictExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_reveal_expr(self, o: RevealExpr) -> Value: - assert False, "can't compile analysis-only expressions" - - def visit_var(self, o: Var) -> None: - assert False, "can't compile Var; should have been handled already?" - - def visit_cast_expr(self, o: CastExpr) -> Value: - assert False, "CastExpr should have been handled in CallExpr" - - def visit_star_expr(self, o: StarExpr) -> Value: - assert False, "should have been handled in Tuple/List/Set/DictExpr or CallExpr" - # Helpers def enter(self, fn_info: Union[FuncInfo, str] = '') -> None: @@ -1529,7 +1248,7 @@ def accept(self, node: Union[Statement, Expression]) -> Optional[Value]: with self.catch_errors(node.line): if isinstance(node, Expression): try: - res = node.accept(self) + res = node.accept(self.visitor) res = self.coerce(res, self.node_type(node), node.line) # If we hit an error during compilation, we want to # keep trying, so we can produce more error @@ -1540,7 +1259,7 @@ def accept(self, node: Union[Statement, Expression]) -> Optional[Value]: return res else: try: - node.accept(self) + node.accept(self.visitor) except UnsupportedException: pass return None @@ -1619,13 +1338,3 @@ def warning(self, msg: str, line: int) -> None: def error(self, msg: str, line: int) -> None: self.errors.error(msg, self.module_path, line) - - def bail(self, msg: str, line: int) -> 'NoReturn': - """Reports an error and aborts compilation up until the last accept() call - - (accept() catches the UnsupportedException and keeps on - processing. This allows errors to be non-blocking without always - needing to write handling for them. - """ - self.error(msg, line) - raise UnsupportedException() diff --git a/mypyc/genopsmain.py b/mypyc/genopsmain.py new file mode 100644 index 000000000000..2a59e1400780 --- /dev/null +++ b/mypyc/genopsmain.py @@ -0,0 +1,91 @@ +"""Transform a mypy AST to the IR form (Intermediate Representation). + +For example, consider a function like this: + + def f(x: int) -> int: + return x * 2 + 1 + +It would be translated to something that conceptually looks like this: + + r0 = 2 + r1 = 1 + r2 = x * r0 :: int + r3 = r2 + r1 :: int + return r3 + +The IR is implemented in mypyc.ops. + +For the core of the implementation, look at build_ir() below, +mypyc.genops, and mypyc.genopsvisitor. +""" + +from collections import OrderedDict +from typing import List, Dict, Callable, Any, TypeVar, cast + +from mypy.nodes import MypyFile, Expression +from mypy.types import Type +from mypy.state import strict_optional_set +from mypy.build import Graph + +from mypyc.errors import Errors +from mypyc.options import CompilerOptions +from mypyc.prebuildvisitor import PreBuildVisitor +from mypyc.genopsvtable import compute_vtable +from mypyc.genopsprepare import build_type_map +from mypyc.genops import IRBuilder +from mypyc.genopsvisitor import IRBuilderVisitor +from mypyc.ops import ModuleIR, ModuleIRs +from mypyc.genopsmapper import Mapper + + +# The stubs for callable contextmanagers are busted so cast it to the +# right type... +F = TypeVar('F', bound=Callable[..., Any]) +strict_optional_dec = cast(Callable[[F], F], strict_optional_set(True)) + + +@strict_optional_dec # Turn on strict optional for any type manipulations we do +def build_ir(modules: List[MypyFile], + graph: Graph, + types: Dict[Expression, Type], + mapper: 'Mapper', + options: CompilerOptions, + errors: Errors) -> ModuleIRs: + + build_type_map(mapper, modules, graph, types, options, errors) + + result = OrderedDict() # type: ModuleIRs + + # Generate IR for all modules. + class_irs = [] + + for module in modules: + # First pass to determine free symbols. + pbv = PreBuildVisitor() + module.accept(pbv) + + # Construct and configure builder objects (cyclic runtime dependency). + visitor = IRBuilderVisitor() + builder = IRBuilder( + module.fullname, types, graph, errors, mapper, pbv, visitor, options + ) + visitor.builder = builder + + # Second pass does the bulk of the work. + builder.visit_mypy_file(module) + module_ir = ModuleIR( + module.fullname, + list(builder.imports), + builder.functions, + builder.classes, + builder.final_names + ) + result[module.fullname] = module_ir + class_irs.extend(builder.classes) + + # Compute vtables. + for cir in class_irs: + if cir.is_ext_class: + compute_vtable(cir) + + return result diff --git a/mypyc/genopsvisitor.py b/mypyc/genopsvisitor.py new file mode 100644 index 000000000000..275c7c40ebb0 --- /dev/null +++ b/mypyc/genopsvisitor.py @@ -0,0 +1,285 @@ +"""Dispatcher used when transforming a mypy AST to the IR form. + +mypyc.genops and mypyc.genopsmain are closely related. +""" + +from typing_extensions import NoReturn + +from mypy.nodes import ( + MypyFile, FuncDef, ReturnStmt, AssignmentStmt, OpExpr, + IntExpr, NameExpr, Var, IfStmt, UnaryExpr, ComparisonExpr, WhileStmt, CallExpr, + IndexExpr, Block, ListExpr, ExpressionStmt, MemberExpr, ForStmt, + BreakStmt, ContinueStmt, ConditionalExpr, OperatorAssignmentStmt, TupleExpr, ClassDef, + Import, ImportFrom, ImportAll, DictExpr, StrExpr, CastExpr, TempNode, + PassStmt, PromoteExpr, AssignmentExpr, AwaitExpr, BackquoteExpr, AssertStmt, BytesExpr, + ComplexExpr, Decorator, DelStmt, DictionaryComprehension, EllipsisExpr, EnumCallExpr, ExecStmt, + FloatExpr, GeneratorExpr, GlobalDecl, LambdaExpr, ListComprehension, SetComprehension, + NamedTupleExpr, NewTypeExpr, NonlocalDecl, OverloadedFuncDef, PrintStmt, RaiseStmt, + RevealExpr, SetExpr, SliceExpr, StarExpr, SuperExpr, TryStmt, TypeAliasExpr, TypeApplication, + TypeVarExpr, TypedDictExpr, UnicodeExpr, WithStmt, YieldFromExpr, YieldExpr +) + +from mypyc.ops import Value +from mypyc.genops import IRVisitor, IRBuilder, UnsupportedException +from mypyc.genclass import BuildClassIR +from mypyc.genfunc import BuildFuncIR +from mypyc.genstatement import BuildStatementIR +from mypyc.genexpr import BuildExpressionIR + + +class IRBuilderVisitor(IRVisitor): + """Mypy node visitor that dispatches to node transform implementations. + + This class should have no non-trivial logic. + + This visitor is separated from the rest of code to improve modularity and + to avoid import cycles. + + This is based on the visitor pattern + (https://en.wikipedia.org/wiki/Visitor_pattern). + """ + + # This gets passed to all the implementations and contains all the + # state and many helpers. The attribute is initialized outside + # this class since this class and IRBuilder form a reference loop. + builder = None # type: IRBuilder + + def visit_mypy_file(self, mypyfile: MypyFile) -> None: + self.builder.visit_mypy_file(mypyfile) + + def visit_class_def(self, cdef: ClassDef) -> None: + BuildClassIR(self.builder).visit_class_def(cdef) + + def visit_import(self, node: Import) -> None: + self.builder.visit_import(node) + + def visit_import_from(self, node: ImportFrom) -> None: + self.builder.visit_import_from(node) + + def visit_import_all(self, node: ImportAll) -> None: + self.builder.visit_import_all(node) + + def visit_func_def(self, fdef: FuncDef) -> None: + BuildFuncIR(self.builder).visit_func_def(fdef) + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + BuildFuncIR(self.builder).visit_overloaded_func_def(o) + + def visit_decorator(self, dec: Decorator) -> None: + BuildFuncIR(self.builder).visit_decorator(dec) + + def visit_block(self, block: Block) -> None: + BuildStatementIR(self.builder).visit_block(block) + + # Statements + + def visit_expression_stmt(self, stmt: ExpressionStmt) -> None: + BuildStatementIR(self.builder).visit_expression_stmt(stmt) + + def visit_return_stmt(self, stmt: ReturnStmt) -> None: + BuildStatementIR(self.builder).visit_return_stmt(stmt) + + def visit_assignment_stmt(self, stmt: AssignmentStmt) -> None: + BuildStatementIR(self.builder).visit_assignment_stmt(stmt) + + def visit_operator_assignment_stmt(self, stmt: OperatorAssignmentStmt) -> None: + BuildStatementIR(self.builder).visit_operator_assignment_stmt(stmt) + + def visit_if_stmt(self, stmt: IfStmt) -> None: + BuildStatementIR(self.builder).visit_if_stmt(stmt) + + def visit_while_stmt(self, stmt: WhileStmt) -> None: + BuildStatementIR(self.builder).visit_while_stmt(stmt) + + def visit_for_stmt(self, stmt: ForStmt) -> None: + BuildStatementIR(self.builder).visit_for_stmt(stmt) + + def visit_break_stmt(self, stmt: BreakStmt) -> None: + BuildStatementIR(self.builder).visit_break_stmt(stmt) + + def visit_continue_stmt(self, stmt: ContinueStmt) -> None: + BuildStatementIR(self.builder).visit_continue_stmt(stmt) + + def visit_raise_stmt(self, stmt: RaiseStmt) -> None: + BuildStatementIR(self.builder).visit_raise_stmt(stmt) + + def visit_try_stmt(self, stmt: TryStmt) -> None: + BuildStatementIR(self.builder).visit_try_stmt(stmt) + + def visit_with_stmt(self, stmt: WithStmt) -> None: + BuildStatementIR(self.builder).visit_with_stmt(stmt) + + def visit_pass_stmt(self, stmt: PassStmt) -> None: + pass + + def visit_assert_stmt(self, stmt: AssertStmt) -> None: + BuildStatementIR(self.builder).visit_assert_stmt(stmt) + + def visit_del_stmt(self, stmt: DelStmt) -> None: + BuildStatementIR(self.builder).visit_del_stmt(stmt) + + def visit_global_decl(self, stmt: GlobalDecl) -> None: + # Pure declaration -- no runtime effect + pass + + def visit_nonlocal_decl(self, stmt: NonlocalDecl) -> None: + # Pure declaration -- no runtime effect + pass + + # Expressions + + def visit_name_expr(self, expr: NameExpr) -> Value: + return BuildExpressionIR(self.builder).visit_name_expr(expr) + + def visit_member_expr(self, expr: MemberExpr) -> Value: + return BuildExpressionIR(self.builder).visit_member_expr(expr) + + def visit_super_expr(self, expr: SuperExpr) -> Value: + return BuildExpressionIR(self.builder).visit_super_expr(expr) + + def visit_call_expr(self, expr: CallExpr) -> Value: + return BuildExpressionIR(self.builder).visit_call_expr(expr) + + def visit_unary_expr(self, expr: UnaryExpr) -> Value: + return BuildExpressionIR(self.builder).visit_unary_expr(expr) + + def visit_op_expr(self, expr: OpExpr) -> Value: + return BuildExpressionIR(self.builder).visit_op_expr(expr) + + def visit_index_expr(self, expr: IndexExpr) -> Value: + return BuildExpressionIR(self.builder).visit_index_expr(expr) + + def visit_conditional_expr(self, expr: ConditionalExpr) -> Value: + return BuildExpressionIR(self.builder).visit_conditional_expr(expr) + + def visit_comparison_expr(self, expr: ComparisonExpr) -> Value: + return BuildExpressionIR(self.builder).visit_comparison_expr(expr) + + def visit_int_expr(self, expr: IntExpr) -> Value: + return BuildExpressionIR(self.builder).visit_int_expr(expr) + + def visit_float_expr(self, expr: FloatExpr) -> Value: + return BuildExpressionIR(self.builder).visit_float_expr(expr) + + def visit_complex_expr(self, expr: ComplexExpr) -> Value: + return BuildExpressionIR(self.builder).visit_complex_expr(expr) + + def visit_str_expr(self, expr: StrExpr) -> Value: + return BuildExpressionIR(self.builder).visit_str_expr(expr) + + def visit_bytes_expr(self, expr: BytesExpr) -> Value: + return BuildExpressionIR(self.builder).visit_bytes_expr(expr) + + def visit_ellipsis(self, expr: EllipsisExpr) -> Value: + return BuildExpressionIR(self.builder).visit_ellipsis(expr) + + def visit_list_expr(self, expr: ListExpr) -> Value: + return BuildExpressionIR(self.builder).visit_list_expr(expr) + + def visit_tuple_expr(self, expr: TupleExpr) -> Value: + return BuildExpressionIR(self.builder).visit_tuple_expr(expr) + + def visit_dict_expr(self, expr: DictExpr) -> Value: + return BuildExpressionIR(self.builder).visit_dict_expr(expr) + + def visit_set_expr(self, expr: SetExpr) -> Value: + return BuildExpressionIR(self.builder).visit_set_expr(expr) + + def visit_list_comprehension(self, expr: ListComprehension) -> Value: + return BuildExpressionIR(self.builder).visit_list_comprehension(expr) + + def visit_set_comprehension(self, expr: SetComprehension) -> Value: + return BuildExpressionIR(self.builder).visit_set_comprehension(expr) + + def visit_dictionary_comprehension(self, expr: DictionaryComprehension) -> Value: + return BuildExpressionIR(self.builder).visit_dictionary_comprehension(expr) + + def visit_slice_expr(self, expr: SliceExpr) -> Value: + return BuildExpressionIR(self.builder).visit_slice_expr(expr) + + def visit_generator_expr(self, expr: GeneratorExpr) -> Value: + return BuildExpressionIR(self.builder).visit_generator_expr(expr) + + def visit_lambda_expr(self, expr: LambdaExpr) -> Value: + return BuildFuncIR(self.builder).visit_lambda_expr(expr) + + def visit_yield_expr(self, expr: YieldExpr) -> Value: + return BuildFuncIR(self.builder).visit_yield_expr(expr) + + def visit_yield_from_expr(self, o: YieldFromExpr) -> Value: + return BuildFuncIR(self.builder).visit_yield_from_expr(o) + + def visit_await_expr(self, o: AwaitExpr) -> Value: + return BuildFuncIR(self.builder).visit_await_expr(o) + + # Unimplemented constructs + + def visit_assignment_expr(self, o: AssignmentExpr) -> Value: + self.bail("I Am The Walrus (unimplemented)", o.line) + + # Unimplemented constructs that shouldn't come up because they are py2 only + + def visit_backquote_expr(self, o: BackquoteExpr) -> Value: + self.bail("Python 2 features are unsupported", o.line) + + def visit_exec_stmt(self, o: ExecStmt) -> None: + self.bail("Python 2 features are unsupported", o.line) + + def visit_print_stmt(self, o: PrintStmt) -> None: + self.bail("Python 2 features are unsupported", o.line) + + def visit_unicode_expr(self, o: UnicodeExpr) -> Value: + self.bail("Python 2 features are unsupported", o.line) + + # Constructs that shouldn't ever show up + + def visit_enum_call_expr(self, o: EnumCallExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit__promote_expr(self, o: PromoteExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_namedtuple_expr(self, o: NamedTupleExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_newtype_expr(self, o: NewTypeExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_temp_node(self, o: TempNode) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_alias_expr(self, o: TypeAliasExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_application(self, o: TypeApplication) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_type_var_expr(self, o: TypeVarExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_typeddict_expr(self, o: TypedDictExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_reveal_expr(self, o: RevealExpr) -> Value: + assert False, "can't compile analysis-only expressions" + + def visit_var(self, o: Var) -> None: + assert False, "can't compile Var; should have been handled already?" + + def visit_cast_expr(self, o: CastExpr) -> Value: + assert False, "CastExpr should have been handled in CallExpr" + + def visit_star_expr(self, o: StarExpr) -> Value: + assert False, "should have been handled in Tuple/List/Set/DictExpr or CallExpr" + + # Helpers + + def bail(self, msg: str, line: int) -> NoReturn: + """Reports an error and aborts compilation up until the last accept() call + + (accept() catches the UnsupportedException and keeps on + processing. This allows errors to be non-blocking without always + needing to write handling for them. + """ + self.builder.error(msg, line) + raise UnsupportedException() diff --git a/mypyc/genstatement.py b/mypyc/genstatement.py index 108cdbafe457..c74d67d9a991 100644 --- a/mypyc/genstatement.py +++ b/mypyc/genstatement.py @@ -1,5 +1,4 @@ from typing import Optional, List, Tuple, Sequence, Callable -from typing_extensions import TYPE_CHECKING from mypy.nodes import ( Block, ExpressionStmt, ReturnStmt, AssignmentStmt, OperatorAssignmentStmt, IfStmt, WhileStmt, @@ -20,16 +19,13 @@ from mypyc.nonlocalcontrol import ( ExceptNonlocalControl, FinallyNonlocalControl, TryFinallyNonlocalControl ) - - -if TYPE_CHECKING: - from mypyc.genops import IRBuilder +from mypyc.genops import IRBuilder GenFunc = Callable[[], None] class BuildStatementIR: - def __init__(self, builder: 'IRBuilder') -> None: + def __init__(self, builder: IRBuilder) -> None: self.builder = builder def visit_block(self, block: Block) -> None: @@ -50,7 +46,7 @@ def visit_expression_stmt(self, stmt: ExpressionStmt) -> None: # Docstring. Ignore return # ExpressionStmts do not need to be coerced like other Expressions. - stmt.expr.accept(self.builder) + stmt.expr.accept(self.builder.visitor) def visit_return_stmt(self, stmt: ReturnStmt) -> None: if stmt.expr: diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 6a74aec16f44..1ebcb686a9d6 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -12,7 +12,7 @@ ClassIR, RInstance, SetAttr, Op, Value, int_rprimitive, bool_rprimitive, list_rprimitive, dict_rprimitive, object_rprimitive, FuncSignature, ) -from mypyc.genops import compute_vtable +from mypyc.genopsvtable import compute_vtable from mypyc.emit import Emitter, EmitterContext from mypyc.emitfunc import generate_native_function, FunctionEmitterVisitor from mypyc.ops_primitive import binary_ops diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 4bc5cbe6a04d..4f6f44b3928e 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -14,10 +14,11 @@ from mypy.test.config import test_temp_dir from mypy.test.helpers import assert_string_arrays_equal -from mypyc import genops +from mypyc.genopsmain import build_ir from mypyc.options import CompilerOptions from mypyc.ops import FuncIR from mypyc.errors import Errors +from mypyc.genopsmapper import Mapper from mypyc.test.config import test_data_prefix # The builtins stub used during icode generation test cases. @@ -105,9 +106,9 @@ def build_ir_for_single_file(input_lines: List[str], raise CompileError(result.errors) errors = Errors() - modules = genops.build_ir( + modules = build_ir( [result.files['__main__']], result.graph, result.types, - genops.Mapper({'__main__': None}), + Mapper({'__main__': None}), compiler_options, errors) assert errors.num_errors == 0 pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy