From f988ff1888f7a0ca6c9a2a638c15dc570a182314 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sat, 30 Sep 2023 18:28:42 -0400 Subject: [PATCH 1/2] Add pretty formatting for types and template specializations - Fixes #3 --- cxxheaderparser/tokfmt.py | 23 ++- cxxheaderparser/types.py | 146 +++++++++++++++--- tests/test_typefmt.py | 313 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 461 insertions(+), 21 deletions(-) create mode 100644 tests/test_typefmt.py diff --git a/cxxheaderparser/tokfmt.py b/cxxheaderparser/tokfmt.py index f2bb67c..c607f93 100644 --- a/cxxheaderparser/tokfmt.py +++ b/cxxheaderparser/tokfmt.py @@ -1,7 +1,7 @@ +from dataclasses import dataclass, field import typing from .lexer import LexToken, PlyLexer, LexerTokenStream -from .types import Token # key: token type, value: (left spacing, right spacing) _want_spacing = { @@ -35,6 +35,27 @@ _want_spacing.update(dict.fromkeys(PlyLexer.keywords, (2, 2))) +@dataclass +class Token: + """ + In an ideal world, this Token class would not be exposed via the user + visible API. Unfortunately, getting to that point would take a significant + amount of effort. + + It is not expected that these will change, but they might. + + At the moment, the only supported use of Token objects are in conjunction + with the ``tokfmt`` function. As this library matures, we'll try to clarify + the expectations around these. File an issue on github if you have ideas! + """ + + #: Raw value of the token + value: str + + #: Lex type of the token + type: str = field(repr=False, compare=False, default="") + + def tokfmt(toks: typing.List[Token]) -> str: """ Helper function that takes a list of tokens and converts them to a string diff --git a/cxxheaderparser/types.py b/cxxheaderparser/types.py index 1b8765e..c594001 100644 --- a/cxxheaderparser/types.py +++ b/cxxheaderparser/types.py @@ -1,26 +1,7 @@ import typing from dataclasses import dataclass, field - -@dataclass -class Token: - """ - In an ideal world, this Token class would not be exposed via the user - visible API. Unfortunately, getting to that point would take a significant - amount of effort. - - It is not expected that these will change, but they might. - - At the moment, the only supported use of Token objects are in conjunction - with the ``tokfmt`` function. As this library matures, we'll try to clarify - the expectations around these. File an issue on github if you have ideas! - """ - - #: Raw value of the token - value: str - - #: Lex type of the token - type: str = field(repr=False, compare=False, default="") +from .tokfmt import tokfmt, Token @dataclass @@ -37,6 +18,9 @@ class Value: #: Tokens corresponding to the value tokens: typing.List[Token] + def format(self) -> str: + return tokfmt(self.tokens) + @dataclass class NamespaceAlias: @@ -94,6 +78,9 @@ class DecltypeSpecifier: #: Unparsed tokens within the decltype tokens: typing.List[Token] + def format(self) -> str: + return f"decltype({tokfmt(self.tokens)})" + @dataclass class FundamentalSpecifier: @@ -107,6 +94,9 @@ class FundamentalSpecifier: name: str + def format(self) -> str: + return self.name + @dataclass class NameSpecifier: @@ -124,6 +114,12 @@ class NameSpecifier: specialization: typing.Optional["TemplateSpecialization"] = None + def format(self) -> str: + if self.specialization: + return f"{self.name}{self.specialization.format()}" + else: + return self.name + @dataclass class AutoSpecifier: @@ -133,6 +129,9 @@ class AutoSpecifier: name: str = "auto" + def format(self) -> str: + return self.name + @dataclass class AnonymousName: @@ -145,6 +144,10 @@ class AnonymousName: #: Unique id associated with this name (only unique per parser instance!) id: int + def format(self) -> str: + # TODO: not sure what makes sense here, subject to change + return f"<>" + PQNameSegment = typing.Union[ AnonymousName, FundamentalSpecifier, NameSpecifier, DecltypeSpecifier, AutoSpecifier @@ -170,6 +173,13 @@ class PQName: #: Set to true if the type was preceded with 'typename' has_typename: bool = False + def format(self) -> str: + tn = "typename " if self.has_typename else "" + if self.classkey: + return f"{tn}{self.classkey} {'::'.join(seg.format() for seg in self.segments)}" + else: + return tn + "::".join(seg.format() for seg in self.segments) + @dataclass class TemplateArgument: @@ -189,6 +199,12 @@ class TemplateArgument: param_pack: bool = False + def format(self) -> str: + if self.param_pack: + return f"{self.arg.format()}..." + else: + return self.arg.format() + @dataclass class TemplateSpecialization: @@ -204,6 +220,9 @@ class TemplateSpecialization: args: typing.List[TemplateArgument] + def format(self) -> str: + return f"<{', '.join(arg.format() for arg in self.args)}>" + @dataclass class FunctionType: @@ -238,6 +257,23 @@ class FunctionType: #: calling convention msvc_convention: typing.Optional[str] = None + def format(self) -> str: + vararg = "..." if self.vararg else "" + params = ", ".join(p.format() for p in self.parameters) + if self.has_trailing_return: + return f"auto ({params}{vararg}) -> {self.return_type.format()}" + else: + return f"{self.return_type.format()} ({params}{vararg})" + + def format_decl(self, name: str) -> str: + """Format as a named declaration""" + vararg = "..." if self.vararg else "" + params = ", ".join(p.format() for p in self.parameters) + if self.has_trailing_return: + return f"auto {name}({params}{vararg}) -> {self.return_type.format()}" + else: + return f"{self.return_type.format()} {name}({params}{vararg})" + @dataclass class Type: @@ -250,6 +286,17 @@ class Type: const: bool = False volatile: bool = False + def format(self) -> str: + c = "const " if self.const else "" + v = "volatile " if self.volatile else "" + return f"{c}{v}{self.typename.format()}" + + def format_decl(self, name: str): + """Format as a named declaration""" + c = "const " if self.const else "" + v = "volatile " if self.volatile else "" + return f"{c}{v}{self.typename.format()} {name}" + @dataclass class Array: @@ -269,6 +316,14 @@ class Array: #: ~~ size: typing.Optional[Value] + def format(self) -> str: + s = self.size.format() if self.size else "" + return f"{self.array_of.format()}[{s}]" + + def format_decl(self, name: str) -> str: + s = self.size.format() if self.size else "" + return f"{self.array_of.format()} {name}[{s}]" + @dataclass class Pointer: @@ -282,6 +337,25 @@ class Pointer: const: bool = False volatile: bool = False + def format(self) -> str: + c = " const" if self.const else "" + v = " volatile" if self.volatile else "" + ptr_to = self.ptr_to + if isinstance(ptr_to, (Array, FunctionType)): + return ptr_to.format_decl(f"(*{c}{v})") + else: + return f"{ptr_to.format()}*{c}{v}" + + def format_decl(self, name: str): + """Format as a named declaration""" + c = " const" if self.const else "" + v = " volatile" if self.volatile else "" + ptr_to = self.ptr_to + if isinstance(ptr_to, (Array, FunctionType)): + return ptr_to.format_decl(f"(*{c}{v} {name})") + else: + return f"{ptr_to.format()}*{c}{v} {name}" + @dataclass class Reference: @@ -291,6 +365,22 @@ class Reference: ref_to: typing.Union[Array, FunctionType, Pointer, Type] + def format(self) -> str: + ref_to = self.ref_to + if isinstance(ref_to, Array): + return ref_to.format_decl("(&)") + else: + return f"{ref_to.format()}&" + + def format_decl(self, name: str): + """Format as a named declaration""" + ref_to = self.ref_to + + if isinstance(ref_to, Array): + return ref_to.format_decl(f"(& {name})") + else: + return f"{ref_to.format()}& {name}" + @dataclass class MoveReference: @@ -300,6 +390,13 @@ class MoveReference: moveref_to: typing.Union[Array, FunctionType, Pointer, Type] + def format(self) -> str: + return f"{self.moveref_to.format()}&&" + + def format_decl(self, name: str): + """Format as a named declaration""" + return f"{self.moveref_to.format()}&& {name}" + #: A type or function type that is decorated with various things #: @@ -505,6 +602,15 @@ class Parameter: default: typing.Optional[Value] = None param_pack: bool = False + def format(self) -> str: + default = f" = {self.default.format()}" if self.default else "" + pp = "... " if self.param_pack else "" + name = self.name + if name: + return f"{self.type.format_decl(f'{pp}{name}')}{default}" + else: + return f"{self.type.format()}{pp}{default}" + @dataclass class Function: diff --git a/tests/test_typefmt.py b/tests/test_typefmt.py new file mode 100644 index 0000000..8aa377a --- /dev/null +++ b/tests/test_typefmt.py @@ -0,0 +1,313 @@ +import typing + +import pytest + +from cxxheaderparser.tokfmt import Token +from cxxheaderparser.types import ( + Array, + DecoratedType, + FunctionType, + FundamentalSpecifier, + Method, + MoveReference, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateSpecialization, + TemplateDecl, + Type, + Value, +) + + +@pytest.mark.parametrize( + "pytype,typestr,declstr", + [ + ( + Type(typename=PQName(segments=[FundamentalSpecifier(name="int")])), + "int", + "int name", + ), + ( + Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]), const=True + ), + "const int", + "const int name", + ), + ( + Type( + typename=PQName(segments=[NameSpecifier(name="S")], classkey="struct") + ), + "struct S", + "struct S name", + ), + ( + Pointer( + ptr_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ), + "int*", + "int* name", + ), + ( + Pointer( + ptr_to=Pointer( + ptr_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ) + ), + "int**", + "int** name", + ), + ( + Reference( + ref_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ), + "int&", + "int& name", + ), + ( + Reference( + ref_to=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ) + ), + "int (&)[3]", + "int (& name)[3]", + ), + ( + MoveReference( + moveref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="T"), NameSpecifier(name="T")] + ) + ) + ), + "T::T&&", + "T::T&& name", + ), + ( + Pointer( + ptr_to=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ) + ), + "int (*)[3]", + "int (* name)[3]", + ), + ( + Pointer( + ptr_to=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ), + const=True, + ), + "int (* const)[3]", + "int (* const name)[3]", + ), + ( + FunctionType( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ) + ], + ), + "int (int)", + "int name(int)", + ), + ( + FunctionType( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ) + ], + has_trailing_return=True, + ), + "auto (int) -> int", + "auto name(int) -> int", + ), + ( + FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")], + ), + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")], + ), + ), + name="a", + ), + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")], + ), + ), + name="b", + ), + ], + ), + "void (int a, int b)", + "void name(int a, int b)", + ), + ( + Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ], + ) + ), + "int (*)(int)", + "int (* name)(int)", + ), + ( + Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="function", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + arg=FunctionType( + return_type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier( + name="int" + ) + ] + ) + ) + ) + ], + ) + ) + ] + ), + ), + ] + ) + ), + "std::function", + "std::function name", + ), + ( + Type( + typename=PQName( + segments=[ + NameSpecifier( + name="foo", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + arg=Type( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="T"), + ], + ) + ), + ) + ] + ), + ) + ] + ), + ), + "foo<::T>", + "foo<::T> name", + ), + ( + Type( + typename=PQName( + segments=[ + NameSpecifier( + name="foo", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + arg=Type( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="T"), + ], + has_typename=True, + ) + ), + ) + ] + ), + ) + ] + ), + ), + "foo", + "foo name", + ), + ], +) +def test_typefmt( + pytype: typing.Union[DecoratedType, FunctionType], typestr: str, declstr: str +): + # basic formatting + assert pytype.format() == typestr + + # as a type declaration + assert pytype.format_decl("name") == declstr From aef8128e13279f550ed15709957df2db23f52008 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Tue, 3 Oct 2023 11:03:34 -0400 Subject: [PATCH 2/2] preprocessor timing mods --- cxxheaderparser/preprocessor.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cxxheaderparser/preprocessor.py b/cxxheaderparser/preprocessor.py index 33382dd..5e16b23 100644 --- a/cxxheaderparser/preprocessor.py +++ b/cxxheaderparser/preprocessor.py @@ -9,6 +9,8 @@ import typing from .options import PreprocessorFunction +from time import monotonic + from pcpp import Preprocessor, OutputDirective, Action @@ -53,7 +55,25 @@ def _filter_self(fname: str, fp: typing.TextIO) -> str: return new_output.read() +class PPTime: + def __init__(self) -> None: + self.start = monotonic() + self.files = 0 + self.parse_total: float = 0 + self.write_total: float = 0 + self.filter_total: float = 0 + + def report(self): + total = monotonic() - self.start + print("-- report --") + print(f" files={self.files} total_time={total:.5f}") + print( + f" preprocessor: parse={self.parse_total:.5f}, write={self.write_total:.5f}, filter={self.filter_total:.5f}" + ) + + def make_pcpp_preprocessor( + pptime: PPTime, *, defines: typing.List[str] = [], include_paths: typing.List[str] = [], @@ -87,19 +107,27 @@ def _preprocess_file(filename: str, content: str) -> str: if not retain_all_content: pp.line_directive = "#line" + pptime.files += 1 + + now = monotonic() pp.parse(content, filename) + pptime.parse_total += monotonic() - now if pp.errors: raise PreprocessorError("\n".join(pp.errors)) elif pp.return_code: raise PreprocessorError("failed with exit code %d" % pp.return_code) + now = monotonic() fp = io.StringIO() pp.write(fp) fp.seek(0) + pptime.write_total += monotonic() - now if retain_all_content: return fp.read() else: + now = monotonic() + # pcpp emits the #line directive using the filename you pass in # but will rewrite it if it's on the include path it uses. This # is copied from pcpp: @@ -112,6 +140,8 @@ def _preprocess_file(filename: str, content: str) -> str: filename = filename.replace(os.sep, "/") break - return _filter_self(filename, fp) + flt = _filter_self(filename, fp) + pptime.filter_total += monotonic() - now + return flt return _preprocess_file pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy