Compare commits

..

No commits in common. "c5048309280fdf64304b5c7e83153fa866df2b5e" and "041f4c31fb05b1ce6be0a6eac332568fe7bbe3a8" have entirely different histories.

6 changed files with 66 additions and 432 deletions

View File

@ -10,7 +10,7 @@ import coloredlogs
from jasminesnake import __version__, __snake__, LOG_LEVELS
from .js_stream import JSBaseStream, JSStringStream, JSFileStream
from .lex.ErrorListeners import LogErrorListener
from .ast import to_ascii_tree, from_parse_tree
from ast import nodes, to_ascii_tree, from_parse_tree
def create_argument_parser():

View File

@ -4,14 +4,14 @@ from enum import Enum
from typing import Union
from antlr4 import ParseTreeWalker
import jasminesnake.lex.JavaScriptParser as Parser
from . import nodes
import lex.JavaScriptParser as Parser
import ast.nodes
from .parse_tree_listeners import ASTListener
JSP = Parser.JavaScriptParser
def from_parse_tree(tree: JSP.ProgramContext) -> nodes.Program:
def from_parse_tree(tree: JSP.ProgramContext) -> ast.nodes.Program:
"""Generate AST from ANTLR parse tree.
Args:
@ -26,7 +26,7 @@ def from_parse_tree(tree: JSP.ProgramContext) -> nodes.Program:
def to_ascii_tree(
node: Union[nodes.Position, nodes.SourceLocation, nodes.Node],
node: Union[ast.nodes.Position, ast.nodes.SourceLocation, ast.nodes.Node],
name_prefix: str = "",
nesting_lvl: int = 0,
):

View File

@ -8,29 +8,20 @@ The module lacks support of:
* debugger statement
* with statement
* RegExp
* ES2015 features:
* ES6 features:
* generators/yield statement
* for-of statement
* template literals
* ES2017 features:
* async/await.
Basically, the whole standard consists of it, so no ES2017 support.
* ES2018 features:
* for-await-of statement
* template literals
* ES2019 features
* catch binding omission.
The only ES2019 feature.
* and other ES6 features :)
More about ESTree standard:
https://github.com/estree/estree/
Todo:
* Add support for lacking features
* Make another attempt to split up this module
"""
from typing import List, Union, Optional, Literal as TypeLiteral, Any
from typing import List, Union, Optional, Literal as TypeLiteral, TypedDict, Any
from enum import Enum
from collections import OrderedDict
@ -39,11 +30,8 @@ from collections import OrderedDict
# Custom types used in the nodes
number = float
"""A type representing Number type in JavaScript."""
bigint = int
"""A type representing BigInt type in JavaScript."""
number = Union[int, float]
"""A type union consisting of int and float Python types. Consider it as Number type from JavaScript."""
SourceTypeLiteral = TypeLiteral["script", "module"]
"""The type for the `sourceType` field."""
@ -54,9 +42,6 @@ VarDeclKind = TypeLiteral["var", "let", "const"]
PropKind = TypeLiteral["init", "get", "set"]
"""A type for a `kind` field of `Property`."""
MethodDefinitionKind = TypeLiteral["constructor", "method", "get", "set"]
"""A type for a `kind` field of `MethodDefinition`."""
class UnaryOperator(Enum):
"""A unary operator token."""
@ -101,7 +86,6 @@ class BinaryOperator(Enum):
AND = "&"
IN = "in"
INSTANCEOF = "instanceof"
POW = "**"
class AssignmentOperator(Enum):
@ -119,7 +103,6 @@ class AssignmentOperator(Enum):
OR = "|="
XOR = "^="
AND = "&="
POW = "**="
class LogicalOperator(Enum):
@ -127,7 +110,6 @@ class LogicalOperator(Enum):
OR = "||"
AND = "&&"
NULLISH_COALESCING = "??"
# Nodes forward declarations
@ -163,10 +145,6 @@ class Identifier:
...
class Literal:
...
# "Node objects" block
@ -240,6 +218,20 @@ class Node:
return self._fields
# "Literal" block
class Literal(Expression):
"""A literal token. Note that a literal can be an expression."""
def __init__(
self, loc: Optional[SourceLocation], value: Union[str, bool, number, None]
):
super().__init__("Literal", loc)
self.value = value
self._fields.update({"value": self.value})
# "Programs" block
@ -567,11 +559,7 @@ class ArrayExpression(Expression):
class ObjectExpression(Expression):
"""An object expression."""
def __init__(
self,
loc: Optional[SourceLocation],
properties: List[Union[Property, SpreadElement]],
):
def __init__(self, loc: Optional[SourceLocation], properties: List[Property]):
super().__init__("ObjectExpression", loc)
self.properties = properties
self._fields.update({"properties": self.properties})
@ -966,9 +954,6 @@ InExpression = _generate_binary_expression(BinaryOperator.IN, """An "in" express
InstanceofExpression = _generate_binary_expression(
BinaryOperator.INSTANCEOF, """An "instanceof" expression."""
)
PowBinaryExpression = _generate_binary_expression(
BinaryOperator.POW, """A power expression, e.g. ``2**3``."""
)
SimpleAssignExpression = _generate_assignment_expression(
AssignmentOperator.ASSIGN, """An assignment done with operator ``=`` expression."""
)
@ -1012,51 +997,12 @@ AndAssignExpression = _generate_assignment_expression(
AssignmentOperator.AND,
"""A "bit and" assignment done with operator ``&=`` expression.""",
)
PowAssignExpression = _generate_assignment_expression(
AssignmentOperator.POW, """A power assignment expression, e.g. ``x**=2``."""
)
OrLogicExpression = _generate_logical_expression(
LogicalOperator.OR, """An "or" logical expression."""
)
AndLogicExpression = _generate_logical_expression(
LogicalOperator.AND, """An "and" logical expression."""
)
NullishCoalescingLogicExpression = _generate_logical_expression(
LogicalOperator.NULLISH_COALESCING, """A nullish coalescing logical expression."""
)
# "Literal" block
class Literal(Expression):
"""A literal token. Note that a literal can be an expression."""
def __init__(
self,
loc: Optional[SourceLocation],
value: Union[str, bool, number, bigint, None],
):
super().__init__("Literal", loc)
self.value = value
self._fields.update({"value": self.value})
class BigIntLiteral(Literal):
"""`bigint` property is the string representation of the ``BigInt`` value. It doesn't include the suffix ``n``.
In environments that don't support ``BigInt`` values, value property will be `None` as the ``BigInt`` value can't
be represented natively.
"""
def __init__(
self,
loc: Optional[SourceLocation],
value: Union[str, bool, number, bigint, None],
bigint: str,
):
super().__init__(loc, value)
self.bigint = bigint
self._fields.update({"bigint": self.bigint})
# "Property" block
@ -1070,7 +1016,7 @@ class Property(Node):
def __init__(
self,
loc: Optional[SourceLocation],
key: Expression,
key: Union[Literal, Identifier],
value: Expression,
kind: PropKind,
method: bool,
@ -1100,7 +1046,7 @@ class AssignmentProperty(Property):
def __init__(
self,
loc: Optional[SourceLocation],
key: Expression,
key: Union[Literal, Identifier],
value: Pattern,
shorthand: bool,
computed: bool,
@ -1121,18 +1067,14 @@ class Pattern(Node):
super().__init__(node_type, loc)
class RestElement(Pattern):
def __init__(self, loc: Optional[SourceLocation], argument: Pattern):
super().__init__("RestElement", loc)
self.argument = argument
self._fields.update({"argument": self.argument})
class ObjectPatternKeyValue(TypedDict):
key: Union[Literal, Identifier]
value: Pattern
class ObjectPattern(Pattern):
def __init__(
self,
loc: Optional[SourceLocation],
properties: List[Union[AssignmentProperty, RestElement]],
self, loc: Optional[SourceLocation], properties: List[ObjectPatternKeyValue]
):
super().__init__("ObjectPattern", loc)
self.properties = properties
@ -1148,14 +1090,6 @@ class ArrayPattern(Pattern):
self._fields.update({"elements": self.elements})
class AssignmentPattern(Pattern):
def __init__(self, loc: Optional[SourceLocation], left: Pattern, right: Expression):
super().__init__("AssignmentPattern", loc)
self.left = left
self.right = right
self._fields.update({"left": self.left, "right": self.right})
# "Identifier" block
@ -1165,281 +1099,3 @@ class Identifier(Expression, Pattern):
def __init__(self, loc: Optional[SourceLocation], name: str):
super().__init__("Identifier", loc)
self.name = name
self._fields.update({"name": self.name})
# "Classes" block
class MethodDefinition(Node):
def __init__(
self,
loc: Optional[SourceLocation],
key: Expression,
value: FunctionExpression,
kind: MethodDefinitionKind,
computed: bool,
static: bool,
):
super().__init__("MethodDefinition", loc)
self.key = key
self.value = value
self.kind = kind
self.computed = computed
self.static = static
self._fields.update(
{
"key": self.key,
"value": self.value,
"kind": self.kind,
"computed": self.computed,
"static": self.static,
}
)
class ClassBody(Node):
def __init__(self, loc: Optional[SourceLocation], body: List[MethodDefinition]):
super().__init__("ClassBody", loc)
self.body = body
self._fields.update({"body": self.body})
class Class(Node):
def __init__(
self,
node_type: str,
loc: Optional[SourceLocation],
class_id: Optional[Identifier],
super_class: Optional[Expression],
body: ClassBody,
):
super().__init__(node_type, loc)
self.id = class_id
self.super_class = super_class
self.body = body
self._fields.update(
{"id": self.id, "superClass": self.super_class, "body": self.body}
)
class ClassDeclaration(Class, Declaration):
def __init__(
self,
loc: Optional[SourceLocation],
class_id: Identifier,
super_class: Optional[Expression],
body: ClassBody,
):
super().__init__("ClassDeclaration", loc, class_id, super_class, body)
class ClassExpression(Class, Expression):
def __init__(
self,
loc: Optional[SourceLocation],
class_id: Optional[Identifier],
super_class: Optional[Expression],
body: ClassBody,
):
super().__init__("ClassExpression", loc, class_id, super_class, body)
class MetaProperty(Expression):
"""`MetaProperty` node represents ``new.target`` meta property in ES2015.
In the future, it will represent other meta properties as well.
"""
def __init__(
self, loc: Optional[SourceLocation], meta: Identifier, meta_property: Identifier
):
super().__init__("MetaProperty", loc)
self.meta = (meta,)
self.property = meta_property
self._fields.update({"meta": self.meta, "property": self.property})
# "Modules" block
class ModuleDeclaration(Node):
"""A module ``import`` or ``export`` declaration."""
def __init__(self, node_type: str, loc: Optional[SourceLocation]):
super().__init__(node_type, loc)
class ModuleSpecifier(Node):
"""A specifier in an import or export declaration."""
def __init__(
self, node_type: str, loc: Optional[SourceLocation], local: Identifier
):
super().__init__(node_type, loc)
self.local = local
self._fields.update({"local": self.local})
class ImportSpecifier(ModuleSpecifier):
"""An imported variable binding, e.g., ``{foo}`` in ``import {foo} from "mod"``
or ``{foo as bar}`` in ``import {foo as bar} from "mod"``. The `imported` field
refers to the name of the export imported from the module. The `local` field
refers to the binding imported into the local module scope. If it is a basic named
import, such as in ``import {foo} from "mod"``, both `imported` and `local` are
equivalent `Identifier` nodes; in this case an `Identifier` node representing ``foo``.
If it is an aliased import, such as in ``import {foo as bar} from "mod"``, the
`imported` field is an `Identifier` node representing ``foo``, and the `local` field
is an `Identifier` node representing ``bar``.
"""
def __init__(
self, loc: Optional[SourceLocation], local: Identifier, imported: Identifier
):
super().__init__("ImportSpecifier", loc, local)
self.imported = imported
self._fields.update({"imported": self.imported})
class ImportDefaultSpecifier(ModuleSpecifier):
"""A default import specifier, e.g., ``foo`` in ``import foo from "mod.js"``."""
def __init__(self, loc: Optional[SourceLocation], local: Identifier):
super().__init__("ImportDefaultSpecifier", loc, local)
class ImportNamespaceSpecifier(ModuleSpecifier):
"""A namespace import specifier, e.g., ``* as foo`` in ``import * as foo from "mod.js"``."""
def __init__(self, loc: Optional[SourceLocation], local: Identifier):
super().__init__("ImportNamespaceSpecifier", loc, local)
class ImportDeclaration(ModuleDeclaration):
"""An import declaration, e.g., ``import foo from "mod";``."""
def __init__(
self,
loc: Optional[SourceLocation],
specifiers: List[
Union[ImportSpecifier, ImportDefaultSpecifier, ImportNamespaceSpecifier]
],
source: Literal,
):
super().__init__("ImportDeclaration", loc)
self.specifiers = specifiers
self.source = source
self._fields.update({"specifiers": self.specifiers, "source": self.source})
class ImportExpression(Expression):
"""`ImportExpression` node represents Dynamic Imports such as ``import(source)``.
The `source` property is the importing source as similar to ImportDeclaration node,
but it can be an arbitrary expression node.
"""
def __init__(self, loc: Optional[SourceLocation], source: Expression):
super().__init__("ImportExpression", loc)
self.source = source
self._fields.update({"source": self.source})
class ExportSpecifier(ModuleSpecifier):
"""An exported variable binding, e.g., ``{foo}`` in ``export {foo}`` or ``{bar as foo}``
in ``export {bar as foo}``. The `exported` field refers to the name exported in the module.
The `local` field refers to the binding into the local module scope. If it is a basic named
export, such as in ``export {foo}``, both `exported` and `local` are equivalent `Identifier`
nodes; in this case an `Identifier` node representing ``foo``. If it is an aliased export,
such as in ``export {bar as foo}``, the `exported` field is an `Identifier` node representing
``foo``, and the `local` field is an `Identifier` node representing ``bar``.
"""
def __init__(
self, loc: Optional[SourceLocation], local: Identifier, exported: Identifier
):
super().__init__("ExportSpecifier", loc, local)
self.exported = exported
self._fields.update({"exported": self.exported})
class ExportNamedDeclaration(ModuleDeclaration):
"""An export named declaration, e.g., ``export {foo, bar};``, ``export {foo} from "mod";``
or ``export var foo = 1;``.
Notes:
Having `declaration` populated with non-empty `specifiers` or non-null `source` results
in an invalid state.
"""
def __init__(
self,
loc: Optional[SourceLocation],
declaration: Optional[Declaration],
specifiers: List[ExportSpecifier],
source: Optional[Literal],
):
super().__init__("ExportNamedDeclaration", loc)
self.declaration = declaration
self.specifiers = specifiers
self.source = source
self._fields.update(
{
"declaration": self.declaration,
"specifiers": self.specifiers,
"source": self.source,
}
)
class AnonymousDefaultExportedFunctionDeclaration(Function):
def __init__(
self, loc: Optional[SourceLocation], params: List[Pattern], body: FunctionBody
):
super().__init__("FunctionDeclaration", loc, None, params, body)
class AnonymousDefaultExportedClassDeclaration(Class):
def __init__(
self,
loc: Optional[SourceLocation],
super_class: Optional[Expression],
body: ClassBody,
):
super().__init__("ClassDeclaration", loc, None, super_class, body)
class ExportDefaultDeclaration(ModuleDeclaration):
"""An export default declaration, e.g., ``export default function () {};`` or ``export default 1;``."""
def __init__(
self,
loc: Optional[SourceLocation],
declaration: Union[
AnonymousDefaultExportedFunctionDeclaration,
FunctionDeclaration,
AnonymousDefaultExportedClassDeclaration,
ClassDeclaration,
Expression,
],
):
super().__init__("ExportDefaultDeclaration", loc)
self.declaration = declaration
self._fields.update({"declaration": self.declaration})
class ExportAllDeclaration(ModuleDeclaration):
"""An export batch declaration, e.g., ``export * from "mod";``.
The `exported` property contains an `Identifier` when a different exported
name is specified using ``as``, e.g., ``export * as foo from "mod";``.
"""
def __init__(
self,
loc: Optional[SourceLocation],
source: Literal,
exported: Optional[Identifier],
):
super().__init__("ExportAllDeclaration", loc)
self.source = source
self.exported = exported
self._fields.update({"source": self.source, "exported": self.exported})

View File

@ -1,38 +1,32 @@
"""Parse tree listeners.
Basically, you should use ASTListener(source_type: SourceTypeLiteral) in most cases.
Todo:
* Fill `source` field in SourceLocation and pass it to each `_get_source_location()` call.
* Compare `SourceLocation` creation behavior with the one in Acorn/ESPrima
"""
import logging
from typing import Optional, List, Union
import antlr4.ParserRuleContext
from ..lex.JavaScriptParser import JavaScriptParser
from ..lex.JavaScriptParserListener import JavaScriptParserListener as JSBaseListener
from lex.JavaScriptParser import JavaScriptParser
from lex.JavaScriptParserListener import JavaScriptParserListener as JSBaseListener
from . import nodes
import ast.nodes
def _get_source_location(
ctx: antlr4.ParserRuleContext, source: Optional[str]
) -> nodes.SourceLocation:
) -> ast.nodes.SourceLocation:
"""Internal function to obtain `SourceObject` from parser context."""
start_pos = nodes.Position(ctx.start.line, ctx.start.column)
end_pos = nodes.Position(ctx.stop.line, ctx.stop.column)
start_pos = ast.nodes.Position(ctx.start.line, ctx.start.column)
end_pos = ast.nodes.Position(ctx.stop.line, ctx.stop.column)
# If an end is not on a newline, shift end position column by 1
# to match exact token end, not the last character
if end_pos.column != 0:
end_pos.column += 1
return nodes.SourceLocation(source=source, start=start_pos, end=end_pos)
return ast.nodes.SourceLocation(source=source, start=start_pos, end=end_pos)
class AssignableListener(JSBaseListener):
_result: Union[nodes.Identifier, nodes.ObjectPattern, nodes.ArrayPattern]
_result: Union[
ast.nodes.Identifier, ast.nodes.ObjectPattern, ast.nodes.ArrayPattern
]
@property
def result(self):
@ -45,19 +39,19 @@ class AssignableListener(JSBaseListener):
def enterIdentifier(self, ctx: JavaScriptParser.IdentifierContext):
logging.debug("Entered section Identifier")
loc = _get_source_location(ctx, None)
self._result = nodes.Identifier(loc, ctx.getText())
self._result = ast.nodes.Identifier(loc, ctx.getText())
def enterArrayLiteral(self, ctx: JavaScriptParser.ArrayLiteralContext):
logging.debug("Entered section ArrayLiteral")
raise NotImplementedError("ArrayLiteral assignment") # TODO
pass # TODO
def enterObjectLiteral(self, ctx: JavaScriptParser.ObjectLiteralContext):
logging.debug("Entered section ObjectLiteral")
raise NotImplementedError("ObjectLiteral assignment") # TODO
pass # TODO
class VariableDeclarationListener(JSBaseListener):
_var_decl: nodes.VariableDeclarator
_var_decl: ast.nodes.VariableDeclarator
@property
def var_declarator(self):
@ -69,21 +63,17 @@ class VariableDeclarationListener(JSBaseListener):
loc = _get_source_location(ctx, None)
assign_listener = AssignableListener()
ctx.assignable().enterRule(assign_listener)
if ctx.singleExpression() is not None:
raise NotImplementedError("VariableDeclarator initialization")
# ctx.singleExpression().enterRule(expression_listener) # FIXME No ExpressionListener yet
init = None # value from ExpressionListener
self._var_decl = nodes.VariableDeclarator(loc, assign_listener.result, init)
self._var_decl = ast.nodes.VariableDeclarator(
loc, assign_listener.result, None
) # FIXME
class StatementListener(JSBaseListener):
_stmt: nodes.Statement
_stmt: ast.nodes.Statement
@property
def statement(self) -> nodes.Statement:
def statement(self) -> ast.nodes.Statement:
"""Statement AST node generated after parse tree walking."""
return self._stmt
@ -97,14 +87,14 @@ class StatementListener(JSBaseListener):
"""Listener for BlockStatement."""
logging.debug("Entered section Block")
stmt_list: List[nodes.Statement] = []
stmt_list: List[ast.nodes.Statement] = []
for stmt in ctx.statementList().children:
stmt_listener = StatementListener()
stmt.enterRule(stmt_listener)
stmt_list.append(stmt_listener.statement)
loc = _get_source_location(ctx, None)
self._stmt = nodes.BlockStatement(loc, stmt_list)
loc = _get_source_location(ctx, None) # FIXME source param is None
self._stmt = ast.nodes.BlockStatement(loc, stmt_list)
def enterVariableStatement(self, ctx: JavaScriptParser.VariableStatementContext):
logging.debug("Entered section VariableStatement")
@ -116,8 +106,8 @@ class StatementListener(JSBaseListener):
"""Listener for VariableDeclaration."""
logging.debug("Entered section VariableDeclaration")
var_modifier: nodes.VarDeclKind = ctx.varModifier().getText()
var_decls: List[nodes.VariableDeclarator] = []
var_modifier: ast.nodes.VarDeclKind = ctx.varModifier().getText()
var_decls: List[ast.nodes.VariableDeclarator] = []
for var_decl in ctx.variableDeclaration():
var_decl_listener = VariableDeclarationListener()
@ -125,13 +115,11 @@ class StatementListener(JSBaseListener):
var_decls.append(var_decl_listener.var_declarator)
loc = _get_source_location(ctx, None)
self._stmt = nodes.VariableDeclaration(loc, var_modifier, var_decls)
self._stmt = ast.nodes.VariableDeclaration(loc, var_modifier, var_decls)
def enterEmptyStatement(self, ctx: JavaScriptParser.EmptyStatementContext):
"""Listener for EmptyStatement."""
logging.debug("Entered section EmptyStatement")
loc = _get_source_location(ctx, None)
self._stmt = nodes.EmptyStatement(loc)
pass
def enterExpressionStatement(
@ -141,12 +129,11 @@ class StatementListener(JSBaseListener):
TODO: check up expression containers.
"""
logging.debug("Entered section ExpressionStatement")
raise NotImplementedError("ExpressionStatement")
pass
def enterIfStatement(self, ctx: JavaScriptParser.IfStatementContext):
"""Listener for IfStatement."""
logging.debug("Entered section IfStatement")
raise NotImplementedError("ExpressionStatement") # FIXME
pass
def enterFunctionDeclaration(
@ -154,7 +141,7 @@ class StatementListener(JSBaseListener):
):
"""Listener for FunctionDeclaration."""
logging.debug("Entered section FunctionDeclaration")
raise NotImplementedError("FunctionDeclaration")
pass
# TODO: import/export, ClassDeclaration, iter statements, continue. break, return
@ -162,10 +149,10 @@ class StatementListener(JSBaseListener):
class SourceElementListener(JSBaseListener):
"""The proxy between Program and Statement."""
_elems: List[nodes.Statement] = []
_elems: List[ast.nodes.Statement] = []
@property
def source_elements(self) -> List[nodes.Statement]:
def source_elements(self) -> List[ast.nodes.Statement]:
"""Source elements AST nodes generated after parse tree walking."""
return self._elems
@ -181,22 +168,22 @@ class SourceElementListener(JSBaseListener):
class ASTListener(JSBaseListener):
"""AST listener."""
_program_node: Optional[nodes.Program] = None
_source_type: nodes.SourceTypeLiteral
_program_node: Optional[ast.nodes.Program] = None
_source_type: ast.nodes.SourceTypeLiteral
@property
def program_node(self) -> nodes.Program:
def program_node(self) -> ast.nodes.Program:
"""The `Program` AST node generated after parse tree walking."""
if self._program_node is None:
raise ValueError("Program AST node is None, did you run the listener?")
return self._program_node
def __init__(self, source_type: nodes.SourceTypeLiteral = "script"):
def __init__(self, source_type: ast.nodes.SourceTypeLiteral = "script"):
"""AST listener constructor.
Args:
source_type (nodes.SourceTypeLiteral): source type. Could be `script` or `module`. Set to
source_type (ast.nodes.SourceTypeLiteral): source type. Could be `script` or `module`. Set to
`script` by default.
"""
self._source_type = source_type
@ -217,6 +204,6 @@ class ASTListener(JSBaseListener):
elem.enterRule(source_elem_listener)
loc = _get_source_location(ctx, None) # FIXME add source name
self._program_node = nodes.Program(
self._program_node = ast.nodes.Program(
loc, self._source_type, source_elem_listener.source_elements
)

View File

@ -1,6 +0,0 @@
*
!.gitignore
!JavaScriptBaseLexer.py
!JavaScriptBaseParser.py
!ErrorListeners.py
!__init__.py

View File

@ -1,3 +0,0 @@
"""Lexer/parser module.
Consists mostly of auto-generated files spewed by ANTLR.
"""