tokenizer.py

from __python__ import hash_literals  # type: ignore

# mypy
from __python__ import RegExp, String, undefined, parseFloat
from typing import Any, Callable, Optional, Union, List, Literal

from unicode_aliases import ALIAS_MAP  # type: ignore
from utils import make_predicate, characters, charAt, startswith
from ast_types import AST_Token
from errors import EOFError, SyntaxError
from string_interpolation import interpolate  # type: ignore

RE_HEX_NUMBER = RegExp(r"^0x[0-9a-f]+$", "i")
RE_OCT_NUMBER = RegExp(r"^0[0-7]+$")
RE_DEC_NUMBER = RegExp(r"^\d*\.?\d*(?:e[+-]?\d*(?:\d\.?|\.?\d)\d*)?$", "i")

OPERATOR_CHARS = make_predicate(characters("+-*&%=<>!?|~^@"))

ASCII_CONTROL_CHARS = {
    'a': 7,
    'b': 8,
    'f': 12,
    'n': 10,
    'r': 13,
    't': 9,
    'v': 11
}
HEX_PAT = RegExp(r"[a-fA-F0-9]")
NAME_PAT = RegExp(r"[a-zA-Z ]")

OPERATORS = make_predicate([
    "in", "instanceof", "typeof", "new", "void", "del", "+", "-", "not", "~",
    "&", "|", "^^", "^", "**", "*", "//", "/", "%", ">>", "<<", ">>>", "<",
    ">", "<=", ">=", "==", "is", "!=", "=", "+=", "-=", "//=", "/=", "*=",
    "%=", ">>=", "<<=", ">>>=", "|=", "^=", "&=", "and", "or", "@", "->"
])

OP_MAP = {
    'or': "||",
    'and': "&&",
    'not': "!",
    'del': "delete",
    'None': "null",
    'is': "===",
}

WHITESPACE_CHARS = make_predicate(
    characters(
        " \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000"
    ))

PUNC_BEFORE_EXPRESSION = make_predicate(characters("[{(,.;:"))

PUNC_CHARS = make_predicate(characters("[]{}(),;:?"))

keywords = "as assert break class continue def del do elif else except finally for from global if import in is lambda new nonlocal pass raise return yield try while with or and not"

keywords_atom = "False None True"

# see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar
reserved_words = (
    "break case class catch const continue debugger default delete do else export extends"
    " finally for function if import in instanceof new return super switch this throw try typeof var void"
    " while with yield enum implements static private package let public protected interface await null true false"
)

keyword_before_expression = "return yield new del raise elif else if"

ALL_KEYWORDS = keywords + " " + keywords_atom

KEYWORDS = make_predicate(keywords)
RESERVED_WORDS = make_predicate(reserved_words)
KEYWORDS_BEFORE_EXPRESSION = make_predicate(keyword_before_expression)
KEYWORDS_ATOM = make_predicate(keywords_atom)
IDENTIFIER_PAT = RegExp(r"^[a-z_$][_a-z0-9$]*$", "i")


# https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
def is_string_modifier(val: str) -> bool:
    for ch in val:
        if ch not in 'vrufVRUF':
            return False
    return True


def is_letter(code: int) -> bool:
    return code >= 97 and code <= 122 or code >= 65 and code <= 90 or code >= 170 and UNICODE[
        'letter'].test(chr(code))


def is_digit(code: int) -> bool:
    return code >= 48 and code <= 57


def is_dot(code: int) -> bool:
    return code == 46


def is_alphanumeric_char(code: int) -> bool:
    return is_digit(code) or is_letter(code)


def is_unicode_combining_mark(ch: str) -> bool:
    return UNICODE['non_spacing_mark'].test(
        ch) or UNICODE['space_combining_mark'].test(ch)


def is_unicode_connector_punctuation(ch: str) -> bool:
    return UNICODE['connector_punctuation'].test(ch)


def is_identifier(name: str) -> bool:
    return not RESERVED_WORDS[name] and not KEYWORDS[
        name] and not KEYWORDS_ATOM[name] and IDENTIFIER_PAT.test(name)


def is_identifier_start(code: int) -> bool:
    return code is 36 or code is 95 or is_letter(code)


def is_identifier_char(ch: str) -> bool:
    code = ord(ch)
    return is_identifier_start(code) or is_digit(
        code) or code is 8204 or code is 8205 or is_unicode_combining_mark(
            ch) or is_unicode_connector_punctuation(ch)


def parse_js_number(num: str) -> Union[float, int]:
    if RE_HEX_NUMBER.test(num):
        return int(num[2:], 16)
    elif RE_OCT_NUMBER.test(num):
        return int(num[1:], 8)
    elif RE_DEC_NUMBER.test(num):
        return float(num)
    raise ValueError("invalid number")


# regexps adapted from http://xregexp.com/plugins/#unicode
UNICODE = {
    'letter':
    RegExp(
        "[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376\\u0377\\u037A-\\u037D\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u0523\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05D0-\\u05EA\\u05F0-\\u05F2\\u0621-\\u064A\\u066E\\u066F\\u0671-\\u06D3\\u06D5\\u06E5\\u06E6\\u06EE\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4\\u07F5\\u07FA\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971\\u0972\\u097B-\\u097F\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC\\u09DD\\u09DF-\\u09E1\\u09F0\\u09F1\\u0A05-\\u0A0A\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A33\\u0A35\\u0A36\\u0A38\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0\\u0AE1\\u0B05-\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C33\\u0C35-\\u0C39\\u0C3D\\u0C58\\u0C59\\u0C60\\u0C61\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDE\\u0CE0\\u0CE1\\u0D05-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D28\\u0D2A-\\u0D39\\u0D3D\\u0D60\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32\\u0E33\\u0E40-\\u0E46\\u0E81\\u0E82\\u0E84\\u0E87\\u0E88\\u0E8A\\u0E8D\\u0E94-\\u0E97\\u0E99-\\u0E9F\\u0EA1-\\u0EA3\\u0EA5\\u0EA7\\u0EAA\\u0EAB\\u0EAD-\\u0EB0\\u0EB2\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC\\u0EDD\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8B\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10D0-\\u10FA\\u10FC\\u1100-\\u1159\\u115F-\\u11A2\\u11A8-\\u11F9\\u1200-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F4\\u1401-\\u166C\\u166F-\\u1676\\u1681-\\u169A\\u16A0-\\u16EA\\u1700-\\u170C\\u170E-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1877\\u1880-\\u18A8\\u18AA\\u1900-\\u191C\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19A9\\u19C1-\\u19C7\\u1A00-\\u1A16\\u1B05-\\u1B33\\u1B45-\\u1B4B\\u1B83-\\u1BA0\\u1BAE\\u1BAF\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u2094\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183\\u2184\\u2C00-\\u2C2E\\u2C30-\\u2C5E\\u2C60-\\u2C6F\\u2C71-\\u2C7D\\u2C80-\\u2CE4\\u2D00-\\u2D25\\u2D30-\\u2D65\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005\\u3006\\u3031-\\u3035\\u303B\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312D\\u3131-\\u318E\\u31A0-\\u31B7\\u31F0-\\u31FF\\u3400\\u4DB5\\u4E00\\u9FC3\\uA000-\\uA48C\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A\\uA62B\\uA640-\\uA65F\\uA662-\\uA66E\\uA67F-\\uA697\\uA717-\\uA71F\\uA722-\\uA788\\uA78B\\uA78C\\uA7FB-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA90A-\\uA925\\uA930-\\uA946\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAC00\\uD7A3\\uF900-\\uFA2D\\uFA30-\\uFA6A\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC]"
    ),
    'non_spacing_mark':
    RegExp(
        "[\\u0300-\\u036F\\u0483-\\u0487\\u0591-\\u05BD\\u05BF\\u05C1\\u05C2\\u05C4\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065E\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0900-\\u0902\\u093C\\u0941-\\u0948\\u094D\\u0951-\\u0955\\u0962\\u0963\\u0981\\u09BC\\u09C1-\\u09C4\\u09CD\\u09E2\\u09E3\\u0A01\\u0A02\\u0A3C\\u0A41\\u0A42\\u0A47\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70\\u0A71\\u0A75\\u0A81\\u0A82\\u0ABC\\u0AC1-\\u0AC5\\u0AC7\\u0AC8\\u0ACD\\u0AE2\\u0AE3\\u0B01\\u0B3C\\u0B3F\\u0B41-\\u0B44\\u0B4D\\u0B56\\u0B62\\u0B63\\u0B82\\u0BC0\\u0BCD\\u0C3E-\\u0C40\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55\\u0C56\\u0C62\\u0C63\\u0CBC\\u0CBF\\u0CC6\\u0CCC\\u0CCD\\u0CE2\\u0CE3\\u0D41-\\u0D44\\u0D4D\\u0D62\\u0D63\\u0DCA\\u0DD2-\\u0DD4\\u0DD6\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EB9\\u0EBB\\u0EBC\\u0EC8-\\u0ECD\\u0F18\\u0F19\\u0F35\\u0F37\\u0F39\\u0F71-\\u0F7E\\u0F80-\\u0F84\\u0F86\\u0F87\\u0F90-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102D-\\u1030\\u1032-\\u1037\\u1039\\u103A\\u103D\\u103E\\u1058\\u1059\\u105E-\\u1060\\u1071-\\u1074\\u1082\\u1085\\u1086\\u108D\\u109D\\u135F\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17B7-\\u17BD\\u17C6\\u17C9-\\u17D3\\u17DD\\u180B-\\u180D\\u18A9\\u1920-\\u1922\\u1927\\u1928\\u1932\\u1939-\\u193B\\u1A17\\u1A18\\u1A56\\u1A58-\\u1A5E\\u1A60\\u1A62\\u1A65-\\u1A6C\\u1A73-\\u1A7C\\u1A7F\\u1B00-\\u1B03\\u1B34\\u1B36-\\u1B3A\\u1B3C\\u1B42\\u1B6B-\\u1B73\\u1B80\\u1B81\\u1BA2-\\u1BA5\\u1BA8\\u1BA9\\u1C2C-\\u1C33\\u1C36\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE0\\u1CE2-\\u1CE8\\u1CED\\u1DC0-\\u1DE6\\u1DFD-\\u1DFF\\u20D0-\\u20DC\\u20E1\\u20E5-\\u20F0\\u2CEF-\\u2CF1\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099\\u309A\\uA66F\\uA67C\\uA67D\\uA6F0\\uA6F1\\uA802\\uA806\\uA80B\\uA825\\uA826\\uA8C4\\uA8E0-\\uA8F1\\uA926-\\uA92D\\uA947-\\uA951\\uA980-\\uA982\\uA9B3\\uA9B6-\\uA9B9\\uA9BC\\uAA29-\\uAA2E\\uAA31\\uAA32\\uAA35\\uAA36\\uAA43\\uAA4C\\uAAB0\\uAAB2-\\uAAB4\\uAAB7\\uAAB8\\uAABE\\uAABF\\uAAC1\\uABE5\\uABE8\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE26]"
    ),
    'space_combining_mark':
    RegExp(
        "[\\u0903\\u093E-\\u0940\\u0949-\\u094C\\u094E\\u0982\\u0983\\u09BE-\\u09C0\\u09C7\\u09C8\\u09CB\\u09CC\\u09D7\\u0A03\\u0A3E-\\u0A40\\u0A83\\u0ABE-\\u0AC0\\u0AC9\\u0ACB\\u0ACC\\u0B02\\u0B03\\u0B3E\\u0B40\\u0B47\\u0B48\\u0B4B\\u0B4C\\u0B57\\u0BBE\\u0BBF\\u0BC1\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCC\\u0BD7\\u0C01-\\u0C03\\u0C41-\\u0C44\\u0C82\\u0C83\\u0CBE\\u0CC0-\\u0CC4\\u0CC7\\u0CC8\\u0CCA\\u0CCB\\u0CD5\\u0CD6\\u0D02\\u0D03\\u0D3E-\\u0D40\\u0D46-\\u0D48\\u0D4A-\\u0D4C\\u0D57\\u0D82\\u0D83\\u0DCF-\\u0DD1\\u0DD8-\\u0DDF\\u0DF2\\u0DF3\\u0F3E\\u0F3F\\u0F7F\\u102B\\u102C\\u1031\\u1038\\u103B\\u103C\\u1056\\u1057\\u1062-\\u1064\\u1067-\\u106D\\u1083\\u1084\\u1087-\\u108C\\u108F\\u109A-\\u109C\\u17B6\\u17BE-\\u17C5\\u17C7\\u17C8\\u1923-\\u1926\\u1929-\\u192B\\u1930\\u1931\\u1933-\\u1938\\u19B0-\\u19C0\\u19C8\\u19C9\\u1A19-\\u1A1B\\u1A55\\u1A57\\u1A61\\u1A63\\u1A64\\u1A6D-\\u1A72\\u1B04\\u1B35\\u1B3B\\u1B3D-\\u1B41\\u1B43\\u1B44\\u1B82\\u1BA1\\u1BA6\\u1BA7\\u1BAA\\u1C24-\\u1C2B\\u1C34\\u1C35\\u1CE1\\u1CF2\\uA823\\uA824\\uA827\\uA880\\uA881\\uA8B4-\\uA8C3\\uA952\\uA953\\uA983\\uA9B4\\uA9B5\\uA9BA\\uA9BB\\uA9BD-\\uA9C0\\uAA2F\\uAA30\\uAA33\\uAA34\\uAA4D\\uAA7B\\uABE3\\uABE4\\uABE6\\uABE7\\uABE9\\uABEA\\uABEC]"
    ),
    'connector_punctuation':
    RegExp(
        "[\\u005F\\u203F\\u2040\\u2054\\uFE33\\uFE34\\uFE4D-\\uFE4F\\uFF3F]")
}  # }}}


def is_token(token, type, val) -> bool:
    return token.type is type and (val is None or val is undefined
                                   or token.value is val)


def tokenizer(raw_text: str, filename: str) -> Callable[[], Any]:
    S = {
        'exponent': # parse ^ as exponent and ^^ as xor
        False,
        'text':
        raw_text.replace(RegExp(r"\r\n?|[\n\u2028\u2029]","g"),
                         "\n").replace(RegExp(r"\uFEFF","g"), ""),
        'filename':
        filename,
        'pos':
        0,
        'tokpos':
        0,
        'line':
        1,
        'tokline':
        0,
        'col':
        0,
        'tokcol':
        0,
        'newline_before':
        False,
        'regex_allowed':
        False,
        'comments_before':
        [],
        'whitespace_before':
        [],
        'newblock':
        False,
        'endblock':
        False,
        'indentation_matters':
        r'%js [ true ]',
        'cached_whitespace':
        "",
        'prev':
        undefined,
        'index_or_slice':
        r'%js [ false ]',
        'expecting_object_literal_key':
        False,  # This is set by the parser when it is expecting an object literal key
    }

    def peek() -> str:
        return charAt(S['text'], S['pos'])

    def peekpeek():
        return charAt(S['text'], S['pos'] + 1)

    def prevChar():
        return charAt(S['text'], S['tokpos'] - 1)

    def next(signal_eof=False, in_string=False):
        ch = charAt(S['text'], S['pos'])
        S['pos'] += 1
        if signal_eof and not ch:
            raise EOFError

        if ch is "\n":
            S['newline_before'] = S['newline_before'] or not in_string
            S['line'] += 1
            S['col'] = 0
        else:
            S['col'] += 1
        return ch

    def find(what: str, signal_eof: bool = False) -> int:
        pos = S['text'].indexOf(what, S['pos'])
        if signal_eof and pos is -1:
            raise EOFError
        return pos

    def start_token() -> None:
        S['tokline'] = S['line']
        S['tokcol'] = S['col']
        S['tokpos'] = S['pos']

    def token(type: str,
              value: Any,
              is_comment: bool = False,
              keep_newline: bool = False) -> AST_Token:
        if S['exponent'] and type == 'operator':
            if value == '^':
                value = '**'
            elif value == '^^':
                value = '^'
        S['regex_allowed'] = (
            type is "operator"
            or type is "keyword" and KEYWORDS_BEFORE_EXPRESSION[value]
            or type is "punc" and PUNC_BEFORE_EXPRESSION[value])

        if type is "operator" and value is "is" and S['text'].substr(
                S['pos']).trimLeft().substr(0, 4).trimRight() is "not":
            next_token()
            value = "!=="

        if type is "operator" and OP_MAP[value]:
            value = OP_MAP[value]

        ret = {
            'type': type,
            'value': value,
            'line': S['tokline'],
            'col': S['tokcol'],
            'pos': S['tokpos'],
            'endpos': S['pos'],
            'nlb': S['newline_before'],
            'file': filename,
            'leading_whitespace': S['whitespace_before'][-1] or '',
        }
        if not is_comment:
            ret['comments_before'] = S['comments_before']
            S['comments_before'] = []
            # make note of any newlines in the comments that came before
            for i in range(ret['comments_before.length']):
                ret['nlb'] = ret['nlb'] or ret['comments_before'][i]['nlb']

        if not keep_newline:
            S['newline_before'] = False

        if type is "punc":
            if (value is ":" and not S['index_or_slice'][-1]
                    and not S['expecting_object_literal_key'] and
                (not S['text'].substring(S['pos'] + 1, find("\n")).trim()
                 or not S['text'].substring(S['pos'] + 1, find("#")).trim())):
                S['newblock'] = True
                S['indentation_matters'].push(True)

            if value is "[":
                if S['prev'] and (S['prev'].type is "name" or
                                  (S['prev'].type is 'punc'
                                   and S['prev'].value in ')]')):
                    S['index_or_slice'].push(True)
                else:
                    S['index_or_slice'].push(False)
                S['indentation_matters'].push(False)
            elif value is "{" or value is "(":
                S['indentation_matters'].push(False)
            elif value is "]":
                S['index_or_slice'].pop()
                S['indentation_matters'].pop()
            elif value is "}" or value is ")":
                S['indentation_matters'].pop()
        S['prev'] = AST_Token(ret)
        return S['prev']

    # this will transform leading whitespace to block tokens unless
    # part of array/hash, and skip non-leading whitespace
    def parse_whitespace() -> Union[Literal[-1], Literal[1], Literal[0]]:
        leading_whitespace = ""
        whitespace_exists = False
        while WHITESPACE_CHARS[peek()]:
            whitespace_exists = True
            ch = next()
            if ch is "\n":
                leading_whitespace = ""
            else:
                leading_whitespace += ch
        if peek() is not "#":
            if not whitespace_exists:
                leading_whitespace = S['cached_whitespace']
            else:
                S['cached_whitespace'] = leading_whitespace
            if S['newline_before'] or S['endblock']:
                return test_indent_token(leading_whitespace)
        return 0

    def test_indent_token(
            leading_whitespace: str
    ) -> Union[Literal[-1], Literal[1], Literal[0]]:
        most_recent = S['whitespace_before'][-1] or ""
        S['endblock'] = False
        if S['indentation_matters'][
                -1] and leading_whitespace is not most_recent:
            if S['newblock'] and leading_whitespace and startswith(
                    leading_whitespace, most_recent):
                # positive indent, new block
                S['newblock'] = False
                S['whitespace_before'].push(leading_whitespace)
                return 1
            elif most_recent and startswith(most_recent, leading_whitespace):
                # negative indent, block is ending
                S['endblock'] = True
                S['whitespace_before'].pop()
                return -1
            else:
                # indent mismatch, inconsistent indentation
                parse_error("Inconsistent indentation")
        return 0

    def read_while(pred: Callable) -> str:
        ret = ""
        i = 0
        ch = peek()
        while ch and pred(ch, i):
            i += 1
            ret += next()
            ch = peek()
        return ret

    def parse_error(err: str, is_eof: bool = False) -> SyntaxError:
        raise SyntaxError(err, filename, S['tokline'], S['tokcol'],
                          S['tokpos'], is_eof)

    def read_num(prefix: str) -> Optional[AST_Token]:
        has_e = False
        has_x = False
        has_dot = prefix is "."

        # Read a binary number
        if not prefix and peek() is '0' and charAt(S['text'],
                                                   S['pos'] + 1) is 'b':
            next(), next()

            def is01(ch):
                return ch is '0' or ch is '1'

            num = read_while(is01)
            try:
                valid = int(num, 2)  # type: Union[float, int]
            except:
                parse_error('Invalid syntax for a binary number')
            return token('num', valid)
        seen = []  # type: List[str]

        def is_num(ch, i):
            nonlocal has_dot, has_e, has_x
            seen.push(ch)
            if ch is 'x' or ch is 'X':
                if has_x or seen.length is not 2 or seen[0] is not '0':
                    return False
                has_x = True
                return True
            elif ch is 'e' or ch is 'E':
                if has_x:
                    return True
                if has_e or i == 0:
                    return False
                has_e = True
                return True
            elif ch is '-':
                if i is 0 and not prefix:
                    return True
                if has_e and seen[i - 1].toLowerCase() is 'e':
                    return True
                return False
            elif ch is '+':
                if has_e and seen[i - 1].toLowerCase() is 'e':
                    return True
                return False
            elif ch is '.':
                # If next ch after this is also a ., then its
                # something like [389..5077], so we stop
                if peekpeek() is '.':
                    return False
                if not has_dot and not has_x and not has_e:
                    has_dot = True
                    return True
                return False
            return is_alphanumeric_char(ch.charCodeAt(0))

        num = read_while(is_num)
        if prefix:
            num = prefix + num

        try:
            valid = parse_js_number(num)
        except:
            parse_error("SyntaxError: invalid syntax in numeric literal -- " +
                        num)
            return undefined
        return token("num", valid)

    # This returns str or int, since it could be a
    # hex number or a hex character code.
    def read_hex_digits(count: int) -> Union[str, int]:
        ans = ''
        while count > 0:
            count -= 1
            if not HEX_PAT.test(peek()):
                return ans
            ans += next()
        nval = int(ans, 16)
        if nval > 0x10FFFF:
            return ans
        return nval

    def read_escape_sequence():
        q = next(True)
        if q is '\n':
            return ''
        if q is '\\':
            return q
        if '"\''.indexOf(q) is not -1:
            return q
        if ASCII_CONTROL_CHARS[q]:
            return String.fromCharCode(ASCII_CONTROL_CHARS[q])
        if '0' <= q <= '7':
            octal = q
            if '0' <= peek() <= '7':
                octal += next()
            if '0' <= peek() <= '7':
                octal += next()
            code = parseInt(octal, 8)
            if isNaN(code):
                return '\\' + octal
            return String.fromCharCode(code)
        if q is 'x':
            code = read_hex_digits(2)
            if jstype(code) is 'number':
                return String.fromCharCode(code)
            return '\\x' + code
        if q is 'u':
            code = read_hex_digits(4)
            if jstype(code) is 'number':
                return String.fromCharCode(code)
            return '\\u' + code
        if q is 'U':
            code = read_hex_digits(8)
            if jstype(code) is 'number':
                if code <= 0xFFFF:
                    return String.fromCharCode(code)
                code -= 0x10000
                return String.fromCharCode(0xD800 + (code >> 10),
                                           0xDC00 + (code & 0x3FF))
            return '\\U' + code
        if q is 'N' and peek() is '{':
            next()

            def is_name_ch(ch):
                return NAME_PAT.test(ch)

            name = read_while(is_name_ch)
            if peek() is not '}':
                return '\\N{' + name
            next()
            key = (name or '').toLowerCase()
            if not name or not Object.prototype.hasOwnProperty.call(
                    ALIAS_MAP, key):
                return '\\N{' + name + '}'
            code = ALIAS_MAP[key]
            if code <= 0xFFFF:
                return String.fromCharCode(code)
            code -= 0x10000
            return String.fromCharCode(0xD800 + (code >> 10),
                                       0xDC00 + (code & 0x3FF))
        return '\\' + q

    def with_eof_error(eof_error, cont):
        def eof_error():
            try:
                return cont.apply(None, arguments)
            except Exception as ex:
                if ex is EOFError:
                    parse_error(eof_error, True)
                else:
                    raise

        return eof_error

    def _read_string(is_raw_literal, is_js_literal):
        quote = next()
        tok_type = 'js' if is_js_literal else 'string'
        ret = ""
        is_multiline = False
        if peek() is quote:
            # two quotes in a row
            next(True)
            if peek() is quote:
                # multiline string (3 quotes in a row)
                next(True)
                is_multiline = True
            else:
                return token(tok_type, '')

        while True:
            ch = next(True, True)
            if not ch:
                break
            if ch is "\n" and not is_multiline:
                parse_error("End of line while scanning string literal")

            if ch is "\\":
                ret += (
                    '\\' +
                    next(True)) if is_raw_literal else read_escape_sequence()
                continue

            if ch is quote:
                if not is_multiline:
                    break
                if peek() is quote:
                    next()
                    if peek() is quote:
                        next()
                        break
                    else:
                        ch += quote
            ret += ch
        if is_raw_literal and ret[:3] == '%js' and WHITESPACE_CHARS[ret[3]]:
            return token(
                'js', ret[4:].trim())  # trim since really javascript string.
        return token(tok_type, ret)

    read_string = with_eof_error("Unterminated string constant", _read_string)

    def handle_interpolated_string(string, start_tok):
        def raise_error(err):
            raise SyntaxError(err, filename, start_tok.line, start_tok.col,
                              start_tok.pos, False)

        S['text'] = S['text'][:S['pos']] + '(' + interpolate(
            string, raise_error) + ')' + S['text'][S['pos']:]
        return token('punc', next())

    def read_line_comment(shebang):
        if not shebang:
            next()
        i = find("\n")

        if i is -1:
            ret = S['text'].substr(S['pos'])
            S['pos'] = S['text'].length
        else:
            ret = S['text'].substring(S['pos'], i)
            S['pos'] = i

        return token("shebang" if shebang else "comment1", ret, True)

    def read_name():
        name = ch = ""
        while True:
            ch = peek()
            if ch is None: break
            if ch is "\\":
                if charAt(S['text'], S['pos'] + 1) is "\n":
                    S['pos'] += 2
                    continue
                break
            elif is_identifier_char(ch):
                name += next()
            else:
                break
        return name

    def do_read_regexp():
        prev_backslash = False
        regexp = ch = ''
        in_class = False
        verbose_regexp = False
        in_comment = False

        if peek() is '/':
            next(True)
            if peek() is '/':
                verbose_regexp = True
                next(True)
            else:  # empty regexp (//)
                mods = read_name()
                return token("regexp", RegExp(regexp, mods))
        while True:
            ch = next(True)
            if not ch:
                break
            if in_comment:
                if ch is '\n':
                    in_comment = False
                continue
            if prev_backslash:
                regexp += "\\" + ch
                prev_backslash = False
            elif ch is "[":
                in_class = True
                regexp += ch
            elif ch is "]" and in_class:
                in_class = False
                regexp += ch
            elif ch is "/" and not in_class:
                if verbose_regexp:
                    if peek() is not '/':
                        regexp += '\\/'
                        continue
                    next(True)
                    if peek() is not '/':
                        regexp += '\\/\\/'
                        continue
                    next(True)
                break
            elif ch is "\\":
                prev_backslash = True
            elif verbose_regexp and not in_class and ' \n\r\t'.indexOf(
                    ch) is not -1:
                pass
            elif verbose_regexp and not in_class and ch is '#':
                in_comment = True
            else:
                regexp += ch

        mods = read_name()
        return token("regexp", RegExp(regexp, mods))

    read_regexp = with_eof_error("Unterminated regular expression",
                                 do_read_regexp)

    def read_operator(prefix):
        def grow(op):
            if not peek():
                return op

            bigger = op + peek()
            if OPERATORS[bigger]:
                next()
                return grow(bigger)
            else:
                return op

        op = grow(prefix or next())
        if op is '->':
            # pretend that this is an operator as the tokenizer only allows
            # one character punctuation.
            return token('punc', op)
        return token("operator", op)

    def handle_slash():
        next()
        return read_regexp("") if S['regex_allowed'] else read_operator("/")

    def handle_dot():
        next()
        c = peek().charCodeAt(0)

        if is_digit(c):
            return read_num(".")

        if is_dot(c):
            # ellipses: Two dots in a row, e.g., [a..b]
            next()
            return token("punc", "..")

        return token("punc", ".")

    def read_word():
        word = read_name()
        return token("atom", word) if KEYWORDS_ATOM[word] else (
            token("name", word) if not KEYWORDS[word] else
            (token("operator", word) if OPERATORS[word]
             and prevChar() is not "." else token("keyword", word)))

    def next_token():

        indent = parse_whitespace()
        # if indent is 1:
        #     return token("punc", "{")
        if indent is -1:
            return token("punc", "}", False, True)

        start_token()
        ch = peek()
        if not ch:
            return token("eof")

        code = ch.charCodeAt(0)
        tmp_ = code
        if tmp_ is 34 or tmp_ is 39:  # double-quote (") or single quote (')
            return read_string(False)
        elif tmp_ is 35:  # pound-sign (#)
            if S['pos'] is 0 and charAt(S['text'], 1) is '!':
                #shebang
                return read_line_comment(True)
            regex_allowed = S['regex_allowed']
            S['comments_before'].push(read_line_comment())
            S['regex_allowed'] = regex_allowed
            return next_token()
        elif tmp_ is 46:  # dot (.)
            return handle_dot()
        elif tmp_ is 47:  # slash (/)
            return handle_slash()

        if is_digit(code):
            return read_num()

        if PUNC_CHARS[ch]:
            return token("punc", next())

        if OPERATOR_CHARS[ch]:
            return read_operator()

        if code is 92 and charAt(S['text'], S['pos'] + 1) is "\n":
            # backslash(=92) will consume the newline character that follows
            next()
            # backslash
            next()
            # newline
            S['newline_before'] = False
            return next_token()

        if is_identifier_start(code):
            tok = read_word()
            if '\'"'.includes(peek()) and is_string_modifier(tok.value):
                mods = tok.value.toLowerCase()
                start_pos_for_string = S['tokpos']
                stok = read_string(
                    mods.indexOf('r') is not -1,
                    mods.indexOf('v') is not -1)
                tok.endpos = stok.endpos
                if stok.type is not 'js' and mods.indexOf('f') is not -1:
                    tok.col += start_pos_for_string - tok.pos
                    return handle_interpolated_string(stok.value, tok)
                tok.value = stok.value
                tok.type = stok.type
            return tok

        parse_error("Unexpected character '" + ch + "'")

    def context(nc):
        nonlocal S
        if nc:
            S = nc
        return S

    next_token.context = context  # type: ignore

    return next_token