
    0hh                       S SK r S SKrS SKrS SKrS SKrS SKJrJr  S SKJ	r	  S SKJ
r
Jr  S SKJr  S SKJr  S SKJrJrJr  S SKJrJr  S	 r " S
 S\
5      r " S S\5      rS r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r " S S\
5      r S r! " S S\
5      r" " S S\
5      r# " S  S!\
5      r$ " S" S#\R                  5      r%\&S$:X  a  \RN                  " 5         gg)%    N)BytesIOStringIO)dedent)TestCasemock)support)	os_helper)run_test_scriptmake_scriptrun_python_until_end)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALSc                 F   / n[        UR                  5       5      nUS   S;  nU  Hw  u  pVpxn	U[        R                  :X  a    U$ U(       a  U[        R                  :X  a  US   U:X  a  MD  [        R
                  U   nUR                  SUS SU<S SU SU 35        My     U$ )Nz
r       10 13)len
splitlinestokenize	ENDMARKERNEWLINEtok_nameappend)
token_generatorsource_stringresult	num_linesmissing_trailing_nltypetokenstartendlines
             ;/opt/python-3.13.8/usr/lib/python3.13/test/test_tokenize.pystringify_tokens_from_sourcer'      s    FM,,./I'+69)8%U8%%% M 48+;+;#;A)@S  &T"IQulrl!E7!C5AB *9 M    c                       \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rSrg)TokenizeTest&   c                     [        UR                  S5      5      n[        [        R                  " UR                  5      U5      nU R                  US/UR                  5       R                  5       -   5        g )Nutf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r   encoder'   r   readlineassertEqualrstripr   selfsexpectedfr   s        r&   check_tokenizeTokenizeTest.check_tokenize-   s`     AHHW%&-h.?.?

.KQOFG!*5578	9r(   c                 D   S nU R                  [        5         [        [        R                  " U" 5       R                  5      5        S S S 5        S nU R                  [        5         [        [        R
                  " U" 5       R                  5      5        S S S 5        S nU R                  [        5         [        [        R
                  " U" 5       R                  5      5        S S S 5        g ! , (       d  f       N= f! , (       d  f       No= f! , (       d  f       g = f)Nc               3      #    Sv   Sv   g 7f)Nsdfosdg r<   r(   r&   gen/TokenizeTest.test_invalid_readline.<locals>.gen7   s     OO   
c               3      #    Sv   Sv   g 7f)Ns   sdfosdgr<   r<   r(   r&   r=   r>   =   s     r?   c               3      #    Sv   SS-    g 7f)Nr;      r   r<   r<   r(   r&   r=   r>   C   s     OaCs   )assertRaises	TypeErrorlistr   __next__generate_tokensZeroDivisionError)r3   r=   s     r&   test_invalid_readline"TokenizeTest.test_invalid_readline6   s    	 y)""35>>23 *	 y)))#%..9: *	 01))#%..9: 21 *) *) 21s#   /C/(/D 7/D/
C= 
D
Dc                 >   [        SR                  S5      5      n[        [        R                  " UR                  5      5      nU R                  US   R                  [        R                  5        U R                  US   R                  [        R                  5        g )Nxr-   r   )	r   r.   rE   r   r/   r0   r!   r   r   )r3   r6   tokenss      r&   test_implicit_newline"TokenizeTest.test_implicit_newlineI   sl     CJJw'(h''

34(*:*:;(*<*<=r(   c                 r   U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        S	n[        U5      R                  nU R                  [        S
5       n[
        R
                  " U5       H  nM     S S S 5        U R                  WR                  R                  S5        U R                  UR                  R                  S5        U R                  UR                  R                  S 5        U R                  UR                  R                  S 5        U R                  UR                  R                  S
5        U R                  UR                  R                  S5        U R                  UR                  R                  S5        g ! , (       d  f       GN= f)Nz1 + 1z    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af      NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    z%if True:
    # NL
    foo='bar'

a@      NAME       'if'          (1, 0) (1, 2)
    NAME       'True'        (1, 3) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\r\n'        (1, 8) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\r\n'        (2, 8) (2, 10)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'foo'         (3, 4) (3, 7)
    OP         '='           (3, 7) (3, 8)
    STRING     "'bar'"       (3, 8) (3, 13)
    NEWLINE    '\r\n'        (3, 13) (3, 15)
    NL         '\r\n'        (4, 0) (4, 2)
    DEDENT     ''            (5, 0) (5, 0)
            zx = 1 + \
1
a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (2, 0) (2, 1)
    NEWLINE    '\r\n'        (2, 1) (2, 3)
            s   def k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level   <string>	   z  x += 5)r7   r   r/   assertRaisesRegexIndentationErrorr   r0   	exceptionlinenofilename
end_lineno
end_offsetmsgoffsettext)r3   indent_error_filer/   etoks        r&   
test_basicTokenizeTest.test_basicQ   s{   G & 		
 	 ;=		( 	K N 	  	1 4 	
 ,-66##$4%>?BC((2 3?
 	++Q/--z://6//6KKOOA	C 	++Q/)):6? ?s   6F''
F6c                 H   U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        g )N0xff <= 255z    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0b10 <= 255z    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0o123 <= 0O123z    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    1234567 > ~0x15z    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    2134568 != 1231515z    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    (-124561-1) & 200000000ac      OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    0xdeadbeef != -1z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    0xdeadc0de & 12345z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    0xFF & 0x15 | 1234z    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    r7   r3   s    r&   test_intTokenizeTest.test_int   s    M , 		
 	M , 		
 	, / 		
 	- 0 		 	0 3 		
 	5 	8 			 	. 1 		 	0 3 		
 	0 3 		r(   c                     U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        g )	Nzx = 0z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    rn   ro   s    r&   	test_longTokenizeTest.test_long   s_    G & 		
 	/ 2 		
 	7 : 		
 	4 7 		r(   c                     U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        g )Nx = 3.14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    x = 314159.z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    x = .314159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    x = 3e14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    	x = 3E123z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    x+y = 3e-1230z    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    x = 3.14e159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rn   ro   s    r&   
test_floatTokenizeTest.test_float   s    M , 		
 	M , 		
 	M , 		
 	M , 		
 	K * 		
 	O . 		 	N - 		r(   c                    S n[          H#  nSU;   a  M  U R                  U" U5      U5        M%     SS1n[         H+  nX#;   a  M
   U" U5        U R                  U" U5      U5        M-     g ! [        R                   a     MG  f = f)Nc                     [        U R                  S5      5      n[        R                  " UR                  5       H  u  p#pEnU[        R                  :X  d  M  Us  $    g)Nr-   zinvalid token)r   r.   r   r/   NUMBER)r4   r6   toktyper"   r#   r$   r%   s          r&   number_token;TokenizeTest.test_underscore_literals.<locals>.number_token  sL    )*A4<4E4Eajj4Q0Dhoo- L 5R #r(   (0_709_99)r   r0   r   r   
TokenErrorassertNotEqual)r3   r   litextra_valid_casess       r&   test_underscore_literals%TokenizeTest.test_underscore_literals  s    	# -Ccz\#.4	 - #G,.C'S! S 137 /
 && s   A++BBc                    U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS 5        U R                  S!S"5        U R                  S#S$5        U R                  S%S&5        U R                  S'S(5        U R                  S)S*5        U R                  S+S,5        U R                  S-S.5        U R                  S/S05        U R                  S1S25        U R                  S3S45        U R                  S5S65        U R                  S7S85        U R                  S9S:5        U R                  S;S<5        U R                  S=S>5        U R                  S?S@5        U R                  SASB5        U R                  SCSD5        U R                  SESF5        U R                  SGSH5        g )INx = ''; y = ""a5      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    x = '"'; y = "'"a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    x = "doesn't "shrink", does it"z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    x = 'abc' + 'ABC'z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    y = "ABC" + "ABC"z    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    %x = r'abc' + r'ABC' + R'ABC' + R'ABC'a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    %y = r"abc" + r"ABC" + R"ABC" + R"ABC"a      NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    u'abc' + U'abc'z    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    u"abc" + U"abc"z    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    b'abc' + B'abc'z    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    b"abc" + B"abc"z    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    %br'abc' + bR'abc' + Br'abc' + BR'abc'a;      STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    %br"abc" + bR"abc" + Br"abc" + BR"abc"a;      STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    %rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    %rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    "a\
de\
fg"3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    u"a\
de"/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    rb"a\
d"/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    
"""a\
b"""0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    u"""a\
b"""1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    rb"""a\
b\
c"""7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    f"abc"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc'         (1, 2) (1, 5)
    FSTRING_END '"'           (1, 5) (1, 6)
    	fR"a{b}c"a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    OP         '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    zfR"a{{{b!r}}}c"a      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a{'          (1, 3) (1, 5)
    OP         '{'           (1, 6) (1, 7)
    NAME       'b'           (1, 7) (1, 8)
    OP         '!'           (1, 8) (1, 9)
    NAME       'r'           (1, 9) (1, 10)
    OP         '}'           (1, 10) (1, 11)
    FSTRING_MIDDLE '}'           (1, 11) (1, 12)
    FSTRING_MIDDLE 'c'           (1, 13) (1, 14)
    FSTRING_END '"'           (1, 14) (1, 15)
    zf"{{{1+1}}}"a      FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE '{'           (1, 2) (1, 3)
    OP         '{'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '}'           (1, 8) (1, 9)
    FSTRING_MIDDLE '}'           (1, 9) (1, 10)
    FSTRING_END '"'           (1, 11) (1, 12)
    zf"""{f'''{f'{f"{1+1}"}'}'''}"""ad      FSTRING_START 'f"""'        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    FSTRING_START "f'''"        (1, 5) (1, 9)
    OP         '{'           (1, 9) (1, 10)
    FSTRING_START "f'"          (1, 10) (1, 12)
    OP         '{'           (1, 12) (1, 13)
    FSTRING_START 'f"'          (1, 13) (1, 15)
    OP         '{'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '+'           (1, 17) (1, 18)
    NUMBER     '1'           (1, 18) (1, 19)
    OP         '}'           (1, 19) (1, 20)
    FSTRING_END '"'           (1, 20) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_END "'"           (1, 22) (1, 23)
    OP         '}'           (1, 23) (1, 24)
    FSTRING_END "'''"         (1, 24) (1, 27)
    OP         '}'           (1, 27) (1, 28)
    FSTRING_END '"""'         (1, 28) (1, 31)
    z.f"""     x
str(data, encoding={invalid!r})
"""a      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '     x\nstr(data, encoding=' (1, 4) (2, 19)
    OP         '{'           (2, 19) (2, 20)
    NAME       'invalid'     (2, 20) (2, 27)
    OP         '!'           (2, 27) (2, 28)
    NAME       'r'           (2, 28) (2, 29)
    OP         '}'           (2, 29) (2, 30)
    FSTRING_MIDDLE ')\n'         (2, 30) (3, 0)
    FSTRING_END '"""'         (3, 0) (3, 3)
    z#f"""123456789
something{None}bad"""aO      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
    OP         '{'           (2, 9) (2, 10)
    NAME       'None'        (2, 10) (2, 14)
    OP         '}'           (2, 14) (2, 15)
    FSTRING_MIDDLE 'bad'         (2, 15) (2, 18)
    FSTRING_END '"""'         (2, 18) (2, 21)
    
f"""abc"""    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE 'abc'         (1, 4) (1, 7)
    FSTRING_END '"""'         (1, 7) (1, 10)
    f"abc\
def"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 2) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    Rf"abc\
def"    FSTRING_START 'Rf"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 3) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    z5f'some words {a+b:.3f} more words {c+d=} final words'aA      FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
    OP         '{'           (1, 13) (1, 14)
    NAME       'a'           (1, 14) (1, 15)
    OP         '+'           (1, 15) (1, 16)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    FSTRING_MIDDLE '.3f'         (1, 18) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
    OP         '{'           (1, 34) (1, 35)
    NAME       'c'           (1, 35) (1, 36)
    OP         '+'           (1, 36) (1, 37)
    NAME       'd'           (1, 37) (1, 38)
    OP         '='           (1, 38) (1, 39)
    OP         '}'           (1, 39) (1, 40)
    FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
    FSTRING_END "'"           (1, 52) (1, 53)
    zf'''{
3
=}'''a`      FSTRING_START "f'''"        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NUMBER     '3'           (2, 0) (2, 1)
    NL         '\n'          (2, 1) (2, 2)
    OP         '='           (3, 0) (3, 1)
    OP         '}'           (3, 1) (3, 2)
    FSTRING_END "'''"         (3, 2) (3, 5)
    f'''__{
    x:a
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    +f'''__{
    x:a
    b
     c
      d
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    OP         '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    uz       '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
    aktualni pracownicy, obecni pracownicy'''
u      INDENT     '    '        (1, 0) (1, 4)
    STRING     "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\n    aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
    NEWLINE    '\n'          (2, 45) (2, 46)
    DEDENT     ''            (3, 0) (3, 0)
    rn   ro   s    r&   test_stringTokenizeTest.test_string*  s   . 1 		 	1 4 		 	A D 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		
 	- 0 		 	- 0 		
 	- 0 		
 	C F 		 	C F 		 	C F 		 	C F 		 	 		
 	 		 	 		 	 		 	 		 	 		
 	H ' 		
 	K * 		 	- 0 		 	N 
- 
		 	E H 		* 	N 
Q 
		 	B E 		 	L + 		
 	 		 	 		 	S V 		( 	 				 	 
		 	 
		& 	 		r(   c                 p    U R                  SS5        U R                  SS5        U R                  SS5        g )N!def d22(a, b, c=2, d=2, *k): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    def d01v_(a=1, *k, **w): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    &def d23(a: str, b: int=3) -> int: passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rn   ro   s    r&   test_functionTokenizeTest.test_functionj  sJ    ? B 		, 	; > 		" 	D G 		r(   c                 (    U R                  SS5        g )NXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rn   ro   s    r&   test_comparisonTokenizeTest.test_comparison       M!O"		r(   c                 (    U R                  SS5        g )Nzx = 1 << 1 >> 5a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    rn   ro   s    r&   
test_shiftTokenizeTest.test_shift  s    - 0 		r(   c                 (    U R                  SS5        g )N%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    rn   ro   s    r&   test_additiveTokenizeTest.test_additive      C F 		r(   c                 (    U R                  SS5        g )Nx = 1//1*1/5*12%0x12@42a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rn   ro   s    r&   test_multiplicative TokenizeTest.test_multiplicative      5 8 		r(   c                 L    U R                  SS5        U R                  SS5        g )N~1 ^ 1 & 1 |1 ^ -1a      OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    -1*1/1+1*1//1 - ---1**1aH      OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rn   ro   s    r&   
test_unaryTokenizeTest.test_unary  s0    0 3 		 	5 8 		r(   c                 (    U R                  SS5        g )N/import sys, time
x = sys.modules['time'].time()a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    rn   ro   s    r&   test_selectorTokenizeTest.test_selector(      N Q 		r(   c                 (    U R                  SS5        g )N @staticmethod
def foo(x,y): passa      OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rn   ro   s    r&   test_methodTokenizeTest.test_method>      ? B 		r(   c                 (    U R                  SS5        g )Nzdef f():
	if x
        	passa_      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    rn   ro   s    r&   	test_tabsTokenizeTest.test_tabsO  s     -/		r(   c                 (    U R                  SS5        g )Nu!   Örter = 'places'
grün = 'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    rn   ro   s    r&   test_non_ascii_identifiers'TokenizeTest.test_non_ascii_identifiersd  s    @ C 		r(   c                 (    U R                  SS5        g )N#   Örter = u'places'
grün = U'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rn   ro   s    r&   test_unicodeTokenizeTest.test_unicodep      B E 		r(   c                    U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS 5        U R                  S!S"5        U R                  S#S$5        U R                  S%S&5        U R                  S'S(5        U R                  S)S*5        g )+N	async = 1z    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    a = (async = 1)a8      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    async()z    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    class async(Bar):passa<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    class async:passz    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    	await = 1z    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    	foo.asyncz    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    async for a in b: passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    async with a as b: passa<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    	async.fooz    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    async/    NAME       'async'       (1, 0) (1, 5)
    async
#comment
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    async
...
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    async
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    foo.async + 1z    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    async def foo(): passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    )async def foo():
  async for i in 1: passa      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    async def foo(async): awaitah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    @def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    rn   ro   s    r&   
test_asyncTokenizeTest.test_async|  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	 		* 	= 	@ 			 	 #		J 	 $		r(   c                 (    U R                  SS5        g )Nz[
    # A comment here
    1
]
ad      OP         '['           (1, 0) (1, 1)
    NL         '\n'          (1, 1) (1, 2)
    COMMENT    '# A comment here' (2, 4) (2, 20)
    NL         '\n'          (2, 20) (2, 21)
    NUMBER     '1'           (3, 4) (3, 5)
    NL         '\n'          (3, 5) (3, 6)
    OP         ']'           (4, 0) (4, 1)
    NEWLINE    '\n'          (4, 1) (4, 2)
    rn   ro   s    r&   3test_newline_after_parenthesized_block_with_comment@TokenizeTest.test_newline_after_parenthesized_block_with_comment  s     
			r(   c                 (    U R                  SS5        g )Nz); xz    OP         ')'           (1, 0) (1, 1)
    OP         ';'           (1, 1) (1, 2)
    NAME       'x'           (1, 3) (1, 4)
    rn   ro   s    r&   ,test_closing_parenthesis_from_different_line9TokenizeTest.test_closing_parenthesis_from_different_line  s    F % 		r(   c                 (    U R                  SS5        g )NuT   a = f'''
    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''u%      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    FSTRING_START "f'''"        (1, 4) (1, 8)
    FSTRING_MIDDLE '\n    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
    FSTRING_END "'''"         (2, 68) (2, 71)
    rn   ro   s    r&    test_multiline_non_ascii_fstring-TokenizeTest.test_multiline_non_ascii_fstring  s     OQ		r(   c                 (    U R                  SS5        g )Nu4   f'''
    🔗 This is a test {test_arg1}🔗
🔗'''uZ      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '\n    🔗 This is a test ' (1, 4) (2, 21)
    OP         '{'           (2, 21) (2, 22)
    NAME       'test_arg1'   (2, 22) (2, 31)
    OP         '}'           (2, 31) (2, 32)
    FSTRING_MIDDLE '🔗\n🔗'        (2, 32) (3, 1)
    FSTRING_END "'''"         (3, 1) (3, 4)
    rn   ro   s    r&   *test_multiline_non_ascii_fstring_with_expr7TokenizeTest.test_multiline_non_ascii_fstring_with_expr  s     		r(   r<   N)__name__
__module____qualname____firstlineno__r7   rI   rO   rb   rp   rs   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r	  r  __static_attributes__r<   r(   r&   r*   r*   &   s    9;&>F7P7	r	0&	P80~	@
:	x$	L
		.	(#	J	,	"	*
	
	P	d	"				r(   r*   c                       \ rS rSrS rSrg)GenerateTokensTesti  c                     [        U5      n[        [        R                  " UR                  5      U5      nU R                  XBR                  5       R                  5       5        g N)r   r'   r   rG   r/   r0   r1   r   r2   s        r&   r7   !GenerateTokensTest.check_tokenize  sG     QK-h.F.Fqzz.RTUV!2!=!=!?@r(   r<   N)r  r  r  r  r7   r  r<   r(   r&   r  r    s    Ar(   r  c           	         / n[         R                   " [        U R                  S5      5      R                  5      nU H  u  p4    nU[         R                  :X  af  SU;   a`  UR                  [         R                  S4[         R                  S4[         R                  [        U5      4[         R                  S4/5        M  UR                  X445        M     [         R                  " U5      R                  S5      R                  5       $ )Nr-   .Decimalr   ))r   r   r.   r/   r   extendNAMEOPSTRINGreprr   
untokenizedecodestrip)r4   r   gtoknumtokval_s         r&   decistmtr(    s    F'!((7"34==>A$%1aX__$MM	*c"$v,/c"	  MM6*+ %& v&--g6<<>>r(   c                        \ rS rSrS rS rSrg)TestMisci  c                     SSK Jn  SnU R                  [        U5      S5        U R	                  [        [        U5      5      S5        U R                  [        [        U5      5      U" S5      5        g )Nr   )r  z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalr  r0   r(  assertRegexr   eval)r3   r  r4   s      r&   test_decistmtTestMisc.test_decistmt  sd     	$"!S	U 	d1g(BC 	hqk* !DE	Gr(   c                 n    [         R                  / SQ-   nU R                  [        R                  U5        g )N)	TokenInfor   rG   detect_encodingr!  openr   )r"   __all__assertCountEqualr   )r3   r5   s     r&   test___all__TestMisc.test___all__  s-    == $
 
 	h..9r(   r<   N)r  r  r  r  r/  r7  r  r<   r(   r&   r*  r*    s    G(:r(   r*  c                   8    \ rS rSrS rS rS rS rS rS r	Sr
g	)
TestTokenizerAdheresToPep0263i  c                     [         R                  R                  [         R                  R                  [        5      SU5      n[        US5       n[        R                  X5        S S S 5        g ! , (       d  f       g = f)Ntokenizedatarb)ospathjoindirname__file__r4  TestRoundtripcheck_roundtrip)r3   rY   r?  r6   s       r&   	_testFile'TestTokenizerAdheresToPep0263._testFile  sK    ww||BGGOOH5~xP$))$2 s   A..
A<c                 *    SnU R                  U5        g )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtrE  r3   r6   s     r&   'test_utf8_coding_cookie_and_no_utf8_bomETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom      Gqr(   c                 J    SnU R                  [        U R                  U5        g )Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtrC   SyntaxErrorrE  rI  s     r&   &test_latin1_coding_cookie_and_utf8_bomDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom  s      G+t~~q9r(   c                 *    SnU R                  U5        g )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrH  rI  s     r&   "test_no_coding_cookie_and_utf8_bom@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom  rL  r(   c                 *    SnU R                  U5        g )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrH  rI  s     r&   $test_utf8_coding_cookie_and_utf8_bomBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom  s    Dqr(   c                     U R                  [        U R                  S5        U R                  [        U R                  S5        g )Nzbad_coding.pyzbad_coding2.pyrN  ro   s    r&   test_bad_coding_cookie4TestTokenizerAdheresToPep0263.test_bad_coding_cookie  s/    +t~~G+t~~7GHr(   r<   N)r  r  r  r  rE  rJ  rP  rS  rV  rY  r  r<   r(   r&   r:  r:    s!    
3
	:Ir(   r:  c                       \ rS rSrS rSrg)Test_Tokenizei  c                    ^^ SnUR                  S5      mSmUU4S jn[        [        R                  " U" 5       R                  SSS95      S S n[        R
                  " SSS	S
S5      /nU R                  X4S5        g )Nu   "ЉЊЈЁЂ"r-   Fc               3   2   >#    T (       d  Sm Tv   g Sv   g 7f)NTr(   r<   )firstr%   s   r&   r/   NTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline!  s     
	s   T)encodingextra_tokensrM   rR   rB   r   rB      zbytes not decoded with encoding)r.   rE   r   !_generate_tokens_from_c_tokenizerrF   r2  r0   )r3   literalr/   rN   expected_tokensr_  r%   s        @@r&   .test__tokenize_decodes_with_specified_encoding<Test_Tokenize.test__tokenize_decodes_with_specified_encoding  s     ~~g&	 h@@ATATJQNRT UUXVXZ $--aQ_`a:	<r(   r<   N)r  r  r  r  ri  r  r<   r(   r&   r\  r\    s    <r(   r\  c                       \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rSrg)TestDetectEncodingi2  c                    ^^ SmUU4S jnU$ )Nr   c                  F   > T[        T5      :X  a  [        eTT   n TS-  mU $ NrB   )r   StopIteration)r%   indexliness    r&   r/   1TestDetectEncoding.get_readline.<locals>.readline6  s,    E
"##<DQJEKr(   r<   )r3   rr  r/   rq  s    ` @r&   get_readlineTestDetectEncoding.get_readline4  s    	 r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  U[	        US S 5      5        g )N)   #!/home/ä/bin/python
s   # something €
   print(something)
   do_something(else)
r-      r   r3  rt  r0   rE   r3   rr  ra  consumed_liness       r&   test_no_bom_no_encoding_cookie1TestDetectEncoding.test_no_bom_no_encoding_cookie?  sS    
 $,#;#;D<M<Me<T#U 7+eBQi9r(   c                     SnU R                  [        5         [        R                  " U R	                  U5      5        S S S 5        g ! , (       d  f       g = f)N)s   #!/home//bin/python

rx  ry  rC   rO  r   r3  rt  r3   rr  s     r&   /test_no_bom_no_encoding_cookie_first_line_errorBTestDetectEncoding.test_no_bom_no_encoding_cookie_first_line_errorJ  >    

 {+$$T%6%6u%=> ,++   &A
Ac                     SnU R                  [        5         [        R                  " U R	                  U5      5        S S S 5        g ! , (       d  f       g = f)N)   #!/usr/bin/python
   # something 
rx  ry  r  r  s     r&   0test_no_bom_no_encoding_cookie_second_line_errorCTestDetectEncoding.test_no_bom_no_encoding_cookie_second_line_errorS  >    
 {+$$T%6%6u%=> ,++r  c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  USS/5        g )N)s   ﻿#!/home/ä/bin/python
rx  ry  	utf-8-sigrw  rx  r   r3  rt  r0   r|  s       r&   test_bom_no_cookie%TestDetectEncoding.test_bom_no_cookie]  sS    

 $,#;#;D<M<Me<T#U ;/:<QR	Tr(   c                     SnU R                  [        5         [        R                  " U R	                  U5      5        S S S 5        g ! , (       d  f       g = f)N)s   ﻿#!/home//bin/python
rx  ry  r  r  s     r&   #test_bom_no_cookie_first_line_error6TestDetectEncoding.test_bom_no_cookie_first_line_errorh  r  r  c                     SnU R                  [        5         [        R                  " U R	                  U5      5        S S S 5        g ! , (       d  f       g = f)N)s   ﻿#!/usr/bin/python
r  rx  ry  r  r  s     r&   $test_bom_no_cookie_second_line_error7TestDetectEncoding.test_bom_no_cookie_second_line_errorq  r  r  c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  US/5        g )N)   # -*- coding: latin-1 -*-
rx  ry  
iso-8859-1r  r  r|  s       r&   test_cookie_first_line_no_bom0TestDetectEncoding.test_cookie_first_line_no_bom{  sM    

 $,#;#;D<M<Me<T#U <0*H)IJr(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  US/5        g )N)s   ﻿# coding=utf-8
rx  ry  r  s   # coding=utf-8
r  r|  s       r&   &test_matched_bom_and_cookie_first_line9TestDetectEncoding.test_matched_bom_and_cookie_first_line  sM    

 $,#;#;D<M<Me<T#U ;/*=)>?r(   c                 t    SnU R                  U5      nU R                  [        [        R                  U5        g )N)s#   ﻿# vim: set fileencoding=ascii :
rx  ry  rt  rC   rO  r   r3  r3   rr  r/   s      r&   <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerror  s3    

 $$U++x'?'?Jr(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        SS/nU R                  X45        g )N)   #! something
    # vim: set fileencoding=ascii :
rx  ry  asciir  r  r  r3   rr  ra  r}  r5   s        r&   test_cookie_second_line_no_bom1TestDetectEncoding.test_cookie_second_line_no_bom  sR    
 $,#;#;D<M<Me<T#U 7+%'KL2r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  USS/5        g )N)   ﻿#! something
   f# coding=utf-8
rx  ry  r  r  r  r  r|  s       r&   'test_matched_bom_and_cookie_second_line:TestDetectEncoding.test_matched_bom_and_cookie_second_line  sS    
 $,#;#;D<M<Me<T#U ;/+-AB	Dr(   c                 t    SnU R                  U5      nU R                  [        [        R                  U5        g )N)r  r  rx  ry  r  r  s      r&   =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror  s3    
 $$U++x'?'?Jr(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        S/nU R                  X45        g )N)   print('£')
%   # vim: set fileencoding=iso8859-15 :
   print('€')
r-   r  r  r  s        r&   /test_cookie_second_line_noncommented_first_lineBTestDetectEncoding.test_cookie_second_line_noncommented_first_line  sO    

 $,#;#;D<M<Me<T#U 7+*+2r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        SS/nU R                  X45        g )N)   
r  r  
iso8859-15r  r  r  r  s        r&   (test_cookie_second_line_empty_first_line;TestDetectEncoding.test_cookie_second_line_empty_first_line  sR    

 $,#;#;D<M<Me<T#U <0DE2r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  U[	        US S 5      5        g )N)rw  s   # something
r  rx  ry  r-   rz  r{  r|  s       r&   test_cookie_third_line)TestDetectEncoding.test_cookie_third_line  sS    
 $,#;#;D<M<Me<T#U 7+eBQi9r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  U[	        US S 5      5        g )N)s   #coding:iso8859-15
   #coding:latin1
rx  r  rB   r{  r|  s       r&   test_double_coding_line*TestDetectEncoding.test_double_coding_line  sS    

 $,#;#;D<M<Me<T#U <0eBQi9r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  U[	        US S 5      5        g )N)s!   #coding:iso8859-15 coding:latin1
rx  r  rB   r{  r|  s       r&   test_double_coding_same_line/TestDetectEncoding.test_double_coding_same_line  sS    
 $,#;#;D<M<Me<T#U <0eBQi9r(   c                     Sn[         R                  " U R                  U5      5      u  p#U R                  US5        U R                  U[	        US S 5      5        g )N)s   #coding:utf-8
r  rx  r-   rB   r{  r|  s       r&   test_double_coding_utf8*TestDetectEncoding.test_double_coding_utf8  sS    

 $,#;#;D<M<Me<T#U 7+eBQi9r(   c                     SnU Ht  nS Hk  nUR                  SU5      nSSUR                  S5      -   S-   SS	4nU R                  U5      n[        R                  " U5      u  pxU R                  US
5        Mm     Mv     g )N)latin-1r  ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac-r'  r  r  
   # coding: r  r  s   print(things)
s   do_something += 4
r  replacer.   rt  r   r3  r0   	r3   	encodingsra  repencrr  rlfoundr}  s	            r&   test_latin1_normalization,TestDetectEncoding.test_latin1_normalization  s    ;	!H!&&sC0/&G)<<uD+/1 &&u-(0(@(@(D%  5 " "r(   c                 t    SnU R                  U5      nU R                  [        [        R                  U5        g )Ns
   print("")r  r  s      r&   test_syntaxerror_latin1*TestDetectEncoding.test_syntaxerror_latin1	  s3     $$U++x'?'?Jr(   c                     SnU Hs  nS Hj  nUR                  SU5      nSSUR                  S5      -   S-   S4nU R                  U5      n[        R                  " U5      u  pxU R                  US	5        Ml     Mu     g )
N)r-   z	utf-8-macz
utf-8-unixr  r  r  r  r  r  s   1 + 3
r-   r  r  s	            r&   test_utf8_normalization*TestDetectEncoding.test_utf8_normalization  s    8	!H!&&sC0/&G)<<uD#% &&u-(0(@(@(D%  0 " "r(   c                    U R                  S5      n[        R                  " U5      u  p#U R                  US5        U R                  US/5        [        R                  " U R                  S5      5      u  p#U R                  US5        U R                  U/ 5        U R                  S5      n[        R                  " U5      u  p#U R                  US5        U R                  US/5        U R                  S5      n[        R                  " U5      u  p#U R                  US5        U R                  U/ 5        U R                  S5      nU R	                  [
        [        R                  U5        g )	N)rx  r-   rx  r<   )s   ﻿print(something)
r  )   ﻿)s   # coding: bad
)rt  r   r3  r0   rC   rO  r3   r/   ra  r}  s       r&   test_short_files#TestDetectEncoding.test_short_files  s=   $$%=>#+#;#;H#E 7+*?)@A#+#;#;D<M<Mb<Q#R 7+,$$%IJ#+#;#;H#E ;/*?)@A$$%78#+#;#;H#E ;/,$$%:;+x'?'?Jr(   c                     U R                  S5      n[        R                  " U5      u  p#U R                  US5        U R                  US/5        g )N)   print("#coding=fake")r-   r  )rt  r   r3  r0   r  s       r&   test_false_encoding&TestDetectEncoding.test_false_encoding6  sJ    $$%@A#+#;#;H#E 7+*B)CDr(   c                    [         R                  S-   nU R                  [         R                  U5        S H  n[	        USUS9 n[        SU-  US9  [        SUS9  S S S 5        [        R                  " U5       nU R                  UR                  U5        U R                  UR                  S5        S S S 5        M     [	        USS	S9 n[        SUS9  S S S 5        [        R                  " U5       nU R                  UR                  S	5        U R                  UR                  S5        S S S 5        g ! , (       d  f       N= f! , (       d  f       GM   = f! , (       d  f       N= f! , (       d  f       g = f)
Nz.py)ziso-8859-15r-   wra  z# coding: %sfileu   print('euro:€')rr  )
r	   TESTFN
addCleanupunlinkr4  printr   r0   ra  mode)r3   rY   ra  fps       r&   	test_openTestDetectEncoding.test_open=  s   ##e+	(((3 1Hhh72nx/b9,26 8 x(B  h7  #. )(	 1 (C+6"(r2 7]]8$R[[+6RWWc* %$ 87 )(
 76$$s0   D5:9E
E39E*5
E	
E	
E'*
E8c                   ^ SnSm " U4S jS5      nU R                  [        5         U" TU5      nU?[        R                  " UR
                  5        S S S 5        U R                  [        SR                  U5      5         U" TU5      n[        R                  " UR
                  5        S S S 5        g ! , (       d  f       Nf= f! , (       d  f       g = f)Nsome_file_pathr  c                   (   > \ rS rSrS rU 4S jrSrg);TestDetectEncoding.test_filename_in_exception.<locals>.BunkiW  c                 *    X l         Xl        SU l        g )Nr   )name_lines_index)r3   rr  r?  s      r&   __init__DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__X  s     	#r(   c                    > U R                   [        T5      :X  a  [        eTU R                      nU =R                   S-  sl         U$ ro  )r   r   rp  )r3   r%   rr  s     r&   r/   DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline]  s:    ;;#e*,''T[[)q r(   )r   r  r  N)r  r  r  r  r  r/   r  )rr  s   r&   Bunkr  W  s     
 r(   r  z.*{})rC   rO  r  r   r3  r/   rU   format)r3   r?  r  insrr  s       @r&   test_filename_in_exception-TestDetectEncoding.test_filename_in_exceptionQ  s    	 	 {+ud#C$$S\\2	 ,
 ##Kt1DEud#C$$S\\2 FE ,+
 FEs   ,B4*C4
C
Cc                     [        S5      n[        R                  " SUS9   U R                  [        [
        R                  S5        S S S 5        U R                  UR                  5        g ! , (       d  f       N*= f)Ns   #coding:xxxztokenize._builtin_open)return_valuefoobar)	r   r   patchrC   rO  r   r4  
assertTrueclosed)r3   ms     r&   test_open_error"TestDetectEncoding.test_open_errorm  sQ    N#ZZ0qAk8==(C B! BAs   &A++
A9r<   N) r  r  r  r  rt  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r<   r(   r&   rl  rl  2  s    		:??	T??K@K
3
DK	3	3
:	:::6K1K.E+(38"r(   rl  c                   D    \ rS rSrS rS rS rS rS rS r	S r
S	 rS
rg)TestTokenizeiu  c                 V  ^^	^
 SS K nSm	S m
U	4S jnU
4S jnSmU4S jnUR                  nUR                  nX!l        X1l         [         R                   " U5      nU R                  [	        U5      SS  / SQ5        XQl        Xal        U R                  T
T	5        g ! XQl        Xal        f = f)Nr   r-   c                    > TSS/4$ )N   first   secondr<   )r/   ra  s    r&   mock_detect_encoding8TestTokenize.test_tokenize.<locals>.mock_detect_encoding{  s    h	222r(   c                 z   > Um/ n  U " 5       nU(       a  UR                  U5        M#  U$ ! [          a    Us $ f = fr  )rp  r   )r/   ra  kwargsout	next_lineencoding_useds        r&   mock__tokenize2TestTokenize.test_tokenize.<locals>.mock__tokenize~  sK    $MC (
I JJy)
 % Js   + ::c                  N   > T S-  m T S:X  a  g[        T 5      R                  5       $ )NrB      r(   )strr.   )counters   r&   mock_readline1TestTokenize.test_tokenize.<locals>.mock_readline  s)    qLG!|w<&&((r(   rB   )r  r     1   2   3   4)r   r3  rf  r0   rE   )r3   tokenize_moduler  r   r&  orig_detect_encodingorig_c_tokenresultsr%  ra  r  s           @@@r&   test_tokenizeTestTokenize.test_tokenizew  s    *	3	 	)  />>&HH*>'<J9	M''6GT']12.JL /C+@L=1 /C+@L=s   6B B(c           	      l   / n[        S5       H!  nUR                  SR                  US95        M#     UR                  S5        SR                  U5      n[	        [
        R
                  " [        UR                  S5      5      R                  5      5      nU R                  US   R                  S5        g )Ni  zdef i{i}(): return {i})iOK
r-   )ranger   r  r@  rE   r   r   r.   r/   r0   string)r3   bufr3  tokss       r&   test_oneline_defsTestTokenize.test_oneline_defs  s    sAJJ/666;< 

4iin H%%gcjj.A&B&K&KLMb$/r(   c           	      l   [        [        R                  " [        UR                  S5      5      R                  5      5      n[        U5      nU R                  [        U5      SU-   5        U R                  [        R                  US   R                     [        R                  [        R                     5        [        U5       HH  nU R                  [        R                  X5S-      R                     [        R                  X%      5        MJ     U R                  [        R                  USU-      R                     [        R                  [        R                     5        U R                  [        R                  USU-      R                     [        R                  [        R                     5        g )Nr-   rR   r   rB   rz  )rE   r   r   r.   r/   r   r0   r   
exact_typeENCODINGr7  r"   r   r   )r3   opstroptypesrN   num_optypesr3  s         r&   assertExactTypeEqual!TestTokenize.assertExactTypeEqual  sE   h''W0E(F(O(OPQ'lVa+o6**6!9+?+?@!**8+<+<=	?{#AX..v!e}/G/GH%..wz:< $ 	**6!k/+B+M+MN!**5==9	;**6!k/+B+M+MN!**5??;	=r(   c                    U R                  S[        R                  [        R                  5        U R                  S[        R                  [        R
                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S	[        R                  5        U R                  S
[        R                  5        U R                  S[        R                  5        U R                  S[        R                  5        U R                  S[        R                   5        U R                  S[        R"                  5        U R                  S[        R$                  5        U R                  S[        R&                  5        U R                  S[        R(                  [        R*                  5        U R                  S[        R,                  5        U R                  S[        R.                  5        U R                  S[        R0                  5        U R                  S[        R2                  5        U R                  S[        R4                  5        U R                  S[        R6                  5        U R                  S[        R8                  5        U R                  S[        R:                  5        U R                  S[        R<                  5        U R                  S[        R>                  5        U R                  S[        R@                  5        U R                  S[        RB                  5        U R                  S[        RD                  5        U R                  S[        RF                  5        U R                  S [        RH                  5        U R                  S![        RJ                  5        U R                  S"[        RL                  5        U R                  S"[        RL                  5        U R                  S#[        RN                  5        U R                  S$[        RP                  5        U R                  S%[        RR                  5        U R                  S&[        RT                  5        U R                  S'[        RV                  5        U R                  S([        RX                  5        U R                  S)[        RZ                  5        U R                  S*[        R\                  5        U R                  S+[        R^                  5        U R                  S,[        R`                  5        U R                  S-[b        Rd                  [        R<                  [b        Rf                  [        R                  [b        Rd                  [        R<                  [b        Rf                  [        R,                  [b        Rd                  [        R<                  [b        Rf                  5        U R                  S.[        R(                  [        Rf                  [        R                  [        Rf                  [        R                  [        Rf                  [        R*                  5        U R                  S/[        R6                  [        R                  [        Rd                  [        R                  [        Rf                  [        R                  5        g )0Nz()z[]:,;+r  */|&<>=r  %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))4rC  r"   LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL	LESSEQUALGREATEREQUALTILDE
CIRCUMFLEX	LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR	PLUSEQUALMINEQUAL	STAREQUAL
SLASHEQUALPERCENTEQUAL
AMPEREQUAL	VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUALDOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr   r  r   ro   s    r&   test_exact_typeTestTokenize.test_exact_type  s   !!$

EJJ?!!$

EJJ?!!#u{{3!!#u{{3!!#uzz2!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u}}5!!#u{{3!!#uyy1!!#u}}5!!$ellC!!$6!!$7!!$8!!$(:(:;!!#u{{3!!#u'7'78!!$8!!$(8(89!!$(8(89!!$8!!$7!!$8!!$(8(89!!$(:(:;!!$(8(89!!$8!!$(=(=>!!$(=(=>!!%)=)=>!!%)>)>?!!%)>)>?!!$(9(9:!!%)?)?@!!$(8(89!!%8!!$5!!#uxx0!!$6!!"3"*--1A1A8??"'**"*--1A1A8??"'--"*--1A1A8??	T 	!!+"',,"',,"',,"',,"',,	0 	!!,"'"2"2"'**"'**ekk5<<"'**		.r(   c                 D    U R                  S[        R                  5        g )Nz@          )rC  r"   r  ro   s    r&   %test_pathological_trailing_whitespace2TestTokenize.test_pathological_trailing_whitespace  s    !!-:r(   c                 J   Sn[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSS	S9[         R                  " [        R
                  S
SSS	S9[         R                  " [        R                  SSSS	S9[         R                  " [        R                  SSSS	S9[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9/	n[        [         R                   " [        UR                  S5      5      R                  5      5      nU R                  X25        g )Nzb = 1

#testr-   r   r    r!   r8  r#   r$   r%   brc  rB   rB   zb = 1
rP  rB   rz  rB   rR   1rB      rB   r#  r5  rB      rz  r   rz  rB   z#testrR   r   )rR   r#  )rR   r  )r  r   )r   r2  r"   r?  r  r  r   r   NLCOMMENTr   rE   r   r.   r/   r0   r3   sourcerh  rN   s       r&   5test_comment_at_the_end_of_the_source_without_newlineBTestTokenize.test_comment_at_the_end_of_the_source_without_newline  s?   !ENN7&V\cefEJJs&f[deEHHSFYbcELLFPV]fgEMM$fRX_hiEHHTVZ^_EMM'U[bijEHHRv6X_`EOOBfRX_ab

 h''g0F(G(P(PQR1r(   c                 "   Sn[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSS	S9[         R                  " [        R
                  S
SSS	S9[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9/n[        [         R                   " [        UR                  S5      5      R                  5      5      nU R                  X25        g )Nza
 r-   r  r  )r8  r#   r$   r%   arc  r  za
r5  r  r  )rz  rz  r   r  )r   r2  r"   r?  r  r   r  r   rE   r   r.   r/   r0   r  s       r&   ?test_newline_and_space_at_the_end_of_the_source_without_newlineLTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline  s    u~~gVQW^`auzz#VV[\u}}TVZ_`uxx&fSVWurVZ\]
 h''g0F(G(P(PQR1r(   c                 f   Sn[         R                  " 5        n[        R                  R	                  US5      n[        US5       nUR                  U5        S S S 5        [        U5      u  pVU R                  SUR                  5        S S S 5        g ! , (       d  f       N@= f! , (       d  f       g = f)Ns   F"""
        """z	script.pywbs   SyntaxError)
r	   temp_dirr>  r?  r@  r4  writer   assertInerr)r3   scriptr  rY   r  rsr'  s          r&   (test_invalid_character_in_fstring_middle5TestTokenize.test_invalid_character_in_fstring_middle!  s     !Xww||Hk:Hh%

6" &(2EBMM."&&1 "!%% "!s#   -B"B1B"
B	B""
B0r<   N)r  r  r  r  r0  r;  rC  r  r  r  r  r  r  r<   r(   r&   r  r  u  s,    )2V	0=?.B;2$2
2r(   r  c                   &    \ rS rSrS rS rS rSrg)UntokenizeTesti.  c                 ^   [         R                  " 5       nSUl        SUl        U R	                  [
        5       nUR                  S5        S S S 5        U R                  WR                  R                  S   S5        U R	                  [
        UR                  S5        g ! , (       d  f       NY= f)Nrz  r  r   z'start (1,3) precedes previous end (2,2)r  )
r   Untokenizerprev_rowprev_colrC   
ValueErroradd_whitespacer0   rW   args)r3   ucms      r&   test_bad_input_order#UntokenizeTest.test_bad_input_order0  s      "

z*bU# +**1-9	; 	*a&6&6> +*s   B
B,c                 L   [         R                  " 5       nSUl        SUl        / Ul        UR                  S5        U R                  UR                  S/5        SUl        UR                  S5        U R                  UR                  / SQ5        [        R                  U S5        g )NrB   r  \
rz  )r  r  )r  z\
\
r   za
  b
    c
  \
  c
)	r   r  r  r  rN   r  r0   rC  rD  )r3   r  s     r&   test_backslash_continuation*UntokenizeTest.test_backslash_continuation<  s      "

	 F8,
	 #?@%%d,HIr(   c                 ~   [         R                  " 5       n[         R                  S4n[         R                  S4U/nUR	                  U[        / 5      5        U R                  UR                  S/5        [         R                  " 5       nU R                  UR                  [        U/5      5      S5        [         R                  " 5       nU R                  UR                  [        U5      5      S5        U R                  UR                  S5        U R                  [         R                  " [        U5      5      S5        g )NHellor-   zHello s   Hello )
r   r  r  r?  compatiterr0   rN   r!  ra  )r3   r  r"   rN   s       r&   test_iter_compatUntokenizeTest.test_iter_compatI  s      "($$g.6	R!H:.  "dE7m4h?  "d6l3X>W-,,T&\:IFr(   r<   N)r  r  r  r  r  r  r  r  r<   r(   r&   r  r  .  s    
?JGr(   r  c                 T    [         R                  " S5      nUR                  U 5      S L$ )Ns   \n\s*\\\r?\n)recompilesearch)r  patterns     r&   contains_ambiguous_backslashr  W  s'     jj)*G>>&!--r(   c                   J    \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rSrg)rC  ig  c                    [        U[        5      (       a  UR                  S5      nOUR                  5       n[	        UR                  SS95      R                  n[        [        R                  " U5      5      nU Vs/ s H  oUS S PM	     nn[        R                  " U5      n[	        UR                  SS95      R                  n[        R                  " U5       Vs/ s H  oUS S PM	     n	nU R                  X5        [        R                  " U5      n
[	        U
R                  SS95      R                  n[        R                  " U5       Vs/ s H  oUS S PM	     nnU R                  X5        [        U5      (       do  UR                  S5      n[	        UR                  SS95      R                  n[        R                  " [        R                  " U5      5      nU R                  X5        g g s  snf s  snf s  snf )Nr-   Tkeependsrz  r  )
isinstancer$  r.   readr  r   rF   rE   r   r!  r0   r  removeprefix)r3   r6   coder/   tokens5ra   tokens2bytes_from2	readline2tokens2_from2bytes_from5	readline5tokens2_from5code_without_bomuntokenized_codes                  r&   rD  TestRoundtrip.check_roundtripi  s     a88G$D668D67@@x((23&-.gsr7g.))'2///>?HH	,4,=,=i,HI,HSRa,HI0))'2///>?HH	,4,=,=i,HI,HSRa,HI0+D11#00A,777FGPPH'2283D3DX3NO-@ 2 / J
 Js   =G/G4G9c                    [        U[        5      (       a  UR                  S5      nOUR                  5       n[	        UR                  SS95      R                  n[        R                  " U5       Hv  nUR                  [        R                  [        R                  1;   a  M3  U R                  UR                  UR                  UR                  S   UR                  S    5        Mx     g )Nr-   Tr  rB   )r  r$  r.   r  r  r   rF   r   r!   r?  r   r0   r8  r%   r#   r$   )r3   r6   r  r/   ra   s        r&   check_line_extraction#TestRoundtrip.check_line_extraction  s    a88G$D668D67@@$$X.CxxX..0B0BCCSZZ#))A,
)KL /r(   c                    U R                  S5        U R                  S5        U R                  S5        [        R                  " SSS9n[        US5       nU R                  U5        S S S 5        U R                  S5        U R                  S	5        U R                  S
5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S5        U R                  S 5        U R                  S!5        U R                  S"5        U R                  S#5        U R                  S$5        U R                  S%5        U R                  S&5        U R                  S'5        U R                  S(5        U R                  S)5        U R                  S*5        / S+QnU H  nU R                  U5        M     g ! , (       d  f       GN= f),Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtr<  )subdirr=  zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
zclass Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
zf'\N{EXCLAMATION MARK}'zf'\\N{SNAKE}'zf'\\N{{SNAKE}}'zf'\N{SNAKE}'zf'\\\N{SNAKE}'zf'\\\\\N{SNAKE}'zf'\\\\\\\N{SNAKE}'z	f'\\N{1}'zf'\\\\N{2}'zf'\\\\\\N{3}'zf'\\\\\\\\N{4}'zf'\\N{{'z
f'\\\\N{{'zf'\\\\\\N{{'zf'\\\\\\\\N{{'zf'\n{{foo}}'zf'\\n{{foo}}'zf'\\\n{{foo}}'zf'\\\\n{{foo}}'zf'\t{{foo}}'zf'\\t{{foo}}'zf'\\\t{{foo}}'zf'\\\\t{{foo}}'zrf'\t{{foo}}'zrf'\\t{{foo}}'zrf'\\\t{{foo}}'zrf'\\\\t{{foo}}'zrf'\{{foo}}'zf'\\{{foo}}'zrf'\\\{{foo}}'zf'\\\\{{foo}}')z
if 1:
    "foo"
"bar"
z
if 1:
    ("foo"
     "bar")
z
if 1:
    "foo"
    "bar"
)rD  r   findfiler4  )r3   fnr6   casescases        r&   test_roundtripTestRoundtrip.test_roundtrip  s    	 . 	/ - 	. 	 , 	-2>J"d^q  #  1 	2
 	 3 	4 	 B 	C 	 9 	:
 	78-./0_-./0123\*^,-./0[)]+_-./_--..//0_--..//0-..//001_-_-././  D  & Y ^s   K!!
K0c                 &    U R                  S5        g )Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
rD  ro   s    r&   test_continuationTestRoundtrip.test_continuation  s     	& 		'r(   c                 H    U R                  S5        U R                  S5        g )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r  ro   s    r&   r  )TestRoundtrip.test_backslash_continuation
  s*     - 	. 	 % 	&r(   c                 &    U R                  S5        g )Nz'' ''r  ro   s    r&   test_string_concatenation'TestRoundtrip.test_string_concatenation  s    W%r(   c                    SS K nSS Kn[        R                  R	                  [
        5      =(       d    [        R                  nUR                  [        R                  R                  UR                  U5      S5      5      n[        R                  " S5      (       d  UR                  US5      nU Hr  n[        R                  S:  a  [        SU5        [        US5       nU R                  US9   U R!                  U5        U R#                  U5        S S S 5        S S S 5        Mt     g ! , (       d  f       N= f! , (       d  f       M  = f)	Nr   ztest*.pycpu
   rz  r   r=  r  )globrandomr>  r?  rA  rB  curdirr@  escaper   is_resource_enabledsampleverboser  r4  subTestrD  r  )r3   r  r  tempdir	testfilestestfiler6   s          r&   test_random_filesTestRoundtrip.test_random_files  s     	''//(+8ryyIIbggll4;;w+?LM	**511i4I!H!#j(+h%\\x\0((+..q1 1 &% " 10 &%s$   D8/#D'D8'
D51D88
E	c                     [        U[        5      (       a  UR                  S5      n[        R                  " [        R                  " [        U5      R                  5      5      R                  S5      $ )Nr-   )r  r$  r.   r   r!  r   r/   r"  )r3   r  s     r&   	roundtripTestRoundtrip.roundtrip+  sO    dC  ;;w'D""8#4#4WT]5K5K#LMTTU\]]r(   c                     SnU R                  U5      R                  S5      nU R                  US   US   5        U R                  U5        g )Nzif False:
	x=3
	x=3
r5  rB   rz  )r  splitr0   rD  )r3   r  	codeliness      r&   #test_indentation_semantics_retained1TestRoundtrip.test_indentation_semantics_retained0  sH    
 +NN4(..t4	1y|4T"r(   r<   N)r  r  r  r  rD  r  r  r  r  r  r   r  r  r  r<   r(   r&   rC  rC  g  s5    'AR	M]'@'&&2(^
#r(   rC  c                        \ rS rSrS rS rSrg)InvalidPythonTestsi;  c                    Sn[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9[         R                  " [        R
                  SSS	SS9[         R                  " [        R                  S
S	SSS9[         R                  " [        R
                  SSSSS9[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9/n[        [         R                  " [        U5      R                  5      5      nU R                  X25        g )Nz2sin(x)2rc  r  r  sinr  r   r  rL   r  r  rd  r  )rB      r  )r   r2  r"   r   r  r  r   r   rE   rG   r   r/   r0   r  s       r&   test_number_followed_by_name/InvalidPythonTests.test_number_followed_by_name<  s    ELLFPV]fgEJJuFPV]fgEHHSFYbcEJJs&f[deEHHSFYbcEMM"FPV]fgEOOBfRX_ab
 h..x/?/H/HIJ1r(   c           
      p   Sn[         R                  " [        R                  SSSSS9[         R                  " [        R                  SSSSS9[         R                  " [        R
                  SSSSS9/n[        [         R                  " [        U5      R                  5      5      nU R                  X25        g )N01234rc  r  r  r  r  r  )r   r2  r"   r   r   r   rE   rG   r   r/   r0   r  s       r&   test_number_starting_with_zero1InvalidPythonTests.test_number_starting_with_zeroL  s    ELLTZahiEMM"FPV]deEOOBfRX_ab
 h..x/?/H/HIJ1r(   r<   N)r  r  r  r  r  r  r  r<   r(   r&   r  r  ;  s    2 	2r(   r  c                       \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS rS rS rS rSrg)CTokenizeTestiW  c                    [        U5      nU R                  US9   [        [        R                  " UR
                  5      U5      nU R                  XBR                  5       R                  5       5        S S S 5        g ! , (       d  f       g = f)N)r  )	r   r  r'   r   rf  r/   r0   r1   r   r2   s        r&   r7   CTokenizeTest.check_tokenizeX  se     QK\\\#1::1::FF V__%6%A%A%CD	 $##s   AA<<
B
c                 f   S n[         R                  " [         R                  SSSSS9[         R                  " [         R                  SSSSS9[         R                  " [         R                  SSS	SS9[         R                  " [         R                  S
S	SSS9[         R                  " [         R
                  S
SSS
S9/nS H[  nU R                  US9   [        [         R                  " U" U5      R                  SUS95      nU R                  XB5        S S S 5        M]     g ! , (       d  f       Mo  = f)Nc              3   2   #    SR                  U 5      v   g 7f)N1+1)r.   r  s    r&   r/   -CTokenizeTest.test_encoding.<locals>.readlinec  s     ,,x((   r  rc  r  r  r  rI  r  r  r  r  r  )r-   r  zutf-16r  T)rb  ra  )r   r2  r   r  r   r   r  rE   rf  rF   r0   )r3   r/   r5   ra  rN   s        r&   test_encodingCTokenizeTest.test_encodingb  s   	) HOOCvSY`efHKK6v\abHOOCvSY`efH$4$4RvSY`efH$6$6rU[bde
 7Hx0hHHX&//!%% 
   2 10 700s   AD!!
D0	c                 H   U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        g )Nre   z    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rf   z    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rg   z    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    rh   z    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    ri   z    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rj   ac      LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    rk   z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rl   z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rm   z    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rn   ro   s    r&   rp   CTokenizeTest.test_intv  s    M , 		 	M , 		 	, / 		 	- 0 		 	0 3 		 	5 	8 			 	. 1 		 	0 3 		 	0 3 		r(   c                     U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        g )Nrv   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    rw   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    rx   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    ry   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    rz   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    r{   z    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    r|   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rn   ro   s    r&   r}   CTokenizeTest.test_float  s    M , 		 	M , 		 	M , 		 	M , 		 	K * 		 	O . 		 	N - 		r(   c                 `   U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS 5        U R                  S!S"5        U R                  S#S$5        U R                  S%S&5        U R                  S'S(5        U R                  S)S*5        U R                  S+S,5        U R                  S-S.5        U R                  S/S05        U R                  S1S25        U R                  S3S45        U R                  S5S65        U R                  S7S85        U R                  S9S:5        U R                  S;S<5        U R                  S=S>5        g )?Nr   a5      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    r   a6      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    r   z    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    r   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    r   a      NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    r   z    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    r   z    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    r   z    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    r   z    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r   a;      STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r   a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    r   r   r   r   r   r   r   r   r   r   r   r   z"hola\\\r\ndfgf"z9    STRING     '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
    r   r   r   a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    RBRACE     '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    r   r   r   r   zf"{
a}"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    LBRACE     '{'           (1, 2) (1, 3)
    NAME       'a'           (2, 0) (2, 1)
    RBRACE     '}'           (2, 1) (2, 2)
    FSTRING_END '"'           (2, 2) (2, 3)
    r   r   zf"hola\\\r\ndfgf"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
    FSTRING_END '"'           (1, 16) (1, 17)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    RBRACE     '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    rn   ro   s    r&   r   CTokenizeTest.test_string  s   . 1 		 	1 4 		 	> A 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		 	- 0 		 	- 0 		 	- 0 		 	C F 		 	C F 		 	C F 		 	C F 		 	- 0 		 	L + 		 	L + 		 	 		 	 		 	 		 	/ 2 		 	H ' 		 	K * 		 	L + 		 	 		 	 			 	 		 	0 3 		 	 

		 	 

		r(   c                 p    U R                  SS5        U R                  SS5        U R                  SS5        g )Nr   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rn   ro   s    r&   r   CTokenizeTest.test_function	  sJ    ? B 		. 	; > 		$ 	D G 		r(   c                 (    U R                  SS5        g )Nr   a      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rn   ro   s    r&   r   CTokenizeTest.test_comparison
  r   r(   c                 (    U R                  SS5        g )Nr   a$      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    rn   ro   s    r&   r   CTokenizeTest.test_additiveC
  r   r(   c                 (    U R                  SS5        g )Nr   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rn   ro   s    r&   r   !CTokenizeTest.test_multiplicativeZ
  r   r(   c                 L    U R                  SS5        U R                  SS5        g )Nr   a      TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r   aI      MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rn   ro   s    r&   r   CTokenizeTest.test_unaryn
  s0    0 3 		 	5 8 		r(   c                 (    U R                  SS5        g )Nr   a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    rn   ro   s    r&   r   CTokenizeTest.test_selector
  r   r(   c                 (    U R                  SS5        g Nr   a      AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rn   ro   s    r&   r   CTokenizeTest.test_method
  r   r(   c                 (    U R                  SS5        g r4  rn   ro   s    r&   r   CTokenizeTest.test_tabs
  r   r(   c                    U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  S	S
5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS5        U R                  SS 5        U R                  S!S"5        U R                  S#S$5        U R                  S%S&5        U R                  S'S(5        U R                  S)S*5        g )+Nr   z    NAME       'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   a8      NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r   z    NAME       'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r   a<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r   z    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r   z    NAME       'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    r   a;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r   a<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r   z    NAME       'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r   r   r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    NAME       'await'       (3, 0) (3, 5)
    r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    NAME       'await'       (3, 0) (3, 5)
    r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    NAME       'await'       (2, 0) (2, 5)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r   a;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    NAME       'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r   ah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    r   a~      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    NAME       'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    NAME       'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    rn   ro   s    r&   r   CTokenizeTest.test_async
  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	H K 		& 	9 	< 			 	 !		F 	 "		r(   c                 (    U R                  SS5        g )Nr   u7      NAME       'Örter'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    ''            (1, 17) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    EQUAL      '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rn   ro   s    r&   r   CTokenizeTest.test_unicode  r   r(   c                     S nS H;  nU R                  US9   U R                  [        R                  X5        S S S 5        M=     g ! , (       d  f       MO  = f)Nc                 j    [        U 5      n[        [        R                  " UR                  5      5      $ r  )r   rE   r   rf  r/   )r8  
the_strings     r&   
get_tokens5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens  s(    !&)JBB:CVCVWXXr(   )z(1+2]z(1+2}z{1+2]1_1.2_1e2_1e+    u   €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0xrA  012rB  rC  rD  z'sdfsdfz'''sdfsdf''a  ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))]z6            f'__{
                x:d
            }__')r  )r  rC   r   r   )r3   r?  r  s      r&   test_invalid_syntax!CTokenizeTest.test_invalid_syntax  sN    	Y$
DJ 4(!!("5"5zH )(K$
J )(s   !A
A	c                   ^ SnS nU" US-
  5      n[        U5      m[        [        R                  " TR                  5      5      nU R                  US   R                  [        R                  5        U R                  US   R                  [        R                  5        [        USS5        U" U5      n[        U5      mU R                  [        U4S j5        U R                  [        [        USS5        g )	Nd   c                 ^    SR                  S [        U 5       5       5      nUSU -  S-   -  nU$ )Nr  c              3   2   #    U  H  nS U-  S-   v   M     g7f)  z	if True:
Nr<   ).0rL   s     r&   	<genexpr>ICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>  s     O1dQh,6r  rZ  zpass
)r@  r7  )indentsr  s     r&   generate_source6CTokenizeTest.test_max_indent.<locals>.generate_source  s2    WWOgOOFdWnx//FMr(   rB   rM   r   rS   execc                  V   > [        [        R                  " T R                  5      5      $ r  )rE   r   rf  r/   )	the_inputs   r&   <lambda>/CTokenizeTest.test_max_indent.<locals>.<lambda>   s    D9c9cdmdvdv9w4xr(   )r   rE   r   rf  r/   r0   r!   DEDENTr   r  rC   rV   )r3   	MAXINDENTr_  validrN   invalidrc  s         @r&   test_max_indentCTokenizeTest.test_max_indent  s    		
  	A.UO	h@@ASASTU(//:(*<*<=z6*!),W%	*,xygw
F	
r(   c                    S n[        S5      nU R                  US5        [        S5      nU R                  U" U5      U" U5      5        [        S5      nU R                  US5        [        S5      nU R                  U" U5      U" U5      5        [        S5      nU R                  US	5        [        S
5      nU R                  U" U5      U" U5      5        g )Nc                     [        U 5      n[        R                  " UR                  5       VV Vs/ s H  tp o2U 4PM
     snn n$ s  snn nf r  )r   r   rf  r/   )r8  r>  kindr'  s       r&   r?  ECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens&  sL    !&)JAA*BUBUVXV (:q6NVX X Xs   Az
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae      NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj      NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        )r   r7   r0   )r3   r?  r  code_no_conts       r&   #test_continuation_lines_indentation1CTokenizeTest.test_continuation_lines_indentation%  s   	X
    	D # 	,    	D):l+CD   	D # 	   
 	D):l+CD   	D # 	"    	D):l+CDr(   r<   N)r  r  r  r  r7   r  rp   r}   r   r   r   r   r   r   r   r   r   r   r   rT  rj  rq  r  r<   r(   r&   r  r  W  sj    E3(?	B,	\w	r=	~$	L	.	($	L	,	"	"H	T
	+IZ
,bEr(   r  c                       \ rS rSrS rSrg)CTokenizerBufferTestsi  c                     SSS-   SSS-   3n[         R                  " 5        n[        USU5      n[        U5        S S S 5        g ! , (       d  f       g = f)Nz"        #coding: latin-1
        #r  i'  z

        #i'  foo)r	   r  r   r
   )r3   test_scriptr  	file_names       r&   !test_newline_at_the_end_of_buffer7CTokenizerBufferTests.test_newline_at_the_end_of_buffer  sW    

e) 

e) !X#He[AII& "!!s   A
Ar<   N)r  r  r  r  ry  r  r<   r(   r&   rt  rt    s    
'r(   rt  __main__)(r>  r  r"   r   unittestior   r   textwrapr   r   r   testr   test.supportr	   test.support.script_helperr
   r   r   test.support.numbersr   r   r'   r*   r  r(  r*  r:  r\  rl  r  r  r  rC  r  r  rt  r  mainr<   r(   r&   <module>r     s   	 	       #  " Y Y W	8 W	r$A A?:x :<#IH #IL<H <.@" @"F
v28 v2r&GX &GR. Q#H Q#h2 28pEH pEf!'H-- ' zMMO r(   