Nexus File Manager
v2.0
🏠
Dashboard
⚡
System
🌐
›
opt
›
alt
›
python38
›
lib64
›
python3.8
›
test
›
__pycache__
Quick:
⬆️ Parent
🌐 Root
🏠 Home
🌍 WWW
📁 Temp
⚙️ Etc
📤 Upload
📁 Create
⚡ WordPress Admin
🔄 Refresh
✏️ test_tokenize.cpython-38.pyc
← Back
U i�f�� � @ sN d dl mZ d dlmZmZmZmZmZmZmZm Z m Z m Z m Z m ZmZmZmZ d dlmZmZ d dlZd dlmZmZ d dlmZmZ d dlZd dl Z dd� Z G d d � d e�Z G d d � d e �Z d d� Z G dd� de�Z!G dd� de�Z"G dd� de�Z#G dd� de�Z$G dd� de�Z%G dd� de�Z&G dd � d e�Z'e(d k�rJe�)� dS ) � )�support)�tokenize� _tokenize� untokenize�NUMBER�NAME�OP�STRING� ENDMARKER�ENCODING�tok_name�detect_encoding�open� Untokenizer�generate_tokens�NEWLINE)�BytesIO�StringION)�TestCase�mock)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALSc C s� g }t |�� �}|d dk}| D ]b\}}}}} |tkr: q�|rT|tkrT|d |krTq t| }|�d|d�d|d�d|� d|� �� q |S )N���z r � Z10� Z13)�len� splitlinesr r r �append) Ztoken_generatorZ source_string�resultZ num_linesZmissing_trailing_nl�type�token�start�end�line� r$ �7/opt/alt/python38/lib64/python3.8/test/test_tokenize.py� stringify_tokens_from_source s (r& c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)S )*� TokenizeTestc C s< t |�d��}tt|j�|�}| �|dg|�� �� � d S )N�utf-8z* ENCODING 'utf-8' (0, 0) (0, 0))r �encoder& r �readline� assertEqual�rstripr ��self�s�expected�fr r$ r$ r% �check_tokenize* s ��zTokenizeTest.check_tokenizec C sD t d�d��}tt|j��}| �|d jt� | �|d jt� d S )N�xr( ���r ) r r) �listr r* r+ r r r )r. r1 �tokensr$ r$ r% �test_implicit_newline4 s z"TokenizeTest.test_implicit_newlinec C sP | � dd� | � dd� d}t|�j}| �td�� t|�D ]}q<W 5 Q R X d S )Nz1 + 1z� NUMBER '1' (1, 0) (1, 1) OP '+' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) z3if False: # NL True = False # NEWLINE af NAME 'if' (1, 0) (1, 2) NAME 'False' (1, 3) (1, 8) OP ':' (1, 8) (1, 9) NEWLINE '\n' (1, 9) (1, 10) COMMENT '# NL' (2, 4) (2, 8) NL '\n' (2, 8) (2, 9) NL '\n' (3, 4) (3, 5) INDENT ' ' (4, 0) (4, 4) NAME 'True' (4, 4) (4, 8) OP '=' (4, 9) (4, 10) NAME 'False' (4, 11) (4, 16) COMMENT '# NEWLINE' (4, 17) (4, 26) NEWLINE '\n' (4, 26) (4, 27) DEDENT '' (5, 0) (5, 0) s def k(x): x += 2 x += 5 z3unindent does not match any outer indentation level)r2 r r* �assertRaisesRegex�IndentationErrorr )r. Zindent_error_filer* �tokr$ r$ r% � test_basic< s � � zTokenizeTest.test_basicc C sp | � dd� | � dd� | � dd� | � dd� | � d d � | � d d � | � d d� | � dd� | � dd� d S )Nz 0xff <= 255z� NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) z 0b10 <= 255z� NUMBER '0b10' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) z0o123 <= 0O123z� NUMBER '0o123' (1, 0) (1, 5) OP '<=' (1, 6) (1, 8) NUMBER '0O123' (1, 9) (1, 14) z1234567 > ~0x15z� NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) z2134568 != 1231515z� NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) z(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) z0xdeadbeef != -1z� NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) z0xdeadc0de & 12345z� NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) z0xFF & 0x15 | 1234z� NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) �r2 �r. r$ r$ r% �test_inta s zTokenizeTest.test_intc C s4 | � dd� | � dd� | � dd� | � dd� d S ) Nzx = 0z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) zx = 0xfffffffffffz� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xfffffffffff' (1, 4) (1, 17) zx = 123141242151251616110z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151251616110' (1, 4) (1, 25) zx = -15921590215012591z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '15921590215012591' (1, 5) (1, 22) r< r= r$ r$ r% � test_long� s zTokenizeTest.test_longc C sX | � dd� | � dd� | � dd� | � dd� | � d d � | � d d � | � d d� d S )Nz x = 3.14159z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) z x = 314159.z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) z x = .314159z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) z x = 3e14159z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) z x = 3E123z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) z x+y = 3e-1230z� NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) z x = 3.14e159z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) r< r= r$ r$ r% � test_float� s zTokenizeTest.test_floatc C sJ dd� }t D ] }d|krq | �||�|� q tD ]}| �||�|� q0d S )Nc S s< t | �d��}t|j�D ] \}}}}}|tkr| S qdS )Nr( z invalid token)r r) r r* r )r/ r1 Ztoktyper r! r" r# r$ r$ r% � number_token� s z;TokenizeTest.test_underscore_literals.<locals>.number_token�()r r+ r ZassertNotEqual)r. rA Zlitr$ r$ r% �test_underscore_literals� s z%TokenizeTest.test_underscore_literalsc C s< | � dd� | � dd� | � dd� | � dd� | � d d � | � d d � | � d d� | � dd� | � dd� | � dd� | � dd� | � dd� | � dd� | � dd � | � d d � | � d d � | � d!d"� | � d#d$� | � d%d&� | � d'd(� | � d)d*� | � d+d,� | � d-d.� | � d/d0� | � d1d2� | � d3d4� d S )5Nzx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) zx = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) z x = "doesn't "shrink", does it"z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) zx = 'abc' + 'ABC'z� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) zy = "ABC" + "ABC"z� NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a� NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) zu'abc' + U'abc'z� STRING "u'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "U'abc'" (1, 9) (1, 15) zu"abc" + U"abc"z� STRING 'u"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'U"abc"' (1, 9) (1, 15) zb'abc' + B'abc'z� STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) zb"abc" + B"abc"z� STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) z%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) z%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "rB'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Rb'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "RB'abc'" (1, 30) (1, 37) z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'rB"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) z "a\ de\ fg"z3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3) zu"a\ de"z/ STRING 'u"a\\\nde"' (1, 0) (2, 3) zrb"a\ d"z/ STRING 'rb"a\\\nd"' (1, 0) (2, 2) z """a\ b"""z0 STRING '"""a\\\nb"""' (1, 0) (2, 4) z u"""a\ b"""z1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4) zrb"""a\ b\ c"""z7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4) zf"abc"z/ STRING 'f"abc"' (1, 0) (1, 6) z fR"a{b}c"z/ STRING 'fR"a{b}c"' (1, 0) (1, 9) z f"""abc"""z0 STRING 'f"""abc"""' (1, 0) (1, 10) z f"abc\ def"z1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4) z Rf"abc\ def"z2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4) r< r= r$ r$ r% � test_string� sT ������ ��zTokenizeTest.test_stringc C s( | � dd� | � dd� | � dd� d S )Nz!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) z def d01v_(a=1, *k, **w): passa� NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) z&def d23(a: str, b: int=3) -> int: passa� NAME 'def' (1, 0) (1, 3) NAME 'd23' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ':' (1, 9) (1, 10) NAME 'str' (1, 11) (1, 14) OP ',' (1, 14) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'int' (1, 19) (1, 22) OP '=' (1, 22) (1, 23) NUMBER '3' (1, 23) (1, 24) OP ')' (1, 24) (1, 25) OP '->' (1, 26) (1, 28) NAME 'int' (1, 29) (1, 32) OP ':' (1, 32) (1, 33) NAME 'pass' (1, 34) (1, 38) r< r= r$ r$ r% � test_function� s zTokenizeTest.test_functionc C s | � dd� d S )NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa� NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) r< r= r$ r$ r% �test_comparison� s �z TokenizeTest.test_comparisonc C s | � dd� d S )Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) r< r= r$ r$ r% � test_shift� s zTokenizeTest.test_shiftc C s | � dd� d S )Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) r< r= r$ r$ r% � test_additive� s zTokenizeTest.test_additivec C s | � dd� d S )Nzx = 1//1*1/5*12%0x12@42a� NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) OP '@' (1, 20) (1, 21) NUMBER '42' (1, 21) (1, 23) r< r= r$ r$ r% �test_multiplicative s z TokenizeTest.test_multiplicativec C s | � dd� | � dd� d S )Nz~1 ^ 1 & 1 |1 ^ -1a� OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) z-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) r< r= r$ r$ r% � test_unary( s zTokenizeTest.test_unaryc C s | � dd� d S )Nz/import sys, time x = sys.modules['time'].time()a� NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) r< r= r$ r$ r% � test_selectorM s zTokenizeTest.test_selectorc C s | � dd� d S )Nz @staticmethod def foo(x,y): passa OP '@' (1, 0) (1, 1) NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) r< r= r$ r$ r% � test_methodc s zTokenizeTest.test_methodc C s | � dd� d S )Nz def f(): if x passa_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) r< r= r$ r$ r% � test_tabst s �zTokenizeTest.test_tabsc C s | � dd� d S )Nu! Örter = 'places' grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "'places'" (1, 8) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "'green'" (2, 7) (2, 14) r< r= r$ r$ r% �test_non_ascii_identifiers� s z'TokenizeTest.test_non_ascii_identifiersc C s | � dd� d S )Nu# Örter = u'places' grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "u'places'" (1, 8) (1, 17) NEWLINE '\n' (1, 17) (1, 18) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) r< r= r$ r$ r% � test_unicode� s zTokenizeTest.test_unicodec C s | � dd� | � dd� | � dd� | � dd� | � d d � | � d d � | � d d� | � dd� | � dd� | � dd� | � dd� | � dd� | � dd� | � dd � | � d d � | � d d � | � d!d"� | � d#d$� | � d%d&� | � d'd(� | � d)d*� d S )+Nz async = 1z� NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) za = (async = 1)a8 NAME 'a' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '(' (1, 4) (1, 5) NAME 'async' (1, 5) (1, 10) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) zasync()z� NAME 'async' (1, 0) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) zclass async(Bar):passa< NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP '(' (1, 11) (1, 12) NAME 'Bar' (1, 12) (1, 15) OP ')' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 17) (1, 21) zclass async:passz� NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP ':' (1, 11) (1, 12) NAME 'pass' (1, 12) (1, 16) z await = 1z� NAME 'await' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) z foo.asyncz� NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) zasync for a in b: passa; NAME 'async' (1, 0) (1, 5) NAME 'for' (1, 6) (1, 9) NAME 'a' (1, 10) (1, 11) NAME 'in' (1, 12) (1, 14) NAME 'b' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 18) (1, 22) zasync with a as b: passa< NAME 'async' (1, 0) (1, 5) NAME 'with' (1, 6) (1, 10) NAME 'a' (1, 11) (1, 12) NAME 'as' (1, 13) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'pass' (1, 19) (1, 23) z async.fooz� NAME 'async' (1, 0) (1, 5) OP '.' (1, 5) (1, 6) NAME 'foo' (1, 6) (1, 9) �asyncz/ NAME 'async' (1, 0) (1, 5) zasync #comment awaitz� NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) COMMENT '#comment' (2, 0) (2, 8) NL '\n' (2, 8) (2, 9) NAME 'await' (3, 0) (3, 5) zasync ... awaitz� NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) OP '...' (2, 0) (2, 3) NEWLINE '\n' (2, 3) (2, 4) NAME 'await' (3, 0) (3, 5) z async awaitz� NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) NAME 'await' (2, 0) (2, 5) z foo.async + 1z� NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '1' (1, 12) (1, 13) zasync def foo(): passa; NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NAME 'pass' (1, 17) (1, 21) zNasync def foo(): def foo(await): await = 1 if 1: await async += 1 a� NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\n' (3, 13) (3, 14) DEDENT '' (4, 2) (4, 2) NAME 'if' (4, 2) (4, 4) NUMBER '1' (4, 5) (4, 6) OP ':' (4, 6) (4, 7) NEWLINE '\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) NAME 'await' (5, 4) (5, 9) NEWLINE '\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) NAME 'async' (6, 0) (6, 5) OP '+=' (6, 6) (6, 8) NUMBER '1' (6, 9) (6, 10) NEWLINE '\n' (6, 10) (6, 11) z)async def foo(): async for i in 1: passa� NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) NUMBER '1' (2, 17) (2, 18) OP ':' (2, 18) (2, 19) NAME 'pass' (2, 20) (2, 24) DEDENT '' (3, 0) (3, 0) zasync def foo(async): awaitah NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) NAME 'await' (1, 22) (1, 27) z@def f(): def baz(): pass async def bar(): pass await = 2a� NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) zFasync def f(): def baz(): pass async def bar(): pass await = 2a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) OP ')' (1, 12) (1, 13) OP ':' (1, 13) (1, 14) NEWLINE '\n' (1, 14) (1, 15) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) r< r= r$ r$ r% � test_async� s: �,� �%�zTokenizeTest.test_asyncN)�__name__� __module__� __qualname__r2 r7 r; r> r? r@ rC rD rE rF rG rH rI rJ rK rL rM rN rO rQ r$ r$ r$ r% r' # s* %9( '<& % r' c @ s e Zd Zdd� ZdS )�GenerateTokensTestc C s0 t |�}tt|j�|�}| �||�� �� � d S �N)r r& r r* r+ r, r r- r$ r$ r% r2 � s z!GenerateTokensTest.check_tokenizeN)rR rS rT r2 r$ r$ r$ r% rU � s rU c C s� g }t t| �d��j�}|D ]T\}}}}}|tkrbd|krb|�tdftdftt |�ftdfg� q |� ||f� q t |�� d�S )Nr( �.�DecimalrB �)) r r r) r* r �extendr r r �reprr r �decode)r/ r �g�toknum�tokval�_r$ r$ r% �decistmt� s �ra c @ s e Zd Zdd� ZdS )�TestMiscc C sP ddl m} d}| �t|�d� | �tt|��d� | �tt|��|d�� d S )Nr )rX z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrX r+ ra Z assertRegexr[ �eval)r. rX r/ r$ r$ r% � test_decistmt� s ��zTestMisc.test_decistmtN)rR rS rT re r$ r$ r$ r% rb � s rb c @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dS )� TestTokenizerAdheresToPep0263zU Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. c C s, t j�t j�t�|�}t�| t|d�� d S )N�rb)�os�path�join�dirname�__file__� TestRoundtrip�check_roundtripr )r. �filenameri r$ r$ r% � _testFile� s z'TestTokenizerAdheresToPep0263._testFilec C s d}| � |� d S )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt�rp �r. r1 r$ r$ r% �'test_utf8_coding_cookie_and_no_utf8_bom� s zETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomc C s d}| � t| j|� dS )a� As per PEP 0263, if a file starts with a utf-8 BOM signature, the only allowed encoding for the comment is 'utf-8'. The text file used in this test starts with a BOM signature, but specifies latin1 as the coding, so verify that a SyntaxError is raised, which matches the behaviour of the interpreter when it encounters a similar condition. z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN�� assertRaises� SyntaxErrorrp rr r$ r$ r% �&test_latin1_coding_cookie_and_utf8_bom� s zDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomc C s d}| � |� d S )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrq rr r$ r$ r% �"test_no_coding_cookie_and_utf8_bom� s z@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomc C s d}| � |� d S )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrq rr r$ r$ r% �$test_utf8_coding_cookie_and_utf8_bom� s zBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomc C s$ | � t| jd� | � t| jd� d S )Nz bad_coding.pyzbad_coding2.pyrt r= r$ r$ r% �test_bad_coding_cookie s z4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN) rR rS rT �__doc__rp rs rw rx ry rz r$ r$ r$ r% rf � s rf c @ s e Zd Zdd� Zdd� ZdS )� Test_Tokenizec sP d}|� d��d� � �fdd�}tt|dd��dd� }d g}| �||d � d S ) N� "ЉЊЈЁЂ"r( Fc s � s d� �S dS d S �NT� r$ r$ ��firstr# r$ r% r* s zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline��encoding� r4 �� r} )r� r )r� � r} z bytes not decoded with encoding)r) r5 r r+ )r. �literalr* r6 �expected_tokensr$ r� r% �.test__tokenize_decodes_with_specified_encoding s �z<Test_Tokenize.test__tokenize_decodes_with_specified_encodingc sF d�d� � �fdd�}t t|d d��d d� }dg}| �||d� d S ) Nr} Fc s � s d� �S dS d S r~ r$ r$ �r� r� r$ r% r* s zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readliner� r4 r� z*string not tokenized when encoding is None)r5 r r+ )r. r* r6 r� r$ r� r% �1test__tokenize_does_not_decode_with_encoding_none s �z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rR rS rT r� r� r$ r$ r$ r% r| s r| c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)S )*�TestDetectEncodingc s d� � �fdd�}|S )Nr c s$ � t ��krt��� } � d7 � | S �Nr� )r � StopIteration)r# ��index�linesr$ r% r* 1 s z1TestDetectEncoding.get_readline.<locals>.readliner$ �r. r� r* r$ r� r% � get_readline/ s z TestDetectEncoding.get_readlinec C s> d}t | �|��\}}| �|d� | �|t|d d� �� d S )N)� # something � print(something) � do_something(else) r( � )r r� r+ r5 �r. r� r� �consumed_linesr$ r$ r% � test_no_bom_no_encoding_cookie: s z1TestDetectEncoding.test_no_bom_no_encoding_cookiec C s6 d}t | �|��\}}| �|d� | �|ddg� d S )N)s # something r� r� � utf-8-sigr� r� �r r� r+ r� r$ r$ r% �test_bom_no_cookieD s �z%TestDetectEncoding.test_bom_no_cookiec C s4 d}t | �|��\}}| �|d� | �|dg� d S )N)� # -*- coding: latin-1 -*- r� r� � iso-8859-1r� r� r� r$ r$ r% � test_cookie_first_line_no_bomO s z0TestDetectEncoding.test_cookie_first_line_no_bomc C s4 d}t | �|��\}}| �|d� | �|dg� d S )N)s # coding=utf-8 r� r� r� s # coding=utf-8 r� r� r$ r$ r% �&test_matched_bom_and_cookie_first_lineY s z9TestDetectEncoding.test_matched_bom_and_cookie_first_linec C s d}| � |�}| �tt|� d S )N)s# # vim: set fileencoding=ascii : r� r� �r� ru rv r r� r$ r$ r% �<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorc s zOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorc C s: d}t | �|��\}}| �|d� ddg}| �||� d S )N)� #! something � # vim: set fileencoding=ascii : r� r� �asciir� r� r� �r. r� r� r� r0 r$ r$ r% � test_cookie_second_line_no_boml s z1TestDetectEncoding.test_cookie_second_line_no_bomc C s6 d}t | �|��\}}| �|d� | �|ddg� d S )N)� #! something � f# coding=utf-8 r� r� r� r� r� r� r� r$ r$ r% �'test_matched_bom_and_cookie_second_linex s �z:TestDetectEncoding.test_matched_bom_and_cookie_second_linec C s d}| � |�}| �tt|� d S )N)r� r� r� r� r� r� r$ r$ r% �=test_mismatched_bom_and_cookie_second_line_raises_syntaxerror� s zPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorc C s8 d}t | �|��\}}| �|d� dg}| �||� d S )N)� print('£') �% # vim: set fileencoding=iso8859-15 : � print('€') r( r� r� r� r$ r$ r% �/test_cookie_second_line_noncommented_first_line� s zBTestDetectEncoding.test_cookie_second_line_noncommented_first_linec C s: d}t | �|��\}}| �|d� ddg}| �||� d S )N)� #print('£') r� r� � iso8859-15r� r� r� r� r$ r$ r% �,test_cookie_second_line_commented_first_line� s z?TestDetectEncoding.test_cookie_second_line_commented_first_linec C s: d}t | �|��\}}| �|d� ddg}| �||� d S )N)� r� r� r� r� r� r� r� r$ r$ r% �(test_cookie_second_line_empty_first_line� s z;TestDetectEncoding.test_cookie_second_line_empty_first_linec C sd d}|D ]V}dD ]L}|� d|�}dd|�d� d dd f}| �|�}t|�\}}| �|d � qqd S ) N)zlatin-1r� z iso-latin-1z latin-1-unixziso-8859-1-unixziso-latin-1-mac��-r` r� � #!/usr/bin/python � # coding: r� r� s print(things) s do_something += 4 r� ��replacer) r� r r+ � r. Z encodingsr� Zrep�encr� Zrl�foundr� r$ r$ r% �test_latin1_normalization� s � z,TestDetectEncoding.test_latin1_normalizationc C s d}| � |�}| �tt|� d S )N�s print("�")r� r� r$ r$ r% �test_syntaxerror_latin1� s z*TestDetectEncoding.test_syntaxerror_latin1c C sb d}|D ]T}dD ]J}|� d|�}dd|�d� d df}| �|�}t|�\}}| �|d � qqd S ) N)r( z utf-8-macz utf-8-unixr� r� r� r� r� r� s 1 + 3 r( r� r� r$ r$ r% �test_utf8_normalization� s � z*TestDetectEncoding.test_utf8_normalizationc C s� | � d�}t|�\}}| �|d� | �|dg� t| � d��\}}| �|d� | �|g � | � d�}t|�\}}| �|d� | �|dg� | � d�}t|�\}}| �|d� | �|g � | � d�}| �tt|� d S ) N)r� r( r� r$ )s print(something) r� )s )s # coding: bad )r� r r+ ru rv �r. r* r� r� r$ r$ r% �test_short_files� s" z#TestDetectEncoding.test_short_filesc C s4 | � d�}t|�\}}| �|d� | �|dg� d S )N)� print("#coding=fake")r( r� )r� r r+ r� r$ r$ r% �test_false_encoding� s z&TestDetectEncoding.test_false_encodingc C s� t jd }| �t j|� dD ]j}t|d|d��"}td| |d� td|d� W 5 Q R X t|��"}| �|j|� | �|j d� W 5 Q R X q t|dd d��}td|d� W 5 Q R X t|��"}| �|jd � | �|j d� W 5 Q R X d S ) Nz.py)z iso-8859-15r( �wr� z # coding: %s��fileu print('euro:€')�rr� ) r ZTESTFNZ addCleanup�unlinkr �print� tokenize_openr+ r� �mode)r. ro r� �fpr$ r$ r% � test_open� s z TestDetectEncoding.test_openc s~ d}d� G � fdd�d�}| � t�� |� |�}|`t|j� W 5 Q R X | �td�|��� |� |�}t|j� W 5 Q R X d S )NZsome_file_pathr� c s e Zd Zdd� Z� fdd�ZdS )z;TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc S s || _ || _d| _d S )Nr )�nameZ_lines�_index)r. r� ri r$ r$ r% �__init__ s zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__c s. | j t� �krt�� | j }| j d7 _ |S r� )r� r r� )r. r# �r� r$ r% r* s zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN)rR rS rT r� r* r$ r� r$ r% �Bunk s r� z.*{})ru rv r� r r* r8 �format)r. ri r� Zinsr$ r� r% �test_filename_in_exception s z-TestDetectEncoding.test_filename_in_exceptionc C s@ t d�}tjd|d�� | �ttd� W 5 Q R X | �|j� d S )Ns #coding:xxxztokenize._builtin_open)Z return_valueZfoobar)r r Zpatchru rv r� Z assertTrue�closed)r. �mr$ r$ r% �test_open_error# s z"TestDetectEncoding.test_open_errorN)rR rS rT r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r$ r$ r$ r% r� - s( r� c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) � TestTokenizec s� dd l }t� �d ��fdd�}�fdd�}d� � fdd�}|j}|j}||_||_z(t |�}| �t|�dd d d d d g� W 5 ||_||_X | ���� d S )Nr c s � ddgfS )N� first� secondr$ )r* r� r$ r% �mock_detect_encoding1 s z8TestTokenize.test_tokenize.<locals>.mock_detect_encodingc s" |� g }| � }|r |� |� q|S rV )r )r* r� �outZ next_line)� encoding_usedr$ r% �mock__tokenize4 s z2TestTokenize.test_tokenize.<locals>.mock__tokenizec s � d7 � � dkrdS t � ��� S )Nr� � r )�strr) r$ )�counterr$ r% � mock_readline@ s z1TestTokenize.test_tokenize.<locals>.mock_readliner� r� � 1� 2� 3� 4)r �objectr r r+ r5 )r. Ztokenize_moduler� r� r� Zorig_detect_encodingZorig__tokenizeZresultsr$ )r� r� r� r% � test_tokenize- s& �zTestTokenize.test_tokenizec C sf g }t d�D ]}|�dj|d�� q |�d� d�|�}ttt|�d��j��}| � |d j d� d S )Ni� zdef i{i}(): return {i})�iZOK� r( ���) �ranger r� rj r5 r r r) r* r+ �string)r. Zbufr� Ztoksr$ r$ r% �test_oneline_defsU s z TestTokenize.test_oneline_defsc G s� t tt|�d��j��}t|�}| �t|�d| � | �t|d j tt � t |�D ]&}| �t||d j t|| � qV| �t|d| j tt j � | �t|d| j tt j � d S )Nr( r� r r� r� )r5 r r r) r* r r+ r � exact_typer r� r r r )r. ZopstrZoptypesr6 Z num_optypesr� r$ r$ r% �assertExactTypeEquala s � ���z!TestTokenize.assertExactTypeEqualc C s� | � dtjtj� | � dtjtj� | � dtj� | � dtj� | � dtj� | � dtj � | � dtj � | � dtj � | � d tj � | � d tj � | � d tj� | � d tj� | � d tj� | � dtj� | � dtj� | � dtj� | � dtjtj� | � dtj� | � dtj� | � dtj� | � dtj� | � dtj� | � dtj� | � dtj � | � dtj � | � dtj � | � dtj � | � d tj � | � d tj!� | � d tj"� | � d tj#� | � d tj$� | � d!tj%� | � d"tj&� | � d"tj&� | � d#tj'� | � d$tj(� | � d%tj)� | � d&tj*� | � d'tj+� | � d(tj,� | � d)tj-� | � d*tj.� | � d+tj/� | � d,tj0� | � d-t1tj t2tj t1tj t2tjt1tj t2� | � d.tjtj2tjtj2tjtj2tj� | � d/tjtjtj1tjtj2tj� d S )0Nz()z[]�:�,�;�+r� �*�/�|�&�<�>�=rW �%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->�@z@=za**2+b**2==c**2z {1, 2, 3}z ^(x & 0x1))3r� r �LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE� CIRCUMFLEX� LEFTSHIFT� RIGHTSHIFT� DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL� SLASHEQUAL� PERCENTEQUAL� AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL� COLONEQUAL�ELLIPSIS�RARROW�AT�ATEQUALr r r= r$ r$ r% �test_exact_typeo s� � � �z TestTokenize.test_exact_typec C s | � dtj� d S )Nz @ )r� r r1 r= r$ r$ r% �%test_pathological_trailing_whitespace� s z2TestTokenize.test_pathological_trailing_whitespaceN)rR rS rT r� r� r� r3 r4 r$ r$ r$ r% r� + s ( Ar� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�UntokenizeTestc C sZ t � }d|_d|_| �t��}|�d� W 5 Q R X | �|jjd d� | �t|jd� d S )Nr� )r� r� r z'start (1,3) precedes previous end (2,2))r� r� ) r �prev_row�prev_colru � ValueError�add_whitespacer+ Z exception�args)r. �u�cmr$ r$ r% �test_bad_input_order� s �z#UntokenizeTest.test_bad_input_orderc C sf t � }d|_d|_g |_|�d� | �|jdg� d|_|�d� | �|jdddg� t�| d� d S ) Nr� )r� r z\ r� )� r> z\ \ r za b c \ c )r r6 r7 r6 r9 r+ rm rn )r. r; r$ r$ r% �test_backslash_continuation� s z*UntokenizeTest.test_backslash_continuationc C s� t � }tdf}tdf|g}|�|tg �� | �|jdg� t � }| �|�t|g��d� t � }| �|�t|��d� | �|jd� | �tt|��d� d S )NZHellor( zHello s Hello ) r r r �compat�iterr+ r6 r r� )r. r; r r6 r$ r$ r% �test_iter_compat� s z UntokenizeTest.test_iter_compatN)rR rS rT r= r? rB r$ r$ r$ r% r5 � s r5 c @ sL e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )rm c C s� t |t�r|�d�}n|�� }|�� t|jdd��j}tt |��}dd� |D �}t |�}t|jdd��j}dd� t |�D �}| � ||� t |�} t| jdd��j} dd� t | �D �} | � | |� dS ) a) Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized to both 5- and 2-tuples. Both sequences are converted back to source code via tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails if the 3 pair tokenizations do not match. When untokenize bugs are fixed, untokenize with 5-tuples should reproduce code that does not contain a backslash continuation following spaces. A proper test should test this. r( T)�keependsc S s g | ]}|d d� �qS �Nr� r$ ��.0r: r$ r$ r% � <listcomp>� s z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>c S s g | ]}|d d� �qS rD r$ rE r$ r$ r% rG � s c S s g | ]}|d d� �qS rD r$ rE r$ r$ r% rG � s N) � isinstancer� r) �read�closerA r �__next__r5 r r r+ ) r. r1 �coder* Ztokens5Ztokens2Z bytes_from2Z readline2Z tokens2_from2Z bytes_from5Z readline5Z tokens2_from5r$ r$ r% rn � s z TestRoundtrip.check_roundtripc C st | � d� | � d� | � d� t�d�}t|d��}| � |� W 5 Q R X | � d� | � d� | � d� | � d � d S ) Nzif x == 1: print(x) z # This is a comment # This also zif x == 1 : print(x) �tokenize_tests.txtrg zoif x == 1: # A comment by itself. print(x) # Comment here, too. # Another comment. after_if = True zPif (x # The comments need to go in the right place == 1): print('x==1') z�class Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent zttry: import somemodule except ImportError: # comment print('Can not import' # comment2 )else: print('Loaded') )rn r �findfiler )r. �fnr1 r$ r$ r% �test_roundtrip s z TestRoundtrip.test_roundtripc C s | � d� d S )Nzna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] �rn r= r$ r$ r% �test_continuation' s z TestRoundtrip.test_continuationc C s | � d� | � d� d S )Nz)x=1+\ 1 # This is a comment\ # This also z# Comment \ x = 0rQ r= r$ r$ r% r? 4 s z)TestRoundtrip.test_backslash_continuationc C s | � d� d S )Nz'' ''rQ r= r$ r$ r% �test_string_concatenation= s z'TestRoundtrip.test_string_concatenationc C s� dd l }dd l}t�d�}tj�|�p*tj}|� tj�|� |�d��}|� tj�|d�� dD ] }|� tj�|d�| � q^t� d�s�|� |d�}|D ]P}tj d kr�td |� t|d ��(}| j|d �� | �|� W 5 Q R X W 5 Q R X q�d S ) Nr rM ztest*.pyztest_unicode_identifiers.py)�bufferZbuiltinZfileio�inspectrh �platform�sysz test_%s.pyZcpu� r� r rg r� )�glob�randomr rN rh ri rk �curdirrj �escape�removeZis_resource_enabledZsample�verboser� r ZsubTestrn )r. rY rZ rO ZtempdirZ testfilesr1 Ztestfiler$ r$ r% �test_random_filesA s z TestRoundtrip.test_random_filesc C s, t |t�r|�d�}ttt|�j���d�S )Nr( )rH r� r) r r r r* r\ )r. rL r$ r$ r% � roundtrip^ s zTestRoundtrip.roundtripc C s6 d}| � |��d�}| �|d |d � | �|� dS )z� Ensure that although whitespace might be mutated in a roundtrip, the semantic meaning of the indentation remains consistent. zif False: x=3 x=3 r� r� r� N)r` �splitr+ rn )r. rL Z codelinesr$ r$ r% �#test_indentation_semantics_retainedc s z1TestRoundtrip.test_indentation_semantics_retainedN) rR rS rT rn rP rR r? rS r_ r` rb r$ r$ r$ r% rm � s ' rm �__main__)*Ztestr r r r r r r r r r r r r r� r r r �ior r Zunittestr r Ztest.test_grammarr r rh r r& r' rU ra rb rf r| r� r� r5 rm rR �mainr$ r$ r$ r% �<module> s> D %& )
💻
System Info
Current path
/opt/alt/python38/lib64/python3.8/test/__pycache__
Contents
0 folders, 1461 files
Disk free
99.54 GB
PHP version
8.3.30
🚀
Quick Actions
📍 Script location
🌐 Web root
🗑️ Temp
📁 /opt/alt/python38/lib64/python3.8/test/__pycache__
⚡ Nexus File Manager • 1461 items