
    \j                       d Z ddlmZ ddlmZmZ ddlZddlZddlm	Z	m
Z
  G d de      Z G d d	e      Z G d
 de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d dej*                        Ze G d d             Z G d d      Ze G d d             Zd d d!	 	 	 	 	 	 	 d$d"Zd%d#Zy)&zTpytokens - A Fast, spec compliant Python 3.12+ tokenizer that runs on older Pythons.    )annotations)	dataclassfieldN)ClassVarIteratorc                      e Zd Zy)TokenizeErrorN__name__
__module____qualname__     ;/root/env/lib/python3.12/site-packages/pytokens/__init__.pyr	   r	          r   r	   c                      e Zd Zy)IndentationErrorNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)InconsistentUseOfTabsAndSpacesNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy) DedentDoesNotMatchAnyOuterIndentNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)UnterminatedStringNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)UnexpectedEOFNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)!UnexpectedCharacterAfterBackslashNr
   r   r   r   r   r      r   r   r   c                      e Zd Zy)NotAnIndentNr
   r   r   r   r   r       r   r   r   c                      e Zd Zy)	UnderflowNr
   r   r   r   r!   r!   #   r   r   r!   c                      e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZd!dZd!dZ d"dZ!y )#	TokenType                        	   
                                                         c                     d| j                    S )Nz
TokenType.)nameselfs    r   __repr__zTokenType.__repr__I   s    DII;''r   c                x    | j                   dk(  ry| j                         ry| j                   j                         S )N
identifierNAMEOP)rA   is_operatorupperrB   s    r   to_python_tokenzTokenType.to_python_tokenL   s1    99$yy  r   c                Z    t         j                  | cxk  xr t         j                  k  S c S N)r#   	_op_start_op_endrB   s    r   rI   zTokenType.is_operatorU   s$    ""T=I,=,=====r   NreturnstrrQ   bool)"r   r   r   
whitespaceindentdedentnewlinenlcommentrN   	semicolonlparenrparenlbracketrbracketlbracerbracecolonoprO   rF   numberstringfstring_startfstring_middlefstring_endtstring_starttstring_middletstring_end	endmarker
errortokenrD   rK   rI   r   r   r   r#   r#   &   s    JFFG	
BGIIFFHHFFE	BGJFFMNKMNKIJ(!>r   r#   c                  \    e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   ded	<   dd
Zy)Tokenr#   typeintstart_index	end_index
start_line	start_colend_lineend_colc                   | j                   t        j                  k(  s| j                   t        j                  k(  r4| j                  t        |      k(  r| j                  t        |      dz   k(  ry| j                   t        j                  k(  r7| j                  t        |      dz   k(  r| j                  t        |      dz   k(  ry| j                   t        j                  k(  ry|| j                  | j                   S )Nr$    )	rp   r#   rX   rY   rr   lenrs   rW   rl   )rC   sources     r   to_byte_slicezToken.to_byte_slicee   s     YY)+++tyyILL/H  CK/#f+/1 II)))  CK!O3#f+/1 99	+++d&&88r   N)r{   rR   rQ   rR   )r   r   r   __annotations__r|   r   r   r   ro   ro   Y   s+    
ONONML9r   ro   c                      e Zd ZU dZded<   dZded<   dZded<   dZded	<   d
Zded<   dZ	ded<   ddZ
ddZddZddZddZddZddZddZy)FStringStater$   zClassVar[int]not_fstringr%   at_fstring_middler&   at_fstring_lbracer'   in_fstring_exprr(   in_fstring_expr_modifierr)   at_fstring_endc                <    t         j                  | _        g | _        y rM   )r   r   statestackrB   s    r   __init__zFStringState.__init__   s    !--
 "
r   c                x    | j                   j                  | j                         t        j                  | _        y rM   )r   appendr   r   r   rB   s    r   enter_fstringzFStringState.enter_fstring   s%    

$**%!33
r   c                    | j                   t        j                  k(  sJ | j                  j	                         | _         y rM   )r   r   r   r   poprB   s    r   leave_fstringzFStringState.leave_fstring   s,    zz\88888ZZ^^%
r   c                    | j                   t        j                  k(  r%| j                  j	                  | j                          t        j
                  | _         y rM   )r   r   r   r   r   r   rB   s    r   !consume_fstring_middle_for_lbracez.FStringState.consume_fstring_middle_for_lbrace   s7    ::>>>JJdjj)!33
r   c                .    t         j                  | _        y rM   )r   r   r   rB   s    r   consume_fstring_middle_for_endz+FStringState.consume_fstring_middle_for_end   s    !00
r   c                .    t         j                  | _        y rM   )r   r   r   rB   s    r   consume_lbracezFStringState.consume_lbrace   s    !11
r   c                V   | j                   t        j                  k(  s| j                   t        j                  k(  sJ t	        | j
                        dkD  r@| j
                  d   t        j                  k(  r | j
                  j                         | _         y t        j                  | _         y )Nr   )r   r   r   r   rz   r   r   r   rB   s    r   consume_rbracezFStringState.consume_rbrace   sx    JJ,666zz\BBB	
C
 

Oa

2,"G"GG)DJ%77DJr   c                l    | j                   t        j                  k(  sJ t        j                  | _         y rM   )r   r   r   r   rB   s    r   consume_colonzFStringState.consume_colon   s'    zz\99999!::
r   NrQ   None)r   r   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   }   si    !"K"'(}('(}(%&O]&./m/$%NM%#4&4128;r   r   c                  P   e Zd ZU ded<   ded<   dZded<   dZded<   d	Zded
<   d	Zded<   dZded<   dZ	ded<   dZ
ded<   dZded<    ee      Zded<   dZded<    ee      Zded<   dZded<    ee      Zded<    ee      Zded<   dZded<   dZded <   d!Zded"<   d!Zded#<   d!Zded$<   d!Zded%<   dBd&ZdCd'ZdCd(ZdDd)ZdEd*ZdDd+Z dDd,Z!d!d-dFd.Z"dGd/Z#dHd0Z$dDd1Z%dId2Z&dId3Z'dId4Z(dId5Z)dId6Z*dId7Z+dJd8Z,dKd9Z-dId:Z.dId;Z/dId<Z0dBd=Z1dBd>Z2dId?Z3dLd@Z4dIdAZ5y)MTokenIteratorrR   r{   rT   issue_128233_handlingr   rq   current_index
prev_indexr$   line_numberprev_line_numberbyte_offsetprev_byte_offsetTall_whitespace_on_this_linebracket_level)default_factoryz	list[int]bracket_level_stackNzToken | None
prev_tokenz	list[str]indent_stackdedent_counterr   fstring_statezlist[tuple[str, str]]fstring_prefix_quote_stackz
str | Nonefstring_prefixfstring_quoteFweird_op_caseweird_op_case_nlweird_whitespace_caseline_after_escaped_nlc                F    | j                   t        | j                        k  S rM   r   rz   r{   rB   s    r   is_in_boundszTokenIterator.is_in_bounds   s    !!C$444r   c                X    | j                         sJ | j                  | j                     S rM   )r   r{   r   rB   s    r   peekzTokenIterator.peek   s)      """{{4--..r   c                    | j                   dz   t        | j                        k  sJ | j                  | j                   dz      S Nr$   r   rB   s    r   	peek_nextzTokenIterator.peek_next   s>    !!A%DKK(8888{{4--122r   c                X    | xj                   dz  c_         | xj                  dz  c_        y r   r   r   rB   s    r   advancezTokenIterator.advance   s$    aAr   c                X    | xj                   |z  c_         | xj                  |z  c_        y rM   r   )rC   counts     r   
advance_byzTokenIterator.advance_by   s$    e#E!r   c                X    | xj                   dz  c_         d| _        d| _        d| _        y )Nr$   r   TF)r   r   r   r   rB   s    r   	next_linezTokenIterator.next_line   s+    A+/(%*"r   c                    | j                   | j                     dk(  r&| xj                  dz  c_        | j                          y | j                          y )N
r$   )r{   r   r   r   rB   s    r   advance_check_newlinez#TokenIterator.advance_check_newline   s=    ;;t))*d2!#NNLLNr   ignore_casec                   |D ]  }| j                   t        |      z   t        | j                        kD  r2| j                  | j                   | j                   t        |      z    }|r |j                         }|j                         }||k(  s y y)NTF)r   rz   r{   lower)rC   r   optionsoptionsnippets        r   matchzTokenIterator.match   s     		F!!CK/#dkk2BBkk$"4"4t7I7ICPVK7WXG!--/ 		 r   c           	        | j                   yd| j                   v rk|t        j                  k(  rt        j                  }nG|t        j                  k(  rt        j
                  }n#|t        j                  k(  rt        j                  }| j                  rB|j                         s2|t        j                  t        j                  fvrt        j                  n|}| j                  r6| j                  | j                  | j                   }|dk(  rd| _        d| _        t#        || j                  | j                  | j$                  | j&                  | j(                  | j*                        }|t        j,                  k(  s|t        j.                  k(  r| j1                          n.|t        j2                  k(  s|t        j4                  k(  rnd| _        || _        | j                  | _        | j(                  | _        | j*                  | _        d| _        |S )Ntz F)rp   rr   rs   rt   ru   rv   rw   )r   r#   rf   ri   rg   rj   rh   rk   r   rI   rd   re   rc   r{   r   r   r   ro   r   r   r   r   rX   rY   r   rU   rZ   r   )rC   tok_type
token_type	token_strtokens        r   
make_tokenzTokenIterator.make_token  s   *sd6I6I/I9222$22Y555$33Y222$00 !!((*!1!193C3C DD LL  	  DOOd6H6HIIE!380!&D((,,++%%$$
 y(((H	,DNN---Y=N=N1N/4D,,, $ 0 0 $ 0 0"r   c                    | j                   ?| j                  J | j                  j                  | j                   | j                  f       || _         || _        y rM   )r   r   r   r   )rC   prefixquotes      r   push_fstring_prefix_quotez'TokenIterator.push_fstring_prefix_quote8  sZ    *%%111++22$$d&8&89 %"r   c                    | j                   | j                  J t        t        | j                        dk(  rdn| j                  j                         \  | _         | _        y )Nr   )NN)r   r   r!   rz   r   r   rB   s    r   pop_fstring_quotezTokenIterator.pop_fstring_quoteB  s`    &%%---O 4223q8 00446 	0T/r   c                   | j                         r,| j                  | j                     dk(  r| j                          | j                          | j                  sB| j
                  dkD  s3| j                  j                  t        j                  k(  s| j                  rt        j                  nt        j                  }| j                  |      }d| _        |S )Nr   F)r   r{   r   r   r   r   r   r   r   r   r   r#   rY   rX   r   )rC   r   r   s      r   rX   zTokenIterator.newlineM  s    4;;t/A/A#Bd#JLLN %%%%)%%++|/K/KK33 LL "" 	 
+ %r   c                   | j                   dk7  rt        t        | j                        dkD  r9| j                  j	                         }| j                  t        j                        S | j                  t        j                        S )Nr   )	r   r   rz   r   r   r   r#   rW   rl   )rC   _s     r   rl   zTokenIterator.endmarker_  sg    "t  !A%!!%%'A??9#3#344y2233r   c                4   d}| j                   | j                     j                         rd}| j                          | j	                         r| j                   | j                     j                         s| j                   | j                     dk(  re| j                          | j	                         rE| j                   | j                     j                         rH| j                   | j                     dk(  re| j	                         r,| j                   | j                     dk(  r| j                          | j	                         r| j                   | j                     j                         sF| j                   | j                     dk(  r| j                   | j                  dz
     j                         r| j                          | j	                         ro| j                   | j                     j                         rH| j                   | j                     dk(  r+| j                   | j                  dz
     j                         r| j                  dz   t        | j                         k  r<|s+| j                   | j                  dz
     j                         r| j                   | j                     dk(  s| j                   | j                     dk(  r| j                   | j                  dz      j                         s| j                  dz   t        | j                         k  r| j                   | j                  dz      d	k(  s| j                   | j                  dz      d
k(  rJ| j                   | j                  dz      j                         r | j                          | j                          | j	                         r | j                   | j                     j                         sH|s*| j                   | j                  dz
     j                         r| j                   | j                     dk(  r| j                          | j	                         rq| j                   | j                     j                         rH|s*| j                   | j                  dz
     j                         r| j                   | j                     dk(  r| j	                         rt|s*| j                   | j                  dz
     j                         rH| j                   | j                     dk(  s| j                   | j                     dk(  r| j                          | j                  | j                  z
  dk(  r| j                   | j                  dz
     dk(  r| j                  dz   t        | j                         k  rI| j                   | j                  | j                  dz    dk(  r | j                          | j                          | j                  t        j                        S | j                  t        j                        S )NFTr   .r$   eEr%   +-jJz..)r{   r   isdigitr   r   rz   r   r   r#   rc   rd   )rC   digit_before_decimals     r   decimalzTokenIterator.decimali  s   $;;t))*224#' LLN !KK**+335{{4--.#5LLN	 !KK**+335{{4--.#5 4;;t/A/A#Bc#ILLN!KK**+335D../36KK 2 2Q 67??A LLN !KK**+335D../36KK 2 2Q 67??A
 !C$44!T[[1C1Ca1G%H%P%P%RD../36;;t112c9 D..23;;=&&*S-==D$6$6$:;sB;;t'9'9A'=>#ED$6$6$:;CCE LLNLLN !KK**+335%T5G5G!5K)L)T)T)VKK 2 23s: LLN !KK**+335%T5G5G!5K)L)T)T)VKK 2 23s: !T[[1C1Ca1G%H%P%P%RD../36;;t112c9 LLN 0A5D..23s: ""Q&#dkk*::KK 2 2T5G5G!5KLPTT??9<<00y//00r   c                   | j                          | j                          | j                         r| j                  | j                     dk(  s8| j                  | j                     dk(  s| j                  | j                     dk(  rw| j                          | j                         rW| j                  | j                     dk(  r=| j                  | j                     dk(  rZ| j                  | j                     dk(  rw| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     dk(  s8| j                  | j                     dk(  s| j                  | j                     dk(  rw| j                          | j                         rW| j                  | j                     dk(  r=| j                  | j                     dk(  rZ| j                  | j                     dk(  rw| j	                  t
        j                        S )N01r   r   r   r   r   r   r{   r   r   r#   rd   rB   s    r   binaryzTokenIterator.binary  s   !KK**+s2{{4--.#5{{4--.#5LLN !KK**+s2{{4--.#5{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+s2{{4--.#5{{4--.#5LLN !KK**+s2{{4--.#5{{4--.#5 y//00r   c                   | j                          | j                          | j                         r| j                  | j                     dk\  r| j                  | j                     dk  s| j                  | j                     dk(  rv| j                          | j                         rV| j                  | j                     dk\  r| j                  | j                     dk  rY| j                  | j                     dk(  rv| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     dk\  r| j                  | j                     dk  s| j                  | j                     dk(  rv| j                          | j                         rV| j                  | j                     dk\  r| j                  | j                     dk  rY| j                  | j                     dk(  rv| j	                  t
        j                        S )Nr   7r   r   r   r   r   rB   s    r   octalzTokenIterator.octal  s   !KK**+s2D../36{{4--.#5LLN !KK**+s2D../36{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+s2D../36{{4--.#5LLN !KK**+s2D../36{{4--.#5 y//00r   c                   | j                          | j                          | j                         r| j                  | j                     t        j
                  v s| j                  | j                     dk(  rg| j                          | j                         rG| j                  | j                     t        j
                  v rJ| j                  | j                     dk(  rg| j                         r| j                  | j                     dk(  s| j                  | j                     dk(  rL| j                          | j                         r,| j                  | j                     dk(  r| j                          | j                         r| j                  | j                     t        j
                  v s| j                  | j                     dk(  rg| j                          | j                         rG| j                  | j                     t        j
                  v rJ| j                  | j                     dk(  rg| j                  t        j                        S )Nr   r   r   r   )	r   r   r{   r   re   	hexdigitsr   r#   rd   rB   s    r   hexadecimalzTokenIterator.hexadecimal  s   !KK**+v/?/??{{4--.#5LLN	 !KK**+v/?/??{{4--.#5 KK**+s2{{4--.#5LLN  "t{{43E3E'F#'M!KK**+v/?/??{{4--.#5LLN	 !KK**+v/?/??{{4--.#5 y//00r   c                    t        d      D ]9  }| j                  | j                  |z      }|dk(  s|dk(  s*| j                  |z   c S  t        d      )Nr&   "'zQuote not found somehow)ranger{   r   AssertionError)rC   offsetchars      r   find_opening_quotez TokenIterator.find_opening_quote  sZ    Ah 	3F;;t11F:;Ds{dck))F22	3
 677r   c                L   | j                         }| j                  | j                  | }| j                  |   }|dz   t        | j                        k  r<| j                  |dz      |k(  r'| j                  |dz      |k(  r| j                  ||dz    n| j                  ||dz    }||fS )Nr&   r$   r%   )r   r{   r   rz   )rC   quote_indexr   
quote_charr   s        r   string_prefix_and_quotesz&TokenIterator.string_prefix_and_quotes  s    --/T//+>[[-
 a3t{{#33KKa0J>KKa0J>	 KKkAo6 [;?; 	 u}r   c                   | j                   j                  t        j                  t        j                  fv r| j                         \  }}| j                  ||       t        t        |            D ]  }| j                           t        t        |            D ]  }| j                           | j                   j                          | j                  t        j                        S | j                   j                  t        j                  k(  r| j                  J t        | j                        dk(  }| j                   }| j#                         rM| j$                  | j                      }|dk(  r|rt&        |dk(  r| j                          | j(                  J d| j(                  j+                         vrk| j                   dz   t        | j$                        k  rF| j-                         dk(  r3| j/                         dk(  r | j                          | j                          | j#                         r6| j-                         dk(  s#| j-                         dk(  s| j1                          )|dk(  r| j/                         dk(  r"| j                          | j                          c| j                   j3                          | j                   |k(  r| j5                         S | j                  t        j6                        S | j                  J | j9                  | j                        rX| j                   j;                          | j                   |k(  r| j5                         S | j                  t        j6                        S | j1                          | j#                         rMt<        | j                   j                  t        j>                  k(  ru| j                          | j@                  jC                  | jD                         d| _"        | j                   jG                          | j                  t        jH                        S | j                   j                  t        jJ                  k(  r| j                  J t        t        | j                              D ]  }| j                           | j                  t        jL                        }| jO                          | j                   jQ                          |S | j                   j                  t        jR                  k(  rN| j                   }| j#                         r+| j$                  | j                      }| j                  J |dk(  s|dk(  rt        | j                        dk(  r}|dk(  r| j                   j3                          nt        j                  | j                   _        | j                   |k(  r| j5                         S | j                  t        j6                        S |dk(  r>t        j                  | j                   _        | j                  t        j6                        S | j1                          | j#                         r+t<        tU        d	      )
Nr$   r   \rN{}r   zUnhandled f-string state)+r   r   r   r   r   r  r   r   rz   r   r   r   r#   rf   r   r   r   r   r{   r   r   r   r   r   r   r   fstringrg   r   r   r   r   r   r   r   r   r`   r   rh   r   r   r   r   )rC   r   r   r   is_single_quoterr   r   r   s           r   r	  zTokenIterator.fstring"  s   ##$$(((
 
 !99;MFE**6593v;' 3u:& ,,.??9#:#:;;##|'E'EE%%111!$"4"45:O,,K##%{{4#5#564<O,, 4<LLN  ..:::4#6#6#<#<#>> ..2S5EE IIK3. NN,3((*		s*diikS.@224 3;~~'3. **LLN--<#'<<>1#y/G/GHH))555::d001&&EEG))[8#||~-??9+C+CDD**,i ##%l  ##|'E'EELLN$$++D,>,>?!"D--/??9#3#344##|'B'BB%%1113t1123 OOI$9$9:E""$,,.L##|'L'LL,,K##%{{4#5#56))555DLDCKS9K9K5LPQ5Qs{**LLN 4@3O3O**0 ))[8#||~-??9+C+CDDS[/;/K/KD&&,??9+C+CDD**,' ##%*  788r   c                V   | j                         \  }}|r<| j                  r0| j                          | j                  t        j
                        S |D ]  }|dv s| j                         c S  t        t        |            D ]  }| j                           t        t        |            D ]  }| j                           t        |      dk(  }| j                         r| j                  | j                     }|dk(  r|rt        |dk(  r!| j                          | j                          \| j                  |      rHt        t        |            D ]  }| j                           | j                  t        j                        S | j                          | j                         rt         )N)r   )fFr   Tr$   r   r  )r  r   r   r   r#   rc   r	  r   rz   r   r{   r   r   r   r   re   r   )rC   r   r   r   r   r
  s         r   re   zTokenIterator.string  sj   557d((LLN??ILL?99 	&D++||~%	& s6{# 	ALLN	s5z" 	ALLN	 e*/!;;t112Dt|(( t|**, zz% s5z* #ALLN#y'7'788&&(% !( r   c                   | j                   }d}d}| j                         rZ| j                  | j                      }| j                         r| j	                          d}|dk(  s|dk(  rd}nn| j                         rZ| j                         s4| j                   |k(  rt
        | j                  t        j                        S |r(|s&d| _	        | j                  t        j                        S | j                         }|dk(  s|dk(  s| j                         r| j                  t        j                        S | j                  || j                    }t        | j                        dk(  rdn| j                  d	   }t        |      t        |      k(  r>t        |      dk(  rt
        ||k7  rt        | j                  t        j                        S t        |      t        |      kD  rRt        |      dkD  r
||vrt        | j                  j                  |       | j                  t        j                         S | j"                  rt
        t        | j                        dkD  r| j                  d	   }t        |      t        |      k  rt$        t        |      t        |      k(  rnH| j                  j'                         }	| xj(                  d
z  c_        t        | j                        dkD  r| j                  t        j                        S )NFT 	#r  r   ry   r   r$   )r   r   r{   is_whitespacer   r   r   r#   rU   r   r   
is_newlinerz   r   r   r   rV   r   r   r   r   )
rC   rr   saw_whitespacesaw_tab_or_spacer   	next_char
new_indentcurrent_indent
top_indentr   s
             r   rV   zTokenIterator.indent  s   (( !;;t112D!!#!%3;$$,'+$ !   "!![0!!??9#7#788 "2)-D&??9#7#788 IIK	yD0DOO4E??9#7#788[[t/A/AB
"4#4#45:@Q@QRT@Uz?c.11:!#!!^+44??9#7#788_s>22>"Q&>+K44$$Z0??9#3#344''d''(1,!..r2
z?S_4::z?c*o5%%))+##q(# d''(1, ??9#7#788r   c                    | j                         ry| j                  | j                     }|dk(  xs |dk(  xs |dk(  xs |dk(  xs |dk(  S )NFr  r   r  )r  r{   r   )rC   r   s     r   r  zTokenIterator.is_whitespace   sc    ??{{4--.CK t|t| v~ v~	
r   c                    | j                   | j                     dk(  ry| j                   | j                     dk(  rE| j                  dz   t        | j                         k  r | j                   | j                  dz      dk(  ryy)Nr   Tr   r$   F)r{   r   rz   rB   s    r   r  zTokenIterator.is_newline  sq    ;;t))*d2KK**+t3""Q&T[[)99D..23t;r   c                   | j                   r/| j                          | j                  t        j                        S | j
                  }| j                  }t        |      }||k  r4||   }t        |      dk  r|j                         s|dk7  rn|dz  }||k  r4| j                  || j                  z
         | j                  t        j                        S )N   r   r$   )r   r   r   r#   rF   r{   r   rz   ordisalnumr   )rC   r{   indexendr   s        r   rA   zTokenIterator.name  s    LLN??9#7#788 ""&kck%=D4y3t||~$#+QJE	 ck 	 2 223y3344r   c                    | S rM   r   rB   s    r   __iter__zTokenIterator.__iter__,  s    r   c                   | j                   -| j                   j                  t        j                  k(  rt        | j
                  t        | j                        k(  r| j                   | j                         S | j                   j                  t        j                  t        j                  t        j                  hv r| j                         S | j                         S | j
                  t        | j                        kD  r| j                         S | j                  j                  t        j                  k7  r7| j                  j                  t        j                  k7  r| j!                         S | j                  | j
                     }| j"                  s|dk(  r| j%                          | j'                         s| j                         S | j                  | j
                     }|dk7  rAd| _        | j                   .| j                   j                  t        j*                  k(  rd| _        |dk(  r| j(                  r/| j%                          | j/                  t        j*                        S | j'                         rj| j1                         sZ| j"                  s| j                  | j
                     dk(  rn1| j%                          | j'                         r| j1                         sZ| j/                  t        j*                        S | j2                  dkD  r@| xj2                  dz  c_        | j4                  s| j/                  t        j                        S | j1                         r| j                         S |dk(  r&| j%                          | j'                         st6        d}d}| j'                         r| j9                         r| j%                          d}n|s| j1                         rw| j                  | j
                     }|dk(  r| j%                          | j%                          d}d}| j:                  r| j=                          n*| j=                          d| _        nn| j'                         r|st>        d| _        | j/                  t        j@                        S | jB                  dk(  s| jD                  rR| jF                  dk(  rC| j                  j                  t        j                  k(  rd| _"        	 | jI                         }||S | j9                         rp| j'                         rA| j9                         r1| j%                          | j'                         r| j9                         r1| j/                  t        j@                        S |d	v rR| j%                          | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dk(  r| j%                          | jM                         dk(  r/| j%                          | j/                  t        jN                        S | jM                         dk(  r| j%                          | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dk(  ru| j%                          | jM                         dk(  r| j%                          | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dk(  ru| j%                          | jM                         dk(  r| j%                          | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dk(  ru| j%                          | jM                         dk(  r| j%                          | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dk(  r| j%                          | jM                         dk(  r/| j%                          | j/                  t        jN                        S | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dv r/| j%                          | j/                  t        jN                        S |dk(  r/| j%                          | j/                  t        jN                        S |dk(  rD| j%                          | xjF                  dz  c_#        | j/                  t        jP                        S |dk(  rZ| j%                          | xjF                  dz  c_#        | jF                  dk  rd| _#        | j/                  t        jR                        S |dk(  rD| j%                          | xjF                  dz  c_#        | j/                  t        jT                        S |dk(  rZ| j%                          | xjF                  dz  c_#        | jF                  dk  rd| _#        | j/                  t        jV                        S |dk(  rD| j%                          | xjF                  dz  c_#        | j/                  t        jX                        S |dk(  r| j%                          | jF                  dk(  ra| j                  j                  t        j                  k(  r:| j                  j[                          | j\                  j_                         | _#        n+| xjF                  dz  c_#        | jF                  dk  rd| _#        | j/                  t        j`                        S |dk(  r| j%                          | jF                  dk(  re| j                  j                  t        j                  k(  r>t        jb                  | j                  _        | j/                  t        jN                        S | jM                         d
k(  r| j%                          | j/                  t        jN                        S |dv r'| j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| je                         S | j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| jg                         S | j
                  dz   t        | j                        k  r8| j                  | j
                  | j
                  dz    dv r| ji                         S | jk                         S | j
                  dz   t        | j                        k  r| jm                  dd      s| j
                  dz   t        | j                        k  r| jm                  d d!d"d#d$d%d&d'd(d)d*      sC| j
                  d+z   t        | j                        k  r.| jm                  d,d-d.d/d0d1d2d3d4d5d6d7d*      r| jo                         S | jq                         S # tJ        $ r d }Y 	Lw xY w)8Nr   r   Tr  r   r$   r  F)	r   &|^@%=!~r-  <>/*r   ),;`()[]r  r  :z.0123456789r%   )0b0B)0o0O)0x0Xr   r   zb"zb'zr"zr'zf"zf'zu"zu'zt'zt"r   r&   zbr"zbr'zrb"zrb'zfr"zfr'zrf"zrf'ztr'ztr"zrt'zrt")9r   rp   r#   rl   StopIterationr   rz   r{   rX   rY   rW   r   r   r   r   r   r	  r   r   r   r   rZ   r   r   r  r   r   r   r  r   r   r   rU   r   r   r   rV   r   r   rc   r\   r]   r^   r_   r`   r   r   r   ra   r   r   r   r   r   r   re   rA   )rC   current_charfound_whitespaceseen_newliner   indent_tokens         r   __next__zTokenIterator.__next__/  s4   ??&4??+?+?9CVCV+V T[[!11&~~''##!!  ( 
 ~~''||~%DKK 00>>## $$(@(@@""((L,H,HH<<>!{{4#5#56 ))ld.BLLN$$&||~%;;t'9'9:Lt#%)"OO/,,	0A0AA,0D) 3!!y'8'899##%doo.?22D$6$674? ##%doo.? ??9#4#455 "1$ --y'7'788 ??<<>! 4LLN$$&##  % L##%%%'LLN'+$%4??+<;;t'9'9:Dt|LLN'+$#'L 77((;@8 / ##%2 $77)-D&??9#7#788 "d&@&@""a'""((L,D,DD).D&$#{{} '####%$*<*<*> ##%$*<*<*>??9#7#788HHLLNyy{c!??9<<003LLNyy{c!y||44yy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!yy{c!??9<<003LLNyy{c!y||44 yy{c!??9<<00:%LLN??9<<00 3LLN??9<<003LLN!#??9#3#3443LLN!#!!A%%&"??9#3#3443LLN!#??9#5#5663LLN!#!!A%%&"??9#5#5663LLN!#??9#3#3443LLN""a'&&,,0L0LL""113%)%=%=%A%A%C"""a'"%%))*D&??9#3#3443LLN""a'&&,,0L0LL+7+P+P""(y||4499;#%LLNy||44=(!!A%T[[)99dkk""T%7%7!%;?? {{}$##a'3t{{+;;""T%7%7!%;AA zz|###a'3t{{+;;""T%7%7!%;AA ''))||~% !#s4;;'77DJJsC<P""Q&#dkk*::JJ $   ""Q&#dkk*::JJ $  " ;;= yy{y  $#$s   3y yyrS   rP   r   )r   rq   rQ   r   )r   rR   r   rT   rQ   rT   )r   r#   rQ   ro   )r   rR   r   rR   rQ   r   )rQ   ro   )rQ   rq   )rQ   ztuple[str, str])rQ   r   )6r   r   r   r}   r   r   r   r   r   r   r   r   r   listr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rX   rl   r   r   r   r   r   r  r	  re   rV   r  r  rA   r&  rG  r   r   r   r   r      s   KM3JKcKc(,,M3%*4%@@#J##D9L)9NC #("EM<E8=d8S 5S!%NJ% $M:$  M4"d""'4'"'4'5/3"+ 8= 5n#	
$4Q1f1414108"t9l%N?9B

5&yr   r   T)fstring_tokensr   c               L    t        | |      }|rt        |      S t        |      S )N)r   )r   itermerge_fstring_tokens)r{   rI  r   token_iterators       r   tokenizerN  k  s*     #6AVWNN##//r   c           
   #  0  K   | D ]  }|j                   t        j                  t        j                  fvr| 5|}|}d}d}| D ]m  }|j                   t        j                  t        j                  fv r|dz  }|j                   t        j                  t        j
                  fv r|dz  }||k(  sk|} n t        t        j                  |j                  |j                  |j                  |j                  |j                  |j                          yw)zETurn post-Python-3.12 FSTRING-* tokens back to a single STRING token.r$   r   )rp   rr   rt   ru   rs   rv   rw   N)rp   r#   rf   ri   rh   rk   ro   re   rr   rt   ru   rs   rv   rw   )rM  r   start_token	end_tokenfstring_startsfstring_endss         r   rL  rL  x  s	     
::i55y7N7NOOK	# 	Ezzi55y7N7NOO!#zzi33Y5J5JKK!-!		 !!#//"--!++))''%%
 	
)
s   B,D/A'D)r{   rR   rI  rT   r   rT   rQ   Iterator[Token])rM  r   rQ   rT  )__doc__
__future__r   dataclassesr   r   enumre   typingr   r   	Exceptionr	   r   r   r   r   r   r   r   r!   IntEnumr#   ro   r   r   rN  rL  r   r   r   <module>r\     s    Z " (   % $I # +} * <%5 ; >'7 = - , (M ' < ; ") !  	 0> 0>f  9  9  9F0; 0;f w w wz  "&	
0
0 
0  	
0
 
0
r   