
Ѯh                 @   s  d  Z  d  Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z y e j e j	 f Z
 Wn e k
 r e e f Z
 Yn Xe j d  Z Gd d   d e  Z Gd d   d e  Z Gd d	   d	 e  Z Gd
 d   d e  Z Gd d   d  Z d d   Z d d   Z d d   Z d d   Z d d   Z d d   Z Gd d   d e  Z d d d d d e e j  d d d d d d 
 Z  d d d  d!  Z! d" d#   Z" e" Z# d S)$z3.10    Nz^[a-zA-Z0-9_]+$c               @   s   e  Z d  Z d d   Z d S)LexErrorc             C   s   | f |  _  | |  _ d  S)N)argstext)selfmessages r   ../pycparser/ply/lex.py__init__:   s    zLexError.__init__N)__name__
__module____qualname__r
   r   r   r   r	   r   9   s   r   c               @   s(   e  Z d  Z d d   Z d d   Z d S)LexTokenc             C   s    d |  j  |  j |  j |  j f S)NzLexToken(%s,%r,%d,%d))typevaluelinenolexpos)r   r   r   r	   __str__A   s    zLexToken.__str__c             C   s
   t  |   S)N)str)r   r   r   r	   __repr__D   s    zLexToken.__repr__N)r   r   r   r   r   r   r   r   r	   r   @   s   r   c               @   sL   e  Z d  Z d d   Z d d   Z d d   Z d d   Z e Z e Z d	 S)
	PlyLoggerc             C   s   | |  _  d  S)N)f)r   r   r   r   r	   r
   L   s    zPlyLogger.__init__c             O   s   |  j  j | | d  d  S)N
)r   write)r   msgr   kwargsr   r   r	   criticalO   s    zPlyLogger.criticalc             O   s    |  j  j d | | d  d  S)Nz	WARNING: r   )r   r   )r   r   r   r   r   r   r	   warningR   s    zPlyLogger.warningc             O   s    |  j  j d | | d  d  S)NzERROR: r   )r   r   )r   r   r   r   r   r   r	   errorU   s    zPlyLogger.errorN)	r   r   r   r
   r   r   r   infodebugr   r   r   r	   r   K   s   r   c               @   s(   e  Z d  Z d d   Z d d   Z d S)
NullLoggerc             C   s   |  S)Nr   )r   namer   r   r	   __getattribute__^   s    zNullLogger.__getattribute__c             O   s   |  S)Nr   )r   r   r   r   r   r	   __call__a   s    zNullLogger.__call__N)r   r   r   r#   r$   r   r   r   r	   r!   ]   s   r!   c               @   s   e  Z d  Z d d   Z d d d  Z d d d  Z d	 d
   Z d d   Z d d   Z d d   Z	 d d   Z
 d d   Z d d   Z d d   Z d d   Z d d   Z e Z d S)Lexerc             C   s   d  |  _  d  |  _ i  |  _ i  |  _ i  |  _ d |  _ g  |  _ d  |  _ i  |  _ i  |  _	 i  |  _
 d |  _ d  |  _ d |  _ d |  _ d  |  _ d  |  _ d  |  _ d |  _ d |  _ d  |  _ d |  _ d |  _ d  S)NINITIALr       F)lexre	lexretext
lexstaterelexstateretextlexstaterenameslexstatelexstatestacklexstateinfolexstateignorelexstateerrorflexstateeoff
lexreflagslexdatar   lexlen	lexerrorflexeoff	lextokens	lexignorelexliterals	lexmoduler   lexoptimize)r   r   r   r	   r
   t   s.    																						zLexer.__init__Nc             C   s-  t  j  |   } | r)i  } x |  j j   D] \ } } g  } xt | D]l \ } } g  }	 xW | D]O }
 |
 su |
 d r |	 j |
  q] |	 j t | |
 d j  |
 d f  q] WqD W| j | |	 f  | | | <q+ W| | _ i  | _ x6 |  j j   D]% \ } } t | | j  | j | <q W| | _ | S)Nr   r(   )copyr+   itemsappendgetattrr   r2   r<   )r   objectcZnewtabkeyZritemZnewreZcreZfindexZ	newfindexr   efr   r   r	   clone   s(    /			zLexer.cloner'   c             C   sr  t  | t j  r t d   | j d  d } t j j | |  d } t | d  } | j	 d | t
 f  | j	 d t t   | j	 d t t t |  j     | j	 d	 t |  j   | j	 d
 t |  j   | j	 d t |  j   i  } x |  j j   D]r \ } } g  }	 xS t | |  j | |  j |  D]1 \ \ }
 } } } |	 j | t | |  f  qGW|	 | | <qW| j	 d t |   | j	 d t |  j   i  } x6 |  j j   D]% \ } } | r| j n d  | | <qW| j	 d t |   i  } x6 |  j j   D]% \ } } | rB| j n d  | | <q'W| j	 d t |   Wd  QRXd  S)Nz&Won't overwrite existing lextab module.r(   z.pywzJ# %s.py. This file automatically created by PLY (version %s). Don't edit!
z_tabversion   = %s
z_lextokens    = set(%s)
z_lexreflags   = %s
z_lexliterals  = %s
z_lexstateinfo = %s
z_lexstatere   = %s
z_lexstateignore = %s
z_lexstateerrorf = %s
z_lexstateeoff = %s
)
isinstancetypes
ModuleTypeIOErrorsplitospathjoinopenr   __version__repr__tabversion__tuplesortedr9   r4   r;   r0   r+   r?   zipr,   r-   r@   _funcs_to_namesr1   r2   r   r3   )r   lextab	outputdirZbasetabmodulefilenameZtfZtabre	statenamelretitempatfuncZretextrenamesZtaberrrE   Ztabeofr   r   r	   writetab   s6    &6 zLexer.writetabc             C   s  t  | t j  r | } n t d |  t j | } t | d d  t k rZ t d   | j	 |  _
 | j |  _ | j |  _ |  j
 t |  j  B|  _ | j |  _ | j |  _ i  |  _ i  |  _ x | j j   D]t \ } } g  } g  } x? | D]7 \ } }	 | j t j | | j  t |	 |  f  q W| |  j | <| |  j | <q Wi  |  _ x. | j j   D] \ } }
 | |
 |  j | <qbWi  |  _ x. | j  j   D] \ } }
 | |
 |  j | <qW|  j! d  d  S)Nz	import %s_tabversionz0.0zInconsistent PLY versionr&   )"rJ   rK   rL   execsysmodulesrA   rU   ImportErrorZ
_lextokensr9   Z_lexreflagsr4   Z_lexliteralsr;   setlextokens_allZ_lexstateinfor0   Z_lexstateignorer1   r+   r,   Z_lexstaterer?   r@   recompile_names_to_funcsr2   Z_lexstateerrorfr3   Z_lexstateeoffbegin)r   ZtabfilefdictrZ   r]   r^   r_   Ztxtitemr`   Z	func_namerE   r   r   r	   readtab   s8    			/		zLexer.readtabc             C   sP   | d  d  } t  | t  s+ t d   | |  _ d |  _ t |  |  _ d  S)Nr(   zExpected a stringr   )rJ   StringTypes
ValueErrorr5   r   lenr6   )r   r   rC   r   r   r	   input   s    		zLexer.inputc             C   s   | |  j  k r t d   |  j  | |  _ |  j | |  _ |  j j | d  |  _ |  j j | d   |  _	 |  j
 j | d   |  _ | |  _ d  S)NzUndefined stater'   )r+   rr   r)   r,   r*   r1   getr:   r2   r7   r3   r8   r.   )r   stater   r   r	   rn     s    zLexer.beginc             C   s$   |  j  j |  j  |  j |  d  S)N)r/   r@   r.   rn   )r   rv   r   r   r	   
push_state  s    zLexer.push_statec             C   s   |  j  |  j j    d  S)N)rn   r/   pop)r   r   r   r	   	pop_state  s    zLexer.pop_statec             C   s   |  j  S)N)r.   )r   r   r   r	   current_state!  s    zLexer.current_statec             C   s   |  j  | 7_  d  S)N)r   )r   nr   r   r	   skip'  s    z
Lexer.skipc             C   sf  |  j  } |  j } |  j } |  j } x| | k  r| | | k rP | d 7} q' x|  j D]B\ } } | j | |  } | s qZ t   } | j   | _ |  j	 | _	 | | _  | j
 }	 | |	 \ }
 | _ |
 s | j r | j   |  _  | S| j   } P| j   } |  | _ | |  _ | |  _  |
 |  } | sE|  j  } |  j } P|  j s| j |  j k rt d |
 j j |
 j j |
 j | j f | | d     | SW| | |  j k rt   } | | | _ |  j	 | _	 | j | _ | | _  | d |  _  | S|  j rt   } |  j | d   | _ |  j	 | _	 d | _ |  | _ | | _  | |  _  |  j |  } | |  j  k rt d | | | | d     |  j  } | sq' | S| |  _  t d | | | f | | d     q' W|  j r:t   } d | _ d | _ |  j	 | _	 | | _  |  | _ | |  _  |  j |  } | S| d |  _  |  j d  k rbt d   d  S)	Nr(   z4%s:%d: Rule '%s' returned an unknown token type '%s'r   z&Scanning error. Illegal character '%s'z"Illegal character '%s' at index %deofr'   z"No input string given with input())r   r6   r:   r5   r)   matchr   groupr   r   	lastindexr   endlexerZlexmatchr=   rj   r   __code__co_filenameco_firstlinenor   r;   r7   r8   RuntimeError)r   r   r6   r:   r5   r)   lexindexfuncmtokira   Znewtokr   r   r	   token1  s    				
										#								!		+							zLexer.tokenc             C   s   |  S)Nr   )r   r   r   r	   __iter__  s    zLexer.__iter__c             C   s"   |  j    } | d  k r t  | S)N)r   StopIteration)r   tr   r   r	   next  s    z
Lexer.next)r   r   r   r
   rF   rc   rp   rt   rn   rw   ry   rz   r|   r   r   r   __next__r   r   r   r	   r%   s   s   %(
nr%   c             C   s   t  |  d |  j  S)Nregex)rA   __doc__)ra   r   r   r	   
_get_regex  s    r   c             C   sD   t  j |   } | j j   } | j | j k r@ | j | j  | S)N)rf   	_getframe	f_globalsr>   f_localsupdate)levelsr   ldictr   r   r	   get_caller_module_dict  s
    r   c             C   sa   g  } xT t  |  |  D]C \ } } | rL | d rL | j | | d f  q | j |  q W| S)Nr   r(   )rX   r@   )Zfunclistnamelistresultr   r"   r   r   r	   rY     s    rY   c             C   sZ   g  } xM |  D]E } | rE | d rE | j  | | d | d f  q | j  |  q W| S)Nr   r(   )r@   )r   ro   r   r{   r   r   r	   rm     s    "rm   c             C   s  |  s
 g  Sd j  |   } yt j | |  } d  g t | j j    d } | d  d   } x | j j   D] \ } }	 | j | d   }
 t |
  t	 j
 t	 j f k r |
 | | f | |	 <| | |	 <qn |
 d  k	 rn | | |	 <| j d  d k rd | |	 <qn d  | | f | |	 <qn W| | f g | g | g f SWn t k
 rt t |   d  } | d k rpd } t |  d  |  | | |  \ } } } t |  | d   | | |  \ } } } | | | | | | f SYn Xd  S)N|r(   ignore_r      )NN)rQ   rk   rl   max
groupindexvaluesr?   ru   r   rK   FunctionType
MethodTypefind	Exceptionintrs   _form_master_re)Zrelistreflagsr   toknamesr   r)   r   Zlexindexnamesr   r   Zhandler   Zllistr^   ZlnamesZrlistZrreZrnamesr   r   r	   r     s2     
 ((r   c             C   s   d } |  j  d  } x@ t | d d   d  D]% \ } } | | k r/ | d k r/ Pq/ W| d k r} t | d |   } n d } d | k r t |  } d j | | d    } | | f S)Nr(   _ANYr&   )r&   )rN   	enumeraterV   rQ   )r   namesZnonstatepartsr   partstatesZ	tokennamer   r   r	   _statetoken  s    &r   c               @   s   e  Z d  Z d d d d  Z d d   Z d d   Z d	 d
   Z d d   Z d d   Z d d   Z	 d d   Z
 d d   Z d d   Z d d   Z d S)LexerReflectNr   c             C   sp   | |  _  d  |  _ g  |  _ | |  _ d d i |  _ t   |  _ d |  _ | d  k rc t t	 j
  n | |  _ d  S)Nr&   	inclusiveF)r   
error_functokensr   	stateinfori   rg   r   r   rf   stderrlog)r   r   r   r   r   r   r	   r
   /  s    					zLexerReflect.__init__c             C   s,   |  j    |  j   |  j   |  j   d  S)N)
get_tokensget_literals
get_states	get_rules)r   r   r   r	   get_all:  s    


zLexerReflect.get_allc             C   s%   |  j    |  j   |  j   |  j S)N)validate_tokensvalidate_literalsvalidate_rulesr   )r   r   r   r	   validate_allA  s    


zLexerReflect.validate_allc             C   s   |  j  j d d   } | s8 |  j j d  d |  _ d  St | t t f  sj |  j j d  d |  _ d  S| s |  j j d  d |  _ d  S| |  _ d  S)Nr   zNo token list is definedTztokens must be a list or tupleztokens is empty)r   ru   r   r   rJ   listrV   r   )r   r   r   r   r	   r   H  s    			zLexerReflect.get_tokensc             C   sr   i  } xe |  j  D]Z } t j |  sA |  j j d |  d |  _ | | k r` |  j j d |  d | | <q Wd  S)NzBad token name '%s'TzToken '%s' multiply definedr(   )r   _is_identifierr~   r   r   r   )r   	terminalsr{   r   r   r	   r   \  s    	zLexerReflect.validate_tokensc             C   s.   |  j  j d d  |  _ |  j s* d |  _ d  S)Nliteralsr'   )r   ru   r   )r   r   r   r	   r   g  s    	zLexerReflect.get_literalsc             C   s   y\ xU |  j  D]J } t | t  s5 t |  d k r |  j j d t |   d |  _ q WWn+ t k
 r |  j j d  d |  _ Yn Xd  S)Nr(   z.Invalid literal %s. Must be a single characterTzIInvalid literals specification. literals must be a sequence of characters)r   rJ   rq   rs   r   r   rT   	TypeError)r   rC   r   r   r	   r   m  s    "zLexerReflect.validate_literalsc             C   sf  |  j  j d d   |  _ |  j rbt |  j t t f  sU |  j j d  d |  _ nx
|  j D] } t | t  s t |  d k r |  j j d t	 |   d |  _ q_ | \ } } t | t
  s |  j j d t	 |   d |  _ q_ | d k p| d k s#|  j j d	 |  d |  _ q_ | |  j k rQ|  j j d
 |  d |  _ q_ | |  j | <q_ Wd  S)Nr   z)states must be defined as a tuple or listTr   zMInvalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')zState name %s must be a stringr   	exclusivez:State type for state %s must be 'inclusive' or 'exclusive'zState '%s' already defined)r   ru   r   rJ   rV   r   r   r   rs   rT   rq   r   )r   r   r"   Z	statetyper   r   r	   r   x  s0    	"				zLexerReflect.get_statesc       	      C   s	  d d   |  j  D } i  |  _ i  |  _ i  |  _ i  |  _ i  |  _ i  |  _ x+ |  j D]  } g  |  j | <g  |  j | <qV Wt |  d k r |  j	 j
 d  d |  _
 d  Sx| D]} |  j  | } t | |  j  \ } } | |  j | <t | d  r| d k r$x | D] } | |  j | <q
Wq| d k rQx | D] } | |  j | <q7Wq| d	 k r| j j } | j j } |  j	 j
 d
 | | | j  d |  _
 qx | D]  } |  j | j | | f  qWq t | t  r| d	 k r&x | D] } | |  j | <qWd | k r||  j	 j d |  q| d k rQ|  j	 j
 d |  d |  _
 qxG | D]  } |  j | j | | f  qXWq |  j	 j
 d |  d |  _
 q Wx- |  j j   D] } | j d d d    qWx3 |  j j   D]" } | j d d d   d d  qWd  S)Nc             S   s,   g  |  ]" } | d  d  d k r |  q S)Nr   t_r   ).0r   r   r   r	   
<listcomp>  s   	 z*LexerReflect.get_rules.<locals>.<listcomp>r   z+No rules of the form t_rulename are definedTr$   r   r}   ignorez,%s:%d: Rule '%s' must be defined as a string\z#%s contains a literal backslash '\'z'Rule '%s' must be defined as a functionz&%s not defined as a function or stringrD   c             S   s   |  d j  j S)Nr(   )r   r   )xr   r   r	   <lambda>  s    z(LexerReflect.get_rules.<locals>.<lambda>c             S   s   t  |  d  S)Nr(   )rs   )r   r   r   r	   r     s    reverse)r   r   funcsymstrsymr   errorfeoffr   rs   r   r   r   hasattrr   r   r   r   r@   rJ   rq   r   r   sort)	r   Ztsymbolsr   r   r   r   toknamelinefiler   r   r	   r     sb    							!!zLexerReflect.get_rulesc             C   s  x|  j  D]} x|  j | D]\ } } | j j } | j j } t j |  } |  j j |  |  j	 | } t
 | t j  r d } n d } | j j }	 |	 | k r |  j j d | | | j  d |  _ q |	 | k  r|  j j d | | | j  d |  _ q t |  s7|  j j d | | | j  d |  _ q y] t j d | t |  f |  j  }
 |
 j d  r|  j j d	 | | | j  d |  _ Wq t j k
 r} z[ |  j j d
 | | | j |  d t |  k r|  j j d | | | j  d |  _ WYd  d  } ~ Xq Xq WxP|  j | D]A\ } } |  j	 | } | d k rk|  j j d |  d |  _ q'| |  j k r| j d  d k  r|  j j d | |  d |  _ q'yN t j d | | f |  j  }
 |
 j d  r|  j j d |  d |  _ Wq't j k
 rg} zC |  j j d | |  d | k rL|  j j d |  d |  _ WYd  d  } ~ Xq'Xq'W|  j | r|  j | r|  j j d |  d |  _ |  j j | d   } | r
 | } | j j } | j j } t j |  } |  j j |  t
 | t j  rd } n d } | j j }	 |	 | k rZ|  j j d | | | j  d |  _ |	 | k  r
 |  j j d | | | j  d |  _ q
 Wx |  j D] } |  j |  qWd  S)Nr   r(   z'%s:%d: Rule '%s' has too many argumentsTz%%s:%d: Rule '%s' requires an argumentz2%s:%d: No regular expression defined for rule '%s'z
(?P<%s>%s)r'   z<%s:%d: Regular expression for rule '%s' matches empty stringz3%s:%d: Invalid regular expression for rule '%s'. %s#z6%s:%d. Make sure '#' in rule '%s' is escaped with '\#'r   z'Rule '%s' must be defined as a functionr   r   z-Rule '%s' defined for an unspecified token %sz5Regular expression for rule '%s' matches empty stringz,Invalid regular expression for rule '%s'. %sz/Make sure '#' in rule '%s' is escaped with '\#'zNo rules defined for state '%s')r   r   r   r   r   inspect	getmodulerg   addr   rJ   rK   r   co_argcountr   r   r   r   rk   rl   r   r~   r   r   r   r   ru   validate_module)r   rv   fnamer   r   r   moduler   reqargsnargsrC   er"   rZefuncr   r   r	   r     s    				% 	$	 			zLexerReflect.validate_rulesc             C   s  y t  j |  \ } } Wn t k
 r1 d  SYn Xt j d  } t j d  } i  } | d 7} x | D] } | j |  } | s | j |  } | r | j d  }	 | j |	  }
 |
 s | | |	 <n4 t  j |  } |  j	 j
 d | | |	 |
  d |  _
 | d 7} qg Wd  S)Nz\s*def\s+(t_[a-zA-Z_0-9]*)\(z\s*(t_[a-zA-Z_0-9]*)\s*=r(   z7%s:%d: Rule %s redefined. Previously defined on line %dT)r   getsourcelinesrM   rk   rl   r~   r   ru   getsourcefiler   r   )r   r   lineslinenfreZsre	counthashr   r   r"   prevr\   r   r   r	   r   @  s*    	
	zLexerReflect.validate_module)r   r   r   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r	   r   .  s   Bgr   FrZ   c
       %         sR  | d  k r d } d  }
 d d i } t    } | | _ |	 d  k rQ t t j  }	 | rr | d  k rr t t j  } | r~ |     r   f d d   t    D } t |  }
 d |
 k r t j |
 d j |
 d <n t	 d  }
 |
 j
 d	  } | r t | t  r d
 | k r | d
 | } t |
 d |	 d | } | j   | s`| j   r`t d   | r| ry0 | j | |
  | j a | j a | a | SWn t k
 rYn X| r| j d | j  | j d | j  | j d | j  t   | _ x! | j D] } | j j |  qWt | j t t f  r`t | j d    j  | j  | _! n | j | _! | j t | j!  B| _" | j } i  } x | D] } g  } xr | j# | D]c \ } } | j$ j% } | j$ j& } | j' d | t( |  f  | r| j d | t( |  |  qWxN | j) | D]? \ } } | j' d | | f  | r*| j d | | |  q*W| | | <qW| r| j d  x | D] } t* | | | |
 | j+  \ } } } | | j, | <| | j- | <| | j. | <| rx0 t/ |  D]" \ } } | j d | | |  qWqWx | j0   D]u \ } } | d k r1| d k r1| j, | j1 | j, d  | j- | j1 | j- d  | j. | j1 | j. d  q1W| | _2 | j, d | _3 | j- d | _4 | | _5 | j6 | _7 | j7 j
 d d  | _8 | j9 | _: | j9 j
 d d   | _; | j; s:|	 j< d  | j= | _> | j= j
 d d   | _? x | j0   D] \ }  } | d k r|  | j9 k r|	 j< d |   |  | j6 k r/| j8 r/|	 j< d |   qk| d k rk|  | j9 k r| j9 j
 d d   | j9 |  <|  | j6 k rk| j6 j
 d d  | j6 |  <qkW| j a | j a | a | rN| rN| d  k rt | t@ jA  r| j }! nh d
 | k r|
 d }! nO | jB d
  }" d
 j  |" d  d   }# tC d |#  tD t j |# d d  }! tE jF jG |!  } y | jH | |  Wn< tI k
 rM}$ z |	 j< d | |$ f  WYd  d  }$ ~$ Xn X| S)NrZ   r&   r   c                s%   g  |  ] } | t    |  f  q Sr   )rA   )r   k)r   r   r	   r   z  s   	 zlex.<locals>.<listcomp>__file__r   r   __package__rG   r   r   zCan't build lexerzlex: tokens   = %rzlex: literals = %rzlex: states   = %rr   z
(?P<%s>%s)z(lex: Adding rule %s -> '%s' (state '%s')z#lex: ==== MASTER REGEXS FOLLOW ====z"lex: state '%s' : regex[%d] = '%s'r'   zNo t_error rule is definedr   z1No error rule is defined for exclusive state '%s'z2No ignore rule is defined for exclusive state '%s'r(   z	import %sz#Couldn't write lextab module %r. %srI   )Jr%   r=   r   rf   r   dirdictrg   r   r   ru   rJ   r   r   r   r   SyntaxErrorrp   r   rt   r   rh   r   r   r   r   ri   r9   r   r   rV   r   rQ   r;   rj   r   r   r   r   r@   r   r   r   r   r+   r,   r-   r   r?   extendr0   r)   r*   r4   r   r1   r:   r   r2   r7   r   r   r3   r8   rK   rL   rN   re   rA   rO   rP   dirnamerc   rM   )%r   rB   r    optimizerZ   r   Znowarnr[   debuglogerrorlogr   r   Zlexobj_itemspkgZlinfor{   Zregexsrv   Z
regex_listr   r   r   r   r"   r   r)   Zre_textZre_namesr   r   styper   srcfiler   pkgnamer   r   )r   r	   lex_  s    		
		(	 %			 		*r   c             C   s   | sm y3 t  j d } t |  } | j   } | j   Wn1 t k
 rl t  j j d  t  j j   } Yn X|  r |  j	 } n t	 } | |  |  r |  j
 } n t
 } x@ |   } | s Pt  j j d | j | j | j | j f  q Wd  S)Nr(   z/Reading from standard input (type EOF to end):
z(%s,%r,%d,%d)
)rf   argvrR   readclose
IndexErrorstdoutr   stdinrt   r   r   r   r   r   )r   datar\   r   _input_tokenr   r   r   r	   runmain  s*    
	r  c                s     f d d   } | S)Nc                s.   t    d  r! t    |  _ n	   |  _ |  S)Nr$   )r   r   r   )r   )r   r   r	   	set_regexB  s    	zTOKEN.<locals>.set_regexr   )r   r  r   )r   r	   TOKENA  s    r	  )$rS   rU   rk   rf   rK   r>   rO   r   Z
StringTypeZUnicodeTyperq   AttributeErrorr   bytesrl   r   r   r   rB   r   r   r!   r%   r   r   rY   rm   r   r   r   r   VERBOSEr   r  r	  Tokenr   r   r   r	   <module>"   s>    E
( 2$"
