U
    7ÄT_Ó	  ã                   @   sV   d Z ddlmZ ddlmZ ddlmZmZ ddlm	Z	 G dd„ de
ƒZdd	d
„ZdS )z	SQL Lexeré    )Útokens)Ú	SQL_REGEX)Ú	text_typeÚ
file_types)Úconsumec                   @   s   e Zd ZdZeddd„ƒZdS )ÚLexerz?Lexer
    Empty class. Leaving for backwards-compatibility
    Nc                 c   s  t | tƒr|  ¡ } t | tƒrn\t | tƒrh|r8|  |¡} qzz|  d¡} W qz tk
rd   |  d¡} Y qzX ntd t	| ƒ¡ƒ‚t
| ƒ}|D ]€\}}tD ]f\}}|| |ƒ}|s¬q’n2t |tjƒrÈ|| ¡ fV  nt|ƒrÞ|| ¡ ƒV  t|| ¡ | d ƒ  q†q’tj|fV  q†dS )a³  
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.

        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the initial stack (default: ``['root']``)
        zutf-8zunicode-escapez+Expected text or file-like object, got {!r}é   N)Ú
isinstancer   Úreadr   ÚbytesÚdecodeÚUnicodeDecodeErrorÚ	TypeErrorÚformatÚtypeÚ	enumerater   r   Z
_TokenTypeÚgroupÚcallabler   ÚendÚError)ÚtextÚencodingÚiterableÚposÚcharZrexmatchÚactionÚm© r   ú2/tmp/pip-unpacked-wheel-bbdw8hci/sqlparse/lexer.pyÚ
get_tokens   s6    


ÿ
zLexer.get_tokens)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr   r   r   r   r   r      s   r   Nc                 C   s   t ƒ  | |¡S )z�Tokenize sql.

    Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
    of ``(token type, value)`` items.
    )r   r   )Zsqlr   r   r   r   ÚtokenizeL   s    r%   )N)r#   Zsqlparser   Zsqlparse.keywordsr   Zsqlparse.compatr   r   Zsqlparse.utilsr   Úobjectr   r%   r   r   r   r   Ú<module>	   s   6