
    pi                    z    d dl mZ d dlZd dlZd dlZd	dZd
dZdddZ	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZddZ	dddZ
y)    )annotationsNc                   d| vr(t        | dd      5 }|j                         cd d d        S | j                  d      r1dd l}|j	                  |       }|j                          |j                  S 	 dd l}|j                  | d      5 }|j                         cd d d        S # 1 sw Y   |xY w# t        $ r}t        d      |d }~ww xY w# 1 sw Y   y xY w)Nz://rbr   	buffering)zhttp://zhttps://Oblobfile is not installed. Please install it by running `pip install blobfile`.)
openread
startswithrequestsgetraise_for_statuscontentblobfileImportErrorBlobFile)blobpathfr   respr   es         O/opt/services/ai/voice_agent/venv/lib/python3.12/site-packages/tiktoken/load.py	read_filer      s    H(DA. 	!668	 	 23||H%||
 
		8T	* avvx #	 	  ]
	 s/   B 0B, C	 B),	C5CC	Cc                R    t        j                  |       j                         }||k(  S N)hashlibsha256	hexdigest)dataexpected_hashactual_hashs      r   
check_hashr!      s$    ..&002K-''    c                b   d}dt         j                  v rt         j                  d   }nZdt         j                  v rt         j                  d   }n4dd l}t         j                  j	                  |j                         d      }d}|dk(  rt        |       S t        j                  | j                               j                         }t         j                  j	                  ||      }t         j                  j                  |      rMt        |dd	      5 }|j                         }d d d        |t        |      rS 	 t        j                  |       t        |       }	|rt        |	|      st#        d
|  d| d      dd l}
	 t        j&                  |d       |dz   t)        |
j+                               z   dz   }t        |d      5 }|j-                  |	       d d d        t        j.                  ||       |	S # 1 sw Y   xY w# t         $ r Y w xY w# 1 sw Y   <xY w# t         $ r |r Y |	S w xY w)NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsr	   r
   r!   removeOSError
ValueErroruuidmakedirsstruuid4writerename)r   r   user_specified_cache	cache_dirr,   	cache_key
cache_pathr   r   contentsr6   tmp_filenames               r   read_file_cachedrB   $   s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!*da0 	A668D	 Jt]$CK	IIj! "HZ-@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,% 	GGH	
		,
+ O?	 	  			 	     OsI   G7;H  AH HH 7H 	HHHH H.-H.c                   t        d      D cg c]-  }t        |      j                         st        |      dk7  s,|/ }}|D ci c]  }t        |      | c}d}t        d      D ].  }||vs|j                  |       |t        d|z         <   |dz  }0 t	        |      dk(  sJ t        | |      j                         }|j                  d      dd D 	cg c]  }	t        |	j                                }
}	d
fd}t        |      D ci c]  \  }}t        |g      | }}}~t	        |      }|
D ]  \  }}|| ||       ||      z   <   |dz  }  dd l}|j                  t        ||            }|j                         D ci c]  \  }} ||      | }}}|j                  dd        |j                  d	d        |r|D ]  }t	        |      dk(  s||   ||<    ||k(  sJ |S c c}w c c}w c c}	w c c}}w c c}}w )N    r      
c                ,    t        fd| D              S )Nc              3  (   K   | ]	  }|     y wr    ).0bdata_gym_byte_to_bytes     r   	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>r   s     =!*1-=s   )bytes)valuerN   s    r   decode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymq   s    =u===r"   s   <|endoftext|>s   <|startoftext|>)rQ   r8   returnrP   )rangechrisprintableappendlenrB   decodesplittuple	enumeraterP   jsonloadsitemspop)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashclobber_one_byte_tokensrM   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrR   i	bpe_ranksfirstsecondr]   encoder_jsonkvencoder_json_loadedrN   s                        @r   data_gym_to_mergeable_bpe_ranksrs   Z   s4    #(+XQQ1C1C1E#a&TW-qXOX0?@1SVQY@	A4[ O#""1%34!#dQh-0FA	
 4''' *..IPPR<N<T<TUY<Z[\]_<`ay%	)*aJa>
 ,5_+EF41asQFIF 	IA# vFG	/%(?6+BBC	Q 
 ::./@BSTUL=I=O=O=QRTQ?1-q0RR,d3.5$ 	6A1v{215	!	6 ++++_ Y@ b G Ss'   G/G/G/G4 G9G>Hc                h   	 dd l }|j                  |d      5 }t        | j	                         d       D ]I  \  }}|j                  t        j                  |      dz   t        |      j                         z   dz          K 	 d d d        y # t        $ r}t        d      |d }~ww xY w# 1 sw Y   y xY w)Nr   r   r)   c                    | d   S )NrF   rK   )xs    r   <lambda>z#dump_tiktoken_bpe.<locals>.<lambda>   s
    1Q4 r"   )key       
)
r   r   r   sortedr_   r:   base64	b64encoder8   r1   )rl   tiktoken_bpe_filer   r   r   tokenranks          r   dump_tiktoken_bper      s    
 
		,d	3 Qq!)//"3H 	QKE4GGF$$U+d2SY5E5E5GG%OP	QQ Q	  ]
	Q Qs#   B A)B(	B%B  B%(B1c                   t        | |      }i }|j                         D ]:  }|s	 |j                         \  }}t        |      |t	        j
                  |      <   < |S # t        $ r}t        d|d|        |d }~ww xY w)NzError parsing line z in )rB   
splitlinesrZ   intr|   	b64decode	Exceptionr5   )r~   r   r@   retliner   r   r   s           r   load_tiktoken_bper      s     1=AH
C##% [	[**,KE4+.t9C  '([ J  	[24($?P>QRSYZZ	[s   4A	A>'A99A>)r   r8   rS   rP   )r   rP   r   r8   rS   boolr   )r   r8   r   
str | NonerS   rP   )NNF)ra   r8   rb   r8   rc   r   rd   r   re   r   rS   dict[bytes, int])rl   r   r~   r8   rS   None)r~   r8   r   r   rS   r   )
__future__r   r|   r   r*   r   r!   rB   rs   r   r   rK   r"   r   <module>r      su    "   	.(
3r "&$($)777 7 "	7
 "7 7t	Qr"   