piddlmZddlZddlZddlZddlZddlmZddlm Z ddl Z ddl Z ddl m Z m Z mZddl mZddlmZmZmZmZddlmZdd lmZmZmZ dd lmZ dd lmZm Z  dd l!m"Z"d Z#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,iZ-e.e Dcic]}e/e0e |e1se0e ||c}Z2GddeZ3GddeZ4GddeZ5GddeZ6ejjne jpdejjre jpd ejjte jpd!ejjve jpd"ejjxeejjzeejj|e iZ?ejjre jde j#e jd$e j#fejjne jd%e j#e jd&e j#fejjve jde j#e jd'e j#fejjte jd(e j#e jd)e j#fejj|e jde #e jd*e #fejjze jd+e#e jd,e#fiZEejjre jde j#e jd-e j#fejjne jd.e j#e jd&e j#fejjve jde j#e jd/e j#fejjte jd0e j#e jd)e j#fiZFejjre jde j#e jd&e j#fejjne jd1e j#e jd2e j#fejjve jde j#e jd)e j#fejjte jd3e j#e jd4e j#fejj|e jde#e jd,e#fejjze jd5e#e jd6e#fiZGd7d8d9ZHdcd:ZIddd;ZJd<ZK de dfd=ZL de dgd>ZM dc dhd?ZNdid@ZOdidAZPdjdBZQdkdCZRGdDdEZSGdFdGZTGdHdIZUdJZVdKZWdLZXdMZYdldNZZdOZ[dmdPZ\dndQZ]dodRZ^dpdSZ_dqdTZ`drdUZadqdVZbdrdWZcdsdXZddtdYZedudZZfdvd[Zgdwd\Zhdxd]Zidxd^Zjdyd_Zkdyd`ZldydaZmdydbZny#e$rdZYwxYw#e$rdZdZ YwxYw#e$rdZ"YwxYwcc}w)z) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4)to_array_extendedz onnx.quantizez0.1.0ai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlc*eZdZdZdZdZedZy)QuantizationModerc|jSNnameselfs f/opt/services/ai/voice_agent/venv/lib/python3.12/site-packages/onnxruntime/quantization/quant_utils.py__str__zQuantizationMode.__str__D yycD t|S#t$r twxYwr!)rKeyError ValueError)modes r& from_stringzQuantizationMode.from_stringGs) #D) ) ,   N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr' staticmethodr.r)r&rr@s%JJr)rc*eZdZdZdZdZedZy)QuantizedValueTyperrc|jSr!r"r$s r&r'zQuantizedValueType.__str__Sr(r)cD t|S#t$r twxYwr!)r8r+r,)vs r&r.zQuantizedValueType.from_stringVs) %a( ( ,  r/N)r0r1r2Input Initializerr'r5r.r6r)r&r8r8Os% EKr)r8cNeZdZdZdZdZdZdZdZdZ dZ e d Z e d Zy ) QuantTyperrc|jSr!r"r$s r&r'zQuantType.__str__gr(r)cD t|S#t$r twxYwr!)r?r+r,)ts r&r.zQuantType.from_stringjs( Q<  ,  r/c |tjk(rtjS|tjk(rtj S|tj k(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjSt!d|d)NzUnexpected value qtype=.)r?QInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r,r$s r& tensor_typezQuantType.tensor_typeqs 9?? "## # 9## #$$ $ 9$$ $%% % 9## #$$ $ 9** *++ + 9## #$$ $ 9?? "## #24(!<==r)N)r0r1r2rJrLrRrPrNrVrTr'r5r.propertyrXr6r)r&r?r?^sR E FM FG E F >>r)r?c*eZdZdZdZdZedZy) QuantFormatrrc|jSr!r"r$s r&r'zQuantFormat.__str__r(r)cD t|S#t$r twxYwr!)r[r+r,)formats r&r.zQuantFormat.from_strings) v& & ,  r/N)r0r1r2 QOperatorQDQr'r5r.r6r)r&r[r[s%I Cr)r[int8uint8int16uint16dtypeiiiiiiiii@ii@rAzero_point_indexc@g}t|D]\}}tjt|tjr%|j tj |n=t|tjr|j |ntd|d|||k(s|d}|jtjk(s|jtjk(std|jt|dkDr t|S|dS)Nzarg z is not an array: rnzzero_point cannot be rr) enumeratenumpy issubdtypetypenumberappendarray isinstancendarray TypeErrorrffloat32float16lentuple)rpargsnew_argsiar;s r& _check_typersH$ C1   DGU\\ 2 OOEKKN + 5== ) OOA d1#%7s;< <   Aww%--'177emm+C"7y ABB C"(ma/5?@Xa[@r)c|tvs Jd|d|tjjtjjtjj tjj fvr/|dk7rtd|d|jtjk(rtj}nG|jtjk(rtj}ntd|jdtt!t#dgdgt$j&j)d|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|j1d||ddSt|} t3|dd\} } | t5| |n| } | t7| |n| } tj8|j;tj|z j=|z}tj>|| | |t/|j;| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rIzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrSFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrfrsr|FLOATr}FLOAT16r,r r r onnxhelper make_tensorr rrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefrfqminqmaxcliplowcliphigharr_fp32s r&quantize_nparrayrs- ( ( w&de ( ++--))--  ?%(Z[eZhhi&jk k 99 %#))I YY%-- '#++I0 1=> > "Bdkk>U>UVbdikmpqor>s.0LseT  *3 4@*7ItD(UD9:  !,3774sU)CDQGHH%U+,URWX d$'O#dC.&*&63tT?D==#**U]]";e"C!J!J!Lz!YZ 8WhH=8??5122r)c |dkDs|dkrtd|d|tj|tjd|j}tj |tjd|j}|.t ||tj||jz}|rBtj tj|tj|}| }|}||ks Jd|d|tj||z tj}tj|tjtj|tjz }tj||z } | dk\sJd| tj|jjkrFtjd|j} tjd|j} | | gS|r^tjtj||ztjd tjz |j} n:tjtj||| z z |j} | j|j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rezqmin=z > qmax=z scale issue?g@)r,rsminimumrxrfmaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r&compute_scale_zprs ( ax4!8]^b]ccklpkqrss ==u{{1DJJ? @D ==u{{1DJJ? @D!4 nDJJ OOPuyy $@ww 4<55htf55< TD[ 6B T /%++d%--2X XB KKR E A:$}$: u{{4::&+++ Ctzz2[[$**5      TD[EKK5==,QQRZ^ZdZdJU[[u 1D%ETZZXJ TZZ(  r)cd}|tvr|tjk(rddlm}ddlm}|}tdDcgc] }|| }}tj|Dcgc]0}tj|rtj|r/|2c}tj}ntd|d|t|<n|tjk(rddlm}|}|td |d tjt|} tjd|} tj|| z |j } | | gScc}wcc}w) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)float8e4m3_to_float32rrezQuantization to element_type=z not implemented.zUnexpected element_type rI)FLOAT8_DISTRIBUTIONSrrSonnx.numpy_helperr#onnx.reference.custom_element_typesrrangersrxisnanisinfr|r,r{stdrf) element_typerzp_dtyperrr all_valuesfvaluesstd_f8zerors r&compute_scale_zp_float8r:s%H// ;33 3 ? H#H :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rhz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rI)rrrSranyrrsrbravelrrrrrKrMrQrOrWrUr,) rrrrrrrrrquantized_datanp_datas r& quantize_datarsZ82  J ((()%ujI %%ekk288:S@SH ImmD)G=gkkm_Bw{{}o^&&4&8&8&:%;2n>P>P>R=SSUW 5.00  *%ujI5.00 25'; <fvrt3d |d tAtC|j-}tjDjG|||j$|d }|StjDjI|}tj||j|j$}tjJjM||}|S)aG Returns a quantized version of the given ONNX initializer. :param weight: The ONNX initializer to quantize. :param quant_type: The final quantized data type. :param zero_point: The zero-point value to use for quantization. :param scale: The scale value to use for quantization. :param axis: The quantization axis if quantizing per-channel. Defaults to None. :param quant_weight_name: The name of the quantized initializer. If not specified, the quantized name is generated. :return: The quantized ONNX initializer. NrzThe initializer of shape z! could not be created, expecting z, got z and shape=z raw=rIzQuantized weights for z. must be 8-bit before packing as 4-bit values.T)rawre)'tensor_proto_to_arrayrrshapelistrtakerwrsrreshape concatenater#TENSOR_NAME_QUANT_SUFFIXrrrS data_typedimsextendflattencopytobytesraw_datarrstrrWrUrfrarbbytespack_bytes_to_4bitrrtensor_dtype_to_np_dtype numpy_helper from_array)weightrrraxisquant_weight_name weight_data q_weight_data channel_count channel_dimsquantized_channel_data_listr channel_data channel_scalechannel_zero_pointquantized_channel_data q_weight_nameq_weight_initializercheck packed_dataquant_np_dtypes r&quantize_onnx_initializerr s4((/K*.M |([5F5F5H%Q[\ #))$/ K--.  T&(#}% lA&++At4L!!HM!+A %5L..0-AS& " ( . .u}}=S/T/\/\]i/j k l))*EtL ):%6;;-PhOi@jMT%%222#//1)3&!!((5$1!(5(=(=(?(D(D(F(N(N(P%  (&&:;E{{k///5==?mF[F[F]3]"/ 0A0A/BBc$,,.s34F5==?3B;O:PP[\b\h\h[iS!56t<=Q@(  ((--t/?/?/E/EF F   uzz5;;&? ?!7 Ftuv v.}/D/D/FGH  ${{66}jRXR]R]_jpt6u  ==jI m>JRRSYS^S^_ #00;;M=Y r)cj|tjjk(r tdd}|rtj |}n)|r|t vr t |}ntj |}|std|d|\}}|dkDs|dkr'td|d|d|jd |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rrrSrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr,rf)rrrqrangerrs r&rr)s  &&333!"_`` F,007 u ==.u5$((/ 07uvwwJD$ ax4!86$x |?<.Y" 8E7 4  Mr)c.t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r&get_qrange_for_qTyperIs ) RJD$ $;r)c:|dkr||zn|}|dk\xr||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr6)rrank axis_normis_valids r&normalize_axisrSs3 $axt TIA~2)d"2H Y r)ct|}|dk(r tS|dzdz}t|}d}d}||dz kr-||dzdzdz||dzz||<|dz }|dz }||dz kr-||kr ||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rrr@rirB)r~ bytearray)src_8bit num_elemsdst_sizedstsrc_idst_is r&rr_sH IA~{A !#H H CEE )a-  *S0Q68E?S;PQE      )a-   ye_s*E Jr)ceZdZdZggdfdZy)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_yr!) r# initializerrminsrmaxs zero_pointsscalesrrr) r%r#r%r&r'r(r)rrrs r&__init__zQuantizedInitializer.__init__sF &  &  , r)r0r1r2__doc__r*r6r)r&r#r#}s r)r#c eZdZdZ ddZy)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_yr!) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r%r#new_quantized_namer2zero_point_namequantized_value_typerr5r6r7s r&r*zQuantizedValue.__init__sD"( $& . "$$r))NNNNr+r6r)r&r.r.s%r)r.ceZdZdZdZy)BiasToQuantizez+ Represents a bias to be quantized c.||_||_||_yr!) bias_name input_name weight_name)r%r>r?r@s r&r*zBiasToQuantize.__init__s"$&r)Nr+r6r)r&r<r<s 'r)r<c|jdk(rtd|jd|jdk(r|j}n#|jdk(r|j}n|jdk(r |j }n|jdk(r |j }n|jdk(r |j}n|jd k(r |j}n|jd k(r |j}nz|jd k(r |j}n^|jd k(r |j}nB|jd k(r |j}n&td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.rr@rArBrCrDrj rz has unsupported type rI) rur,r#rrsrGgfloatsintsstringstensorsgraphs) attributers r&attribute_to_kwargrLs; ~~:inn%55STUU~~  1   1   1   1   1    1  1 !! 1 !! 2   :inn%55KINNK[[\]^^ NNE ""r)ct|Dcgc]}|j|k(s|}}t|dkDr|dSdScc}w)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. rN)r#r~) item_name item_listitemitemss r& find_by_namerRsA( Bd499 +AT BE B5zA~58/4/ Cs55cRd}tt|D] }|||k(s |}|S)zC Helper function to return index of an item in a node list rn)rr~) elem_name elem_listelem_idxrs r&get_elem_indexrWs9H 3y> " Q<9 $H Or)cHtjjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr )inputsoutputr#s r& get_mul_noder\s! ;; $ ??r)cl|jj|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)filename identifiers r&generate_identified_filenamerd s+ ?? # #HMMJ$>$P QQr)c`ddl}ddlm}ddl}|j|j t dt |t dt ||j||d|jd|jd|jd |jy) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrsset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesripltrss r& apply_plotrws#ES[[1 , $K  *JJtZdJ+JJ~JJxII()HHJr)c B ddlddl}ddlddlmcmcm}ddlmcmcm}ddl m m m tjd|Gfddj}j!||}t#t$j&j)|dd 5}|j+|dddj-d}|j/d } g} t1|j3D]} || } | j5} t7| j9d |j;t7| j9d |j;g}t=t?|}| jA| }| jA|}|jC| |jE| ||jG| ||jI| }| jK||jM| tO| | D]}| jQ|| jS}|jU| |jW| ||jY| }| j[|| j]}t#t$j&j)|d d5}|j+|dddt$j^j9dddvr|jja|d}|jc}te|D]Y}|jg|}tj|jitj|jk[t#t$j&j)|dd 5}t1|j3D]} || } | j5} t7| j9d |j;t7| j9d |j;g}| dzt=t?|z}|j+||j+d dddy#1swYxYw#1swYxYw#1swYyxYw)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: c eZdZfdZy)*write_calibration_table..MyEncodercNt|fr|jSt|jr'|jt |j ddSt|r"|j jt |dSjj||S)Nz numpy.array)rrfCLS)rr) ryto_dictrztolistrrf __class__r0 JSONEncoderdefault)r%objryrzr{jsonnps r&rz2write_calibration_table..MyEncoder.default3s# K89{{}$#rzz* # s399~m\\#01"}}55CII##++D#6 6r)N)r0r1r2r)ryrzr{rrsr& MyEncoderr}2s  7 7r)r)clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG0)r1zcalibration.cache  )6r flatbuffersrs5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibrateryrzr{logginginfordumpsopenospathjoinwriterxBuildersortedkeysrfloatrrPrr CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndrwTrtTableStartDictVectorr~PrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrrr json_datafilerbuilderkey_value_listkeyrd_valuesrFrflat_key flat_value key_value main_dict cal_tablebufdict_lenrryrzr{rrs @@@@@r&write_calibration_tabler"s LLLL]] LL&'8&9:;77D$$7 ,) ># (,,y$/446 7 (,,x.335 6 CK '',))%0 w'2!!':6((1 i(#)& $$Wc..AB#3 '' 23!!#I 7# Wi0$$W-I NN9 .. C bggll3 9:D AT 3 zz~~*C0H<%%77Q? '')x ,A!q)I LL ) LL* + , bggll3 34c : d+0023 C&s+F~~'Hhll9d388:;hll8T2779:F#ICK 00E JJu  JJt    gL  s%$Q;6R/CR;RRRc|dk(jtj}|dk7jtj}|j}|j|z }|sy|t |zt |z }|dksJd|d|d||jtj}|||z| |zzz }|dkjdk(sJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrzn_zeros=z , n_nonzeros=z, eps1=)rrsr|sumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rts r&smooth_distributionr~sQu}}-H6//%--0KllnG'!J  w % "3 3D #:Q'- |74&QQ: 88EMM "DC(Nte{2 22D AI??  !! ! Kr)ctj|jd}td|jj DS)NF)load_external_datac3FK|]}tj|ywr!)ruses_external_data).0 intializers r& z*model_has_external_data..smz#66zBms!)rloadas_posixrgraphr%) model_pathmodels r&model_has_external_datars9 IIj))+ FE mUZU`U`UlUlm mmr)ct}|j|_tj|_i}dg|d<t |j|fddgi|}y)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)ropt_model_path sess_optionkwargs_s r&optimize_modelrsb!"K+9+B+B+DK(+A+R+RK( F%6$7F !,,. jH^G_jcijAr)cddi}|jr8|jD])}|j|j|ji+tj j ||y)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r&add_pre_process_metadatarsZ.0CDN (( :D  ! !488TZZ"8 9 :KK~6r)c|jr2|jD]#}|jdk(s|jdk(s#yy)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rrs r&model_has_pre_process_metadatarsA (( Dxx33 FY8Y  r)cddi}|jr8|jD])}|j|j|ji+tj j ||y)N onnx.inferrr)rrrs r&add_infer_metadatarsZ"$78N %% 4A  ! !155!''"2 3 4KK~6r)c|jr2|jD]#}|jdk(s|jdk(s#yy)NrrTFr)rrs r&model_has_infer_metadatars@ %% Auu $4G)G  r)c|jDcgc] }|jr|jdk(s|"}}t|dk7r td|dj}|Scc}w)Nrrz$Failed to find proper ai.onnx domainr) opset_importdomainr~r,version)ropsetai_onnx_domain opset_versions r&get_opset_versionrse).););m5<a?@@"1%--M  ns A A ct|}|}t|d|}|dkr9|tjjk(rt j d|dd}n?|dk(rt j d|dn |dkrt j d|dd}||k7r+tjj||}t|}|S) NrXz$The original model opset version is z, which does not support quantization to float 8. Please update the model to opset >= 19. Automatically update the model to opset 19. Please verify the quantized model.rze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z, which does not support quantization. Please update the model to opset >= 11. Automatically update the model to opset 11. Please verify the quantized model. ) rgetattrrrrSrwarningversion_converterconvert_version&save_and_reload_model_with_shape_infer)r weight_typertarget_opset_versionweight_quant_types r&update_opset_versionrs%e,M( ]KHr/43C3C3P3PP2=/B1 1 " " 2=/BM M  2=/B1 1 "},&&66u>RS7u= Lr)ct|d}tjjt |t |tj |j }t||j|S)Nz -inferred) rdrshape_inferenceinfer_shapes_pathrrrrunlink)rinferred_model_pathrs r&load_model_with_shape_inferrs`6z;O**3z?C@Sz2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray])ronnx.TensorProtorr;rr:rr:rz int | Nonerz str | Noner>r?)FF)rintrr@r>ztuple[bool, int])rrr>r)rbrrcrr>r)rI)g-C6?)rr)rrrr)rr)rrr>r<)rrr>r@)rrrr?r>r)rrr>r)rrr>r)r%rr>r:)r)rr>r)r>r)o __future__rrrrrenumrpathlibrrsrrrrr r onnx.helperr r r r onnx.referencer onnxruntimerrrrr ImportErrorrronnx.reference.op_runr __producer__ __version__ onnx_domain ms_domain QUANT_OP_NAMEr,DEQUANT_OP_NAMEr8rMODEL_SIZE_THRESHOLDrrryr r@r%rr8r?r[rKrfrMrQrOrSrWrUrrxrbrardrcrrrrrrrrrr rrrrr#r.r<rLrRrWr\rdrwrrrrrrrrrrrrrr*r-r0r3r6r9)ks0r&rQsL #   >>&QQ-PP@ ? 7      ,$2'!47 4Dqq SZ[fhiSjloHp Q'*q  t   #>#>L $  V!4  +%++g"6  +%++g"6!!;5;;x#8''  %  ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#>  BV[@\"]+%++b"={u{{1TX?Y!Z   ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q !  ;5;;q #DkekkRU]b]h]hFi"j+%++c"DkekkRT\a\f\fFg!h!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#={u{{1TX?Y"Z+%++b"={u{{1TX?Y!Z )+ A 13h<~"R#'"&"&;G ;G);G;G ;G ! ;G  ;G ;G);G~hl:=7:=D$( L L )L L   L  L " L L ^@ <>%%8''"#J0@R$Yx2n k 77!H7+,2-3/_ L D E $rsB]](]!]/$ ]/] ] ]]!],+],