pi UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZmZddlmZddlmZmZmZmZmZmZmZmZmZddlmZm Z ddl!Z!ddl"Z"ddl#Z"ddl$m%Z%dd l&m'Z(dd l)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z1ddl2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8ddl9m:Z:m;Z;ddlm?Z?m@Z@ddlAmBZBddlCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOddlPmQZQmRZRmSZSmTZTmUZUmVZVeriddlWmXZXmYZYmZZZddl#m[Z[ddl\m]Z]ddl^m_Z_m`Z`maZambZbddlcmdZdddlemfZfmgZgmhZhddlAmiZiddljmkZke d ZleeeggeffZmenekZoeepe!jfZrepZse"jjevd!ZwejevZyd d"ZzejGd#d$Z|Gd%d&ej$Z}Gd'd(eZ~eId)*Gd+d,e~ZGd-d.ZejGd/d0ZejGd1d2ZejGd3d4ZejGd5d6ZejGd7d8ZeeeeeefZiZd9ed:<Gd;d<ZiZd=ed><iZd?ed@<iZdAedB< d d dCZGdDdEeZ d dFZ d dGZddHZ d ddIZddJZddKZej*ddLZ ddMZ ddNZddOZe"j4e"j6e"j8e"j6ie"j:e"j<e"j>e"j@e"jBe"jDe"jFe"jHe"jJe"jLe"jNf Dcic]}||c}ZdPedQ< ddRZ ddTZGdUdVZGdWdXe1Z0GdYdZZe jZd[e j\\Zdd]ZGd^d_e?eeReZejGd`daZedidbee%jhdcddedfee%jhdgdhdijdkee%jhdldmdnjdoee%jhdpdqdrjdsee%jhdtdudvjdwee%jhdxdydwzd{ee%jhd|d}d~jdee%jhdddddee%jhddedee%jhddedee%jhddedee%jhddedee%jhdddddee%jhdddzdee%jhdddjdee%jhddedee%jhddedee%jhdddjdee%jhdddjdee%jhddedee%jhddedee%jhddedee%jhddedee%jhddedee%jhddedee%jhddedee%jhddedee%jhdd¬edee%jhdĄdŬedee%jhdDŽdȬedee%jhdʄdˬedee%jhd̈́dάedee%jhdЄdѬedee%jhdӄdԬedee%jhdքd׬edee%jhdلdڬedee%jhd܄dݬedee%jhd߄dedee%jhddedee%jhddeZded<ddZGddeEZGddeHZGddeZejGddZGddZeZGddZGddZe depZe dSeeZeree"jeTeeeedfffZGddeeefZGddZGddeeeZejGddZej*ddZGddZGdde@Zycc}w() annotationsN)ABCabstractmethod)autoEnum)chain) AnyCallablecastClassVarGeneric NamedTupleOptional TYPE_CHECKINGUnion)SelfTypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) ConfigModule) OrderedSet)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler)ShapePropagationOpsHandler) boolean_opsDeferredLineBasegenerate_assertget_current_backendIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique)ops OpsHandlerOpsValue ReductionType StoreModeV)IteratorMutableMappingSequence) GraphModule)CustomGraphModulePass)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNode)BlockShapeTypePythonWrapperCodegen_Tschedulecxtjtjrtj d|yy)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs `/opt/services/ai/voice_agent/venv/lib/python3.12/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrTZs*  /6<0cZeZdZUdZded<ded<d dZed dZd dZed d Z y )FileBackedGraphModulez Output of FX wrapper codegen. Exposes the same methods as ModuleType, but these map back to a GraphModule instead of Python source. r;gmzCallable[..., Any] compiled_fnc&tjddd|_tjtj |jj |j5}|j|jdddy#1swYyxYw)Nzw+z.pyF)modesuffixdelete) tempfileNamedTemporaryFileatexitregisterosremovenamewritevalue)selffs rS __post_init__z#FileBackedGraphModule.__post_init__isi!33eE    4==#5#56 ]] a GGDJJ    s "BBc.|jjSN)r^rdrgs rS__file__zFileBackedGraphModule.__file__ss}}!!!rUc |j|Srk)rYrgargss rScallzFileBackedGraphModule.callwst&&rUc.|jjSrk)rXcoderls rSrfzFileBackedGraphModule.valuezsww||rUNreturnNonerustr)rp list[Any]rur ) __name__ __module__ __qualname____doc____annotations__ripropertyrmrqrfrUrSrWrW_sF O## ""'rUrWc<eZdZdZdZdZeddZeddZy) WorkspaceZeroModerrGrc||k(s|tjk(r|S|tjk(r|Std|d|d)NzWorkspaceZeroMode.combine(, ))r UNINITIALIZEDNotImplementedErrorabs rScombinezWorkspaceZeroMode.combinesK 6Q+999H !// /H!$>qe2aU!"LMMrUcF|rtjStjSrk)r ZERO_ON_CALLr) zero_fills rS from_boolzWorkspaceZeroMode.from_bools $11 1 ...rUN)rrrrrur)rboolrur) rzr{r|rrZERO_PER_GRAPH staticmethodrrrrUrSrrs9MLNNN//rUrc4eZdZdZeddZeddZy) CodegenSymbolzP An IR object possibly corresponding to a variable in the wrapper code. cyrkrrls rSget_namezCodegenSymbol.get_name rUcyrkrrls rS get_examplezCodegenSymbol.get_examplerrUNrwruz!Union[torch.Tensor, sympy.Symbol])rzr{r|r}rrrrrUrSrrs/    rUrT)frozenc&eZdZUdZded<ded<ded<ded <d Zded <ejZd ed <e dddZ e d dZ e d!dZ e d!dZ d"dZeZd#dZd$dZd%dZed%dZeZeZeZd&dZd'dZd'dZd(dZd)dZd*dZy)+ WorkspaceArga2A temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.Exprcountr zero_mode torch.devicedevicerx outer_namews_ptr inner_name torch.dtypedtypecP|ttjjSrk)nextr7graph workspace_id)prefixs rS unique_namezWorkspaceArg.unique_names!$qww334566rUc|j|jk(xr4|j|jk(xr|j|jk(Srk)rrrrs rScan_joinzWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rUct|j|jztj|j|j|j |j |j|jSN)rrrrrr) rrrrrrrrrrs rSjoinzWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rUc|j|jk(r2|j|jk(r|j|jk(sJtt j |j |j tj|j|j|j|j|j|jSr) rrrrsympyMaxrrrrrrs rSmaximumzWorkspaceArg.maximums GGqww 188qxx#7ALLALL|jjSrk) get_layoutrrls rSrzWorkspaceArg.get_examples ,,..rUcfddlm}||j|j|jgdgS)Nr)r?rG)rrsizestride)irr?rrr)rgr?s rSrzWorkspaceArg.get_layouts.$;;****3   rUc"|jSrk)rrls rSlayoutzWorkspaceArg.layouts  rUc6tjjSrk)rSZerorls rS get_offsetzWorkspaceArg.get_offsetsww||rUc|jgSrk)rrls rSget_sizezWorkspaceArg.get_sizes |rUc8tjjgSrk)rrOnerls rS get_stridezWorkspaceArg.get_strides }rUc|jSrk)rrls rSrzWorkspaceArg.get_names rUcy)NFrrls rS get_is_pinnedzWorkspaceArg.get_is_pinnedsrUcgSrkrrls rSget_inputs_that_alias_outputz)WorkspaceArg.get_inputs_that_alias_outputs rUN) workspace_)rrxrurx)rrrrrur)rrrrrur)rur)rurr)rur?)rur)rulist[sympy.Expr]rw)rur)ru list[str])rzr{r|r}r~rtorchuint8rrrrrrrget_device_or_errorrrrrrget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrrrrUrSrrs    OJE;$77        %/ !!!O&!rUrceZdZddZddZy)TritonScratchWorkspacec ||_||_yrk)r_generate_dtype_str)rgrgenerate_dtype_strs rS__init__zTritonScratchWorkspace.__init__ s #5 rUc"|jSrk)rrls rSrz)TritonScratchWorkspace.generate_dtype_strs''))rUN)rintrCallable[..., str]rw)rzr{r|rrrrUrSrr s 6*rUrcpeZdZUded<ded<ded<ej j Zded<dZd ed <y) TensorArgrxrdbufferrrroffsetN Optional[str]alias_of) rzr{r|r~rrrrrrrUrSrrs. I K FJ%"Hm"rUrc4eZdZUded<ded<eddZy)SizeArgrxrdrexprcyrkrrls rSrzSizeArg.alias_of srUNrur)rzr{r|r~rrrrUrSrrs I  rUrceZdZUded<y) ConstexprArgrxrdNrzr{r|r~rrUrSrr%s IrUrc6eZdZUded<ded<ded<ded<y) TMADescriptorArgrxrdapi_typezOptional[list[sympy.Expr]] block_shapeOptional[torch.dtype]rNrrrUrSrr*s IM++  rUrc>eZdZUded<ded<dZded<dZded<y) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNOptional[WrapperConstructor]cpp_wrapper_codegenfx_wrapper_codegen)rzr{r|r~rrrrUrSrr2s&%%''8<5<7;4;rUrzdict[str, DeviceCodegen]device_codegensceZdZddZddZddZddZddZddZddZ ddZ dd Z dd Z dd Z dd Zdd ZddZddZddZ d ddZy)DeviceOpOverridesctrkrrgrds rSimport_get_raw_stream_asz*DeviceOpOverrides.import_get_raw_stream_as@!!rUctrkrrg device_idxs rS set_devicezDeviceOpOverrides.set_deviceCrrUctrkrrls rS synchronizezDeviceOpOverrides.synchronizeFrrUctrkrr s rS device_guardzDeviceOpOverrides.device_guardIrrUctrkrrls rScpp_device_guardz"DeviceOpOverrides.cpp_device_guardLrrUctrkrrls rScpp_aoti_device_guardz'DeviceOpOverrides.cpp_aoti_device_guardOrrUctrkrrls rScpp_stream_guardz"DeviceOpOverrides.cpp_stream_guardRrrUctrkrrls rScpp_aoti_stream_guardz'DeviceOpOverrides.cpp_aoti_stream_guardUrrUctrkrrls rScpp_getStreamFromExternalz+DeviceOpOverrides.cpp_getStreamFromExternalXrrUctrkrrls rS kernel_headerzDeviceOpOverrides.kernel_header[rrUctrkrrls rS kernel_driverzDeviceOpOverrides.kernel_driver^rrUctrkrrls rScpp_stream_typez!DeviceOpOverrides.cpp_stream_typearrUctrkrrls rSaoti_get_streamz!DeviceOpOverrides.aoti_get_streamdrrUctrkrrls rScpp_kernel_typez!DeviceOpOverrides.cpp_kernel_typegrrUctrkrrls rScpp_device_ptrz DeviceOpOverrides.cpp_device_ptrjrrUctrkrrls rStma_descriptor_helpersz(DeviceOpOverrides.tma_descriptor_helpersmrrUNctrkr)rgidx workspacers rS cpp_scratchzDeviceOpOverrides.cpp_scratchps "!rUrdrxrurx)r rrurxrwrk)r*rr+rrrruzOptional[tuple[list[str], str]])rzr{r|rr rrrrrrrrrr r"r$r&r(r,rrUrSrr?s~""""""""""""""""TX""#9"CP" ("rUrzdict[str, DeviceOpOverrides]device_op_overrides_dictz*dict[str, Optional[CustomGraphModulePass]]custom_backend_passesz!dict[str, Optional[ConfigModule]]custom_backend_codegen_configsct||||t|<|t|<|r)t|tr|t usJd|dt |t |<y)Nzdevice_custom_config=z: cannot be the same as the default inductor config config=)rrr/ isinstancerr r0)rdevice_schedulingdevice_wrapper_codegendevice_cpp_wrapper_codegendevice_fx_wrapper_codegendevice_custom_passdevice_custom_configs rSregister_backend_for_devicer9sv,"! OF %7&! +\ :$F2 %#%%`Y_Xa b   3 .B"6*rUceZdZeZeZeZeZeZeZ eZ eZ eZ eZ y)BackendFeatureN)rzr{r|rFOREACH BUCKETIZEINPLACE_BUFFERSMASKED_SCATTER_WITH_INDEXSCANSORTTUPLE_REDUCTIONPREFER_STORE_LOOP_ORDERTRITON_TEMPLATESREDUCE_TO_SINGLE_ELEMENTrrUrSr;r;sLfGIfO $ 6D 6DfO"fv#vrUr;c:| tStt|tjr |j }n7t|t sJt ||}tj|}t|}|sJ|d}|j|Srk) rinit_backend_registrationr2rrtyperxget_scheduling_for_deviceget_backend_features)r device_typescheduling_ctorrs rSrJrJs~|&%,,'kk &#&4V 4& k*/ z+init_backend_registration.. s?|F,>,>? KrUcuda)rirhc6tj|Srk)r cuda_backend)r cuda_backendss rSrmz+init_backend_registration..sA}V-@-@A*MrUxpumpsmtia privateuseoner)_get_custom_mod_func SchedulingrICppWrapperCodegenre)#rgr[cpp_wrapper_cpur\cpp_wrapper_cpu_array_refr]cpp_wrapper_gpur^cpp_wrapper_mpsr_cuda_combined_schedulingr`rhrarsrbpython_wrapper_mtiarcrirdwrapperrI wrapper_fxirrerIr9r aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationrv RuntimeError)r[r\r]r^r_r`rarbrcrdrIreprivate_backendrvr3rrrrlrqs @@rSrGrGs #.@..@($6(-. '/ && $  K ""99 "  !(0-&  $  M    !'/#      !'/#      !(0#       hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K !56H!I  _9L+#%#'& :M_  ? +$   s/5E** E65E6cLddlm}g|t||j|S)Nr)FlexibleLayout)rrr-contiguous_strides)index index_varssizesrs rSindex_prevent_reorderingrTs, $ UU TIj.*K*KE*RS TTrUc|t|<yrk)r.)rdevice_op_overridess rSregister_device_op_overridesr_s(;V$rUct|tsJt|tsddlm}m}ddlm}ddl m}ddl m}t|S)NrG)cpu_device_op_overridesmps_device_op_overrides)r) r2rxrHr.rrrnrrtrr)rrrrmtia_op_overridesxpu_op_overridess rSget_device_op_overridesres5 fc "0DL0 " #F-B@ #F ++rUzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcr|tvrtjS|dvrd|vr|dS|dS|dvrtjS|dvrtjS|dk(rd|vr|dS|dS|dk(rd|vr|dS|dS|d vr$|d}t j j|S|d k(rd|vr|dS|d Sy ) zK Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_exprr)randrandn) get_index randint64 load_seed reductionrGconstant)loadstorestore_reductionto_dtype_bitcastN)r&rrfloatint64r7rr)op_namerpkwargsbuf_names rSdeduce_output_dtype_by_namers +-zz  #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rUCSEVariableTypect}tjjr'|dk(r"|j d|dt |dytjj r|dk(rddlm}m }t||sJt||tjk(r|jrd|d }n.d |d |d }n$d |d}|jrd|d}d|d||d}|j d|dyyy)Nriztl.static_assert(z .dtype == rrgrG)CppCSEVariable DTYPE_TO_CPPzIsVecMaskType::valuezstd::is_same_v || std::is_same_vz decltype(z typename z ::value_typezstd::is_same_vzstatic_assert(z);)r)r test_configsruntime_triton_dtype_assert writeliner0static_cpp_dtype_assert cpp_utilsrrr2rHrris_vec)rvarrbackendrr is_same_dt c_var_types rS check_dtypers"#G 667h;N,SEK*R89!:J 4rUc`eZdZd dZd dZd dZd dZd dZddZe ddZ e ddZ y )DataTypePropagationc||_d|jji|_|jj D]\}}|j|j|< yNroot)body root_blockrgraphs subblocksitems)rgrkvs rSrzDataTypePropagation.__init__sU DOO))B  NN((* %DAqWWDKKN %rUc|j}|Dcgc]9}t|tjjs(|j dk7s8|;}}t |dk(rytd|D}|sytjtj|Dcgc])}|jtjj+c}Scc}wcc}w)N placeholderrc3K|]K}tj|jvxr)|jtjjduMywrk)OptimizationContextkeymetar).0ns rS zBDataTypePropagation.deduce_node_dtype_by_inputs..sS)  # #qvv - B*../55TA B) sAA)all_input_nodesr2rfxNodeoplenall functoolsreduce promote_typesrrrr)rgnodeinputsr input_nodesall_input_nodes_propagateds rSdeduce_node_dtype_by_inputsz/DataTypePropagation.deduce_node_dtype_by_inputss%% Auxx}}!=!$$-BWA   { q %() !) & " *   T$++->>+ ;; ! !"3 455d; ;8 ++ L    //55rUcn|jsJd}|jD]}tj|jvr|jtj}n t}|j ||_||jtj<|j dk(s|j }|S)Nr)nodesrrrrrr)rgr graph_dtyperopt_ctxs rSrz#DataTypePropagation.propagate_graph s{{{-1 KK ,D"&&$))3))$7$;$;<-/ 2248GM18DII)-- .{{h&%mm  ,rUc>|j|jdSr)rrrls rS propagatezDataTypePropagation.propagates##DKK$788rUc.||jSrk)r)clsrs rSpropagate_loopbodyz&DataTypePropagation.propagate_loopbody s4y""$$rUcddlm}ddlm}t ||sJt |t |j |sJt |j tj|j S)NrrA)rE) loop_bodyrB schedulerrEr2rH_bodyrr)rrrBrEs rSpropagate_scheduler_nodez,DataTypePropagation.propagate_scheduler_node$sX(-$ .:T :.$**h/Adjj1AA/"55djjAArUN)rrBrurv)r torch.fx.Noderur)rrrur)rztorch.fx.Graphrur)rur)rrBrur)rrErur) rzr{r|rrrrrr classmethodrrrrUrSrrsJ% * 6:$9%%BBrUrcDeZdZddd dfdZddfd ZxZS)rT)simplifypc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) r2rExprhasattrr7rrrsuperdoprint)rgrrr __class__s rSr zPythonPrinter.doprint/sK  44*9U77##,,T2Dwt$$rUct|tjrd|j|dSt||||SN(r)r2rMod_printr parenthesize)rgitemlevelstrictr s rSrzPythonPrinter.parenthesize7s@ dEII &t{{4()+ +7'eV< ?@@rUc tjdtj}tj|tj |tj tj|Sr)r2rrrrr9r*neg)rones rSsigmoidzOpDecompositions.sigmoidnsCll1ekk*{{3SWWSWWQZ-@ ABBrUcrtj|tjdtjSNr)r2rrrrrs rSreluzOpDecompositions.reluss"{{1cll1ekk:;;rUcVtjtj|||Srk)r2r9r"ryzs rSfmazOpDecompositions.fmawswwswwq!}a((rUcTtjtj||Srk)r2rfloorrrs rS floor_to_intzOpDecompositions.floor_to_int|||CIIaL%00rUcTtjtj||Srk)r2rceilrIs rS ceil_to_intzOpDecompositions.ceil_to_ints||CHHQK//rUcTtjtj||Srk)r2rtruncrIs rS trunc_to_intzOpDecompositions.trunc_to_intrKrUc tj||}tjtj|tjdt j tjtj|tj|}tj|tj|||Sr@) r2modand_nerrrsignbitwherer9)rrrconds rS remainderzOpDecompositions.remaindersy GGAqMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq!}a00rUcTtjtj||Srk)r2rroundrIs rS round_to_intzOpDecompositions.round_to_intrKrUN)rfOpVarTrur^rr^rur^)rr^rDr^rEr^rur^)rr^rrrur^rr^rr^rur^)rzr{r|r}rrr r#r(r+r-r3r5r7r:r>rArFrJrNrQrZr]rrUrSrr@s}<<CC<<CCRRQQMMAACC<<))1100111111rUrz[a-z0-9_.]+|\([^)]*\)|)flagsc|ddk7st|dkryd}t|ddD]3\}}|dk(r|dz }n |dk(r|dz}|dk(s!|t|dz k7s3y|dk(sJy)Nrr rFrGrT)r enumerate)stringrichars rS_all_in_parensrgs ayC3v;? EVABZ(4 3; QJE S[ QJE A:!s6{Q.  A:: rUcHeZdZed dZed!dZed"dZed#dZed$dZed$dZ ed$dZ ed$dZ ed$d Z ed%d Z ed&d Z d' d(d Z d)dZd*dZ d+ d,dZd-dZ d.dZ d/dZ d0dZ d1 d2dZd3dZdej4d dd d4dZd5dZd6dZed7dZed8dZ ed9dZ!y): OpOverridescrt|ts tj|s t |r|Sd|dSr )r2 CSEVariable_RE_PAREN_NOT_NEEDED fullmatchrg)rds rSparenzOpOverrides.parens9 v{ +#--f5f%M6(!}rUct|Srk)repr)rfrs rSrzOpOverrides.constants E{rUc2dtj|S)N~rirnrs rS bitwise_notzOpOverrides.bitwise_nots;$$Q'())rUc2tj|dS)Nz == 0rs)rs rS logical_notzOpOverrides.logical_nots##A&'u--rUc\tj|dtj|S)Nz & rsrrDs rS bitwise_andzOpOverrides.bitwise_and+##A&'s;+<+> rsrxs rSbitwise_right_shiftzOpOverrides.bitwise_right_shiftrrUc.tj||Srk)r2rrs rS int_truedivzOpOverrides.int_truedivs {{1a  rUcTtj|tj|Srk)r2rrInteger)rdrs rSrzOpOverrides.load_seedsxxemmF344rUTc*tt|Srk)r.rx)rgrrcheckwrap_negs rSindirect_indexingzOpOverrides.indirect_indexings"#c(++rUcDtt|jd)Nz,: check_bounds should be handled by CSEProxyrrHrzrgrrloweruppers rS check_boundszOpOverrides.check_boundss'"Dz""##O P  rUcDtt|jd)Nz$: load should be handled by CSEProxyrrgrdrs rSrzOpOverrides.loads%!Dz""##G H  rUNcDtt|jd)Nz%: store should be handled by CSEProxyrrgrdrrfr[s rSrzOpOverrides.stores'"Dz""##H I  rUcDtt|jd)Nz/: store_reduction should be handled by CSEProxyrrgrdrrfs rSrzOpOverrides.store_reductions%!Dz""##R S  rUcDtt|jd)Nz): reduction should be handled by CSEProxyrrgr src_dtypereduction_typerfs rSrzOpOverrides.reductions'"Dz""##L M  rUcDtt|jd)Nz$: scan should be handled by CSEProxyrrgdtypes combine_fnvaluess rSscanzOpOverrides.scan s'"Dz""##G H  rUcDtt|jd)Nz$: sort should be handled by CSEProxyrrgrrstable descendings rSsortzOpOverrides.sorts'"Dz""##G H  rUcDtt|jd)Nz): bucketize should be handled by CSEProxyrrgr boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rS bucketizezOpOverrides.bucketize#s'"Dz""##L M  rUcDtt|jd)Nz2: halide_clamp only implemented for Halide backendr)rgrfrrs rS halide_clampzOpOverrides.halide_clamp1s%!Dz""##U V  rUrG) constraintsris_purepackcDtt|jd)Nz<: inline_asm_elementwise only implemented for Triton backendr)rgasmrrrrrs rSinline_asm_elementwisez"OpOverrides.inline_asm_elementwise6s'"Dz""##_ `  rUcDtt|jd)Nz.: ops.output should not appear at codegen timeAssertionErrorrHrzros rSrzOpOverrides.outputCs%Dz""##Q R  rUcDtt|jd)Nz3: ops.placeholder should not appear at codegen timerrgrs rSrzOpOverrides.placeholderHs%Dz""##V W  rUc0dfd }|_d|_|S)NcJtt|jd)Nz does not implement ops.r)rgrprrds rS unimplementedz1OpOverrides._unimplemented..unimplementedOs*%:&&''?vF rUT)rgrirpr rr rur^)rzis_unimplemented)rdrs` rS_unimplementedzOpOverrides._unimplementedMs   "& )- &rUcpt||d}tt|d}| xs||k(xs t|ddS)NrF)getattrr3)rrdfn default_fns rS_is_unimplementedzOpOverrides._is_unimplementedXs? S$ %Zt4 vSz)SWR9KU-SSrUcP|dvsJ|tjD]\}}t||}|/|j|s&t |||j |C||j vsJd|d|j||_t ||t|y)N)rirgcppvecrhrszmultiple definitions of z on ) pointwise_overrides_datarrrsetattrr__dict__rzr)rrfuncnamedataimpls rS_initialize_pointwise_overridesz+OpOverrides._initialize_pointwise_overrides^sEEMvME6<<> ;NHd4(D|((2C3+=+=h+GHs||3.xjS\\NK3!) X|D'9: ;rU)rdr^rur^)rfzUnion[bool, float, int]rrrur^r_)rr^rur^)rr^rDr^rur^r`)rdrxrr^rur^TT) rr^rUnion[sympy.Expr, int]rrrrru sympy.Symbol rrrrrrrrrurv)rdrxrrrur^rk) rdrxrrrfr^r[r6rurv)rdrxrrrfr^rurv) rrrrrr5rf!Union[OpVarT, tuple[OpVarT, ...]]rur)rtuple[torch.dtype, ...]rzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rtuple[OpVarT, ...]rur) rrrrrrrrrurNN)rr^r.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]rr^rrrrr Optional[tuple[str, sympy.Expr]]rzOptional[OpVarT]rur^)rfr^rrrrrur^)rr^rrxrrrrrrrrrur^)rpr^rurv)rrrur^)rdrxruzCallable[..., OpVarT]rdrxrur)rrxrurv)"rzr{r|rrnrrtrvryr|r~rrrrrrrrrrrrrrrr&rrrrrrrrrUrSriris'**..BBBBBBCCCC!! 55 , ,%, ,  ,  ,  &0 9= FJ   NR  * 39 AJ        &  1  +    '     #       '  #        $48+/    C  !  $    1  )      &*"]]      #             TT  ; ;rUric|eZdZUded<ded<dZded<dZded<ejZd ed <dZ ded <dZ ded <y) OverridesDatarxrdrrgNzOptional[Callable[..., str]]rirrtype_promotion_kindrhrs) rzr{r|r~rirrDEFAULTrrhrsrrUrSrrosQ I +/F (/+/F (/'//8,0F (/(,C %,rUrairy_aicd|dS)Nzairy_ai_forward(rrrs rSrmrms(1-rUspecial_airy_ai)rrgrd bessel_j0cd|dS)Nzbessel_j0_forward(rrrs rSrmrm*1#Q/rUcd|dS)Nz libdevice.j0(rrrs rSrmrm=1-rUspecial_bessel_j0)rrgrird bessel_j1cd|dS)Nzbessel_j1_forward(rrrs rSrmrmrrUcd|dS)Nz libdevice.j1(rrrs rSrmrmrrUspecial_bessel_j1 bessel_y0cd|dS)Nzbessel_y0_forward(rrrs rSrmrmrrUcd|dS)Nz libdevice.y0(rrrs rSrmrmrrUspecial_bessel_y0 bessel_y1cd|dS)Nzbessel_y1_forward(rrrs rSrmrmrrUcd|dS)Nz libdevice.y1(rrrs rSrmrmrrUspecial_bessel_y1digammacd|dS)Nz calc_digamma(rrrs rSrmrms aS*rUc |dS)Nz .digamma()rrs rSrmrms A3j)rU)rrgrrdr+cd|dS)Nz calc_erfcx(rrrs rSrmrm A3a(rUcd|dS)Nzlibdevice.erfcx(rrrs rSrmrms+A3a0rU special_erfcxrFcd|d|d|dS)Nz std::fma(rrrrCs rSrmrmsis"QCr!A6rUcd|d|d|dS)Nzfmadd(rrrrCs rSrmrmss"QCr!A6rUcd|d|d|dS)Nzlibdevice.fma(rrrrCs rSrmrmss"QCr!A>rU)rrgrrirdigammacd|d|dSNz calc_igamma(rrrrxs rSrmrm<s"QCq1rUigammaccd|d|dSNz calc_igammac(rrrrxs rSrmrm=2aS2rUgammainccd|d|dSrrrxs rSrmrmrrUspecial_gammainc gammaincccd|d|dSrrrxs rSrmrmrrUspecial_gammaincci0cd|dS)Nzcalc_i0(rrrs rSrmrm1orUcd|dSNzlibdevice.cyl_bessel_i0(rrrs rSrmrm3A3a8rUc |dS)Nz.i0()rrs rSrmrms A3erU)rrgrirrdi0ecd|dS)Nz calc_i0e(rrrs rSrmrm !A&rUc |dS)Nz.i0e()rrs rSrmrms A3frU special_i0ei1cd|dS)Nzcalc_i1(rrrs rSrmrmrrUcd|dSNzlibdevice.cyl_bessel_i1(rrrs rSrmrmrrU special_i1i1ecd|dS)Nz calc_i1e(rrrs rSrmrmrrU special_i1elog_ndtrcd|dS)Nzcalc_log_ndtr(rrrs rSrmrmsqc+rUspecial_log_ndtrmodified_bessel_i0cd|dS)Nzmodified_bessel_i0_forward(rrrs rSrmrm3A3a8rUcd|dSrrrs rSrmrmrrUspecial_modified_bessel_i0modified_bessel_i1cd|dS)Nzmodified_bessel_i1_forward(rrrs rSrmrmr%rUcd|dSrrrs rSrmrmrrUspecial_modified_bessel_i1modified_bessel_k0cd|dS)Nzmodified_bessel_k0_forward(rrrs rSrmrmr%rUspecial_modified_bessel_k0modified_bessel_k1cd|dS)Nzmodified_bessel_k1_forward(rrrs rSrmrmr%rUspecial_modified_bessel_k1ndtrcd|dS)Nz calc_ndtr(rrrs rSrmrms 1#Q'rU special_ndtrndtricd|dS)Nz calc_ndtri(rrrs rSrmrmrrU special_ndtri polygammac *|d|d|d|d|d|d S)Nz == 0 ? calc_digamma(z) : (z == 1 ? trigamma(z) : calc_polygamma(rz))rrxs rSrmrm s8S%aSaS0A!DWXYWZZ\]^\__a brUscaled_modified_bessel_k0cd|dS)Nz"scaled_modified_bessel_k0_forward(rrrs rSrmrm:1#Q?rU!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cd|dS)Nz"scaled_modified_bessel_k1_forward(rrrs rSrmrmr<rU!special_scaled_modified_bessel_k1spherical_bessel_j0cd|dS)Nzspherical_bessel_j0_forward(rrrs rSrmrms4QCq9rUspecial_spherical_bessel_j0zetacd|d|dS)Nzzeta(rrrrxs rSrmrm!s52aS*rU special_zetachebyshev_polynomial_tcd|d|dS)Nzchebyshev_polynomial_t_forward(rrrrxs rSrmrm&:1#Rs!DrUspecial_chebyshev_polynomial_tchebyshev_polynomial_ucd|d|dS)Nzchebyshev_polynomial_u_forward(rrrrxs rSrmrm+rIrUspecial_chebyshev_polynomial_uchebyshev_polynomial_vcd|d|dS)Nzchebyshev_polynomial_v_forward(rrrrxs rSrmrm0rIrUspecial_chebyshev_polynomial_vchebyshev_polynomial_wcd|d|dS)Nzchebyshev_polynomial_w_forward(rrrrxs rSrmrm5rIrUspecial_chebyshev_polynomial_wlegendre_polynomial_pcd|d|dS)Nzlegendre_polynomial_p_forward(rrrrxs rSrmrm:9!BqcCrUspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcd|d|dS)Nz'shifted_chebyshev_polynomial_t_forward(rrrrxs rSrmrm?B1#Rs!LrU&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucd|d|dS)Nz'shifted_chebyshev_polynomial_u_forward(rrrrxs rSrmrmDrZrU&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcd|d|dS)Nz'shifted_chebyshev_polynomial_v_forward(rrrrxs rSrmrmIrZrU&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcd|d|dS)Nz'shifted_chebyshev_polynomial_w_forward(rrrrxs rSrmrmNrZrU&special_shifted_chebyshev_polynomial_whermite_polynomial_hcd|d|dS)Nzhermite_polynomial_h_forward(rrrrxs rSrmrmSs82aSBrUspecial_hermite_polynomial_hhermite_polynomial_hecd|d|dS)Nzhermite_polynomial_he_forward(rrrrxs rSrmrmXrVrUspecial_hermite_polynomial_helaguerre_polynomial_lcd|d|dS)Nzlaguerre_polynomial_l_forward(rrrrxs rSrmrm]rVrUspecial_laguerre_polynomial_lzdict[str, OverridesData]rctfdtjjtjjtjj tjj fDS)Nc3&K|]}|v ywrkr)rrrds rSrz$is_buffer_removed..ds   )anyr7rremoved_bufferskernelinplaced_to_removerds`rSis_buffer_removedrvcsU  GG # # HH $ $ GG & & HH ' '   rUc4eZdZdZdfd ZddZddZxZS) DeferredLinezHA line that can be 'unwritten' by adding name to V.graph.removed_bufferscVt||||_t|trJyrk)rrrdr2r')rgrdliner s rSrzDeferredLine.__init__rs+  d$45555rUcFt|js |jSyrk)rvrdrzrls rS__call__zDeferredLine.__call__ws +99 rUc.t|j|Srk)rxrd)rgrzs rS _new_linezDeferredLine._new_line|sDIIt,,rU)rdrxrzrxr)rzrxrurx)rzr{r|r}rr|r~rrs@rSrxrxosR6  -rUrxceZdZdddZy) BracesBuffercHtjdfd }|S)Nc3KtD](}jdxjdz c_*t D](}xjdzc_jd*dt D](}jdxjdz c_*tD](}xjdzc_jd*yw)N{rG})ranger_indent)_rrgs rSctxz BracesBuffer.indent..ctxs6] "s# !  "F7^ $ ! s# $ F7^ "s# !  "6] $ ! s# $sC C#)ruIterator[None]) contextlibcontextmanager)rgrrs`` rSindentzBracesBuffer.indents$  " " $ # $u rUN)rG)rrruz'contextlib.AbstractContextManager[None])rzr{r|rrrUrSrrsrUrc"eZdZUded<ded<y)InplacedBufferrxrr other_namesNrrrUrSrrs OrUrc.eZdZUded<dZded<ddZy) ArgNamerxrdFr is_constexprcB|j|jrdSdS)Nz : tl.constexprr)rdrrls rS full_namezArgName.full_names*))$2C2C.LMMLMMrUNrw)rzr{r|r~rrrrUrSrrs IL$NrUrceZdZddZy) RemovedArgcy)NREMOVEDrrls rS__str__zRemovedArg.__str__srUNrw)rzr{r|rrrUrSrrsrUrceZdZe ddZddZddZeddZddZddZ ddZ ddZ dd Z dd Z dd Zd d Zd!d Zd"dZd#dZ d$ d%dZ d&dZd'dZd(dZd)dZy)* KernelArgsc~|j|t}t|tr|t |x||<}|S|Srk)getrr2rr)rodictrdresult new_results rS_lookupzKernelArgs._lookupsD */4)A fj )*0#e*'> >E$K*  rUcJi|_i|_i|_i|_g|_yrk) input_buffersoutput_buffersinplace_buffersrworkspace_argsrls rSrzKernelArgs.__init__s)-/ACMO/1 24rUc djdjtt|j|j |j |jgS)NzKernelArgs({})r)formatrmaprprrrrrls rS__repr__zKernelArgs.__repr__sS&& II**++,,    rUc"t|tSrk)r2rrus rS_buffer_is_marked_removedz$KernelArgs._buffer_is_marked_removeds$ ++rUc:tjjr4tjjjj ||}|tjj vsJ|||j vrtt|j |S||jvr'tt|j|jS|jdr|jd|j|S|jd|j|S)Nseedin_ptr)r7rrmutation_real_namerrrrr rxrrrrrrrs rSinputzKernelArgs.inputs 77  77$$77;;D$GD1772228D82 4&& &T0067 7 4'' '(<(R!R  /01[)C03D  ,03D  -! s E3E8cJt|tj|tjj tj }t|jD]\}}tj||r?|j}tj|||j|<|j|fcS|j|jk7r|j|jk7rJ||jj||jdfS)a Allocate or extend a workspace buffer of nbytes bytes. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nbytes (sympy.Expr): The number of bytes to allocate. zero_fill (bool): Whether to initialize the buffer to zero. Returns: Tuple[str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - offset: An integer representing the byte offset in the workspace. )rrrrr)rrrr7rget_current_device_or_throwrrcrrrrrrr)rgnbytesrargre existing_argrs rSr+zKernelArgs.workspaces.'11)<77668#//1   ))<)<= OA|$$\37%++)5):):<)M##A&#..66''3>>9 ++s~~=  >   ""3'~~q  rUc tjj}t|tj t jdd|jd|j|}|jD]*}|j|jk(s||k(r#J||f|jj||jS)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_r)rrrrrr) r7rrrrrruint32rHrrrr)rgmin_sizecurrent_devicerrs rS semaphoreszKernelArgs.semaphores-s<<>'66,, $^%8%8$9>;O;O:PQ!  !// @L&&#..8l*?S,,??* @ ""3'~~rUcft|tsJt||ftj|}||j vr|j |S|j j vr0tfd|j j D|j |<S)Nc3FK|]}|jsdyw)rGN)r)rrrds rSrz)KernelArgs.seed_offset..QsU1!,,tBTQUs!!)r2rrHrrrrsum)rgrdrfs ` rS seed_offsetzKernelArgs.seed_offsetIs%%;U U';;% e$ DMM !==' ' 4=='') )&U (<(<(>UUVW  $ e rUct|tjsJt||f|jdk(rd|j |<y|j d|j |S)Nrks)r2rSymbolrHrdrrrs rSrzKernelArgs.sizeVsX$ -AT D/AA- 99 "(DMM$ ||D$--66rUct|jj|jj|jjSrk)rrkeysrrrls rS call_nameszKernelArgs.call_names]sA    # # %t':':'?'?'A4==CUCUCW  rUc|jj|d}|t|ts |jS|j j|d}|t|ts|S|j j|dS)z; Returns inner name of a given outer name. N)rrr2rrrr)rgrdinplacedrs rSarg_namezKernelArgs.arg_namebs}''++D$7   8Z(H&& &))--dD9  ":k:+N !!%%dD11rUc|Srkr)rgrrs rS wrap_ptr_argzKernelArgs.wrap_ptr_argns rUct|Srk)rx)rgrs rS wrap_size_argzKernelArgs.wrap_size_argqs 4yrUNcddlm}|ddlm}|}g}g}g}t|jj D]}t |tr|jd}|j} tjj|} || } |j| d| |j|j|| |j| d|jj!D]\}} ||jvrtjj|} || } |jd| d| |j|j|| |jd| d|j"j!D]\}} ||jvst | tr%tjj|} || } |j| d| |j|j|| |j| d|j$j!D]\}} |jd|d| |j|j'||jd|tjj(sltjj(j+||j,rJd |||fS) NrG) INDEX_TYPE)rrz* *zconst  zWorkspace not supported on CPU )rrrr1rrr2rrrr7rrrrrrrrr wrapper_codeensure_size_computedr) rgdtype_to_cpp_typerr call_argsarg_defs arg_typesrouterinnerr cpp_dtype maybe_inners rS cpp_argdefszKernelArgs.cpp_argdefsts *  $ / ,   t33::<= .H(J/((,E''EGG%%e,E)%0I OOykE73 4   T..ue< =    {!_ - .!..446 4LE5,,,GG%%e,E)%0I OOfYKr%9 :   T..ue< =   vi[2 3 4#'"5"5";";"= . E;,,, ; 0SGG%%e,E)%0I OOykK=9 :   T..ue< =    {!_ - .!MM//1 ALE5 OOfZL%9 :   T//6 7   vj\2 3ww##$$99%@  A &&I(II&I--rUc g}g}g}g}t|jjD]}t|tr|j t |j|j |jd|j tjj|jd|j t|j|jdtjj|jdt|jj|j jD]\}}||jvst|tr%|j t ||j ||j tjj||j t||tjj||j"jD]\}}|j t ||j ||j t%||j t'||tjj(stjj(j+||j,D]m}|j t |j|j |j.|j ||j |j0o||||fS)Nr)rdrr)r1rrr2rrrrrr7rrrrrrrrrHrrrrrr) rgrrrprecompile_argsrrrrs rSpython_argdefszKernelArgs.python_argdefss#%! ! /1t33::<= H(J/ OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  "    $ $ &(;(;(A(A(C  LE5,,, 5*0M OOGEN +   U #   QWW..u5 6  " " ''++E2  !MM//1 ALE5 OOGEN +   U #   T%[ )  " "75%#8 9ww##$$99%@  A&& (C OOGCNN3 4   S^^ ,  " "3 '   SYY '  ( OY>>rUc#Kt|jjD]}t|tr|j D]}|t jjvs|t jjvr<||jvr|j||jf||jvsvtt|j||jfywrk)r1rrr2rrr7rrtrsrrrr rx)rgrothers rSaliaseszKernelArgs.aliasesst33::<= UH(J/!-- UQWW777 ; ;;D...,,U3X5H5HHHD///sD$7$7$>?ATATTT U Us B9C,<0C,ct|jj|ttxr.t|j j|ttSrk)r2rrrrrrs rS is_removedzKernelArgs.is_removedsK    # #D' 2J N--11$@*M NrUclt}t|jjD]1}t |t r|j |jd3|jjD]5\}}||jvst |t r%|j |7|S)Nr) rr1rrr2rr9rrr)rg live_outsrrrs rSlive_output_bufferszKernelArgs.live_output_bufferss%/\ t33::<= 4H(J/ MM(..r2 3 4!//557 !LE5,,, 5*0M MM%  !rU)rrxrz6Union[dict[_T, Union[str, RemovedArg]], dict[_T, str]]rdrJrurxrtrw)rdr rurr-)rrxrrxrurv)rrrrruztuple[str, int])rrrurx)rdrxrfrrurx)rdrrurx)ruz Iterator[str])rdrxrur)rrxrrrurx)r SymbolLikerurxrk)rz Optional[dict[torch.dtype, str]]ruz&tuple[list[str], list[str], list[str]])ruz?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])ruzIterator[tuple[str, str]]r)ruzOrderedSet[str])rzr{r|rrrrrrrrr+rrrrrrrrrrrrrrUrSrrs  E    5  ,, @B48'!R8 7 2EI..!A.. /..`/? H/?b UN rUrc`eZdZdZ d d fd Zd dZd dZd dZd dZd dZ xZ S)rkaDA CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py ct|t|tsJt |||_||_d|_||_||_ yr) rrr2rrHrdbounds use_countrshape)rgrdrrrr s rSrzCSEVariable.__init__sL &+.<V <.    rUc|jSrkrurls rSrzCSEVariable.__str__s yyrUc,t|jSrk)hashrdrls rS__hash__zCSEVariable.__hash__sDIIrUcXt|txr|j|jk(Srk)r2rkrd)rgrs rS__eq__zCSEVariable.__eq__s!%-I%** 2IIrUcyrkr)rgrdrprs rSupdate_on_argszCSEVariable.update_on_argss rUcN|jjd|jdSr )r rzrdrls rSrzCSEVariable.__repr__s$..))*!DII=::rUr)rdrxrValueRanges[Any]rrrrFrw)rur)robjectrur)rdrxrpr rr rurv) rzr{r|r}rrrrrrrrs@rSrkrksU(, $  ! %   J ;rUrk AugmentedKeyT)default)boundr.cPeZdZdZ d ddZddZddZddZddZddZ dd Z dd Z dd Z e jd d ddd  ddZe jddf ddZe jddf ddZy)CSEz Common subexpression eliminationNc||_||_i|_||_|xsi|_|xsi|_|xst j|_t|_ |xsi|_ yrk) rr\_cache name_prefix store_cachereduction_cache itertoolsriter_buffer_idsrinvalidated_stores varname_map)rgrr\r iter_buffersrrrs rSrz CSE.__init__-sm  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrUc6g|jjD]2\}}||vs |j|=|jj|4|r9|jjDcic] \}}||vs ||c}}|_yi|_ycc}}wrk)rrrr9r )rg keep_varsrdtmprrs rS invalidatezCSE.invalidateEs44++1134 2ID#)#$$T*''++D1 2 ,0KK,=,=,?RDAq1 >1a4RDKDKSs 1 B>Bc t||j|j|j|j|j |j |jS)N)rr\rrrrr)rHrr\rrrrrrls rSclonez CSE.cloneOsPtDz;;;;((--(((( 00  rUc|j}t|j|_t|j|_t|j|_|S)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)rr,r rr)rgnew_cses rS scoped_copyzCSE.scoped_copyZsH**,#DKK0",T-A-A"B()9)9:rUc"tt|S)z@Override this method to augment cache key with backend specifics)r rrg cache_keys rS augment_keyzCSE.augment_keybsM9--rUc@||j|j|<yrkr r")rgr!rs rSputzCSE.putfs36 D$$Y/0rUc<|j||jvSrk)r"r r s rScontainsz CSE.containsis *dkk99rUcX|jj|j|dSrk)r rr"r s rStry_getz CSE.try_getls"{{t// :DAArUc>|j|j|Srkr$r s rSrzCSE.getos{{4++I677rUT)rre assignmentrrc t|tr |j}|s|sJt|trE|jj ||_|xj dz c_tt|St|tr|j}n1t|tr |j}nt|tsJ|}|j|} ||sd}| s|j|||} |j!|| |rt"j$j&r+t"j$j&j)|dt|trP|r |j+|j,| d|j/||j+|j0| St|trM|sJ|j+|j3|j,| d|j|j0| S|r |j,| d||j0} n||j0} |j+| |rPt4j6j8st4j6j:r|t=dk7r t?|| || S| jj || _| xj dz c_| S)NrGrT) only_oncez =z = rg) r2r4rfrkrtightenrr rr*getvaluer'rzrxr)newvarr%r7rs current_nodecodegen_originating_inforrsplicer\r~r rrrr)r) rgrrrrer+rrr!rrzs rSgeneratez CSE.generatersl dH %::D "" dK (++--f5DK NNa N. . n - I . / IdC( ((Ill9% =E++feU3C HHY $88((HH))BB$CdN3!((DKK=R)@AMM$'$$T[[1: 9 &67%%:$$$++se3tyyk$++'WX4 -""&++se3tfT[[MJ"& }5$$T*#"//KK%22JJ!-/1U:#FC7  ++F3CJ MMQ M rUc|jt|j}tjj ||||}||j |<|Srk)rrrr7rscreate_cse_varr)rgrrrvar_namers rSr0z CSE.newvarsT &&'T-A-A(B'CDhh%%huE%(" rUctj|jvfdtjj |||}||j<|S)NcdS)Nzduplicate name: rrusrSrmzCSE.namedvar..s4DTF2KrU)r _check_valuerr7rsr6)rgrdrrrrs ` rSnamedvarz CSE.namedvarsU  (( (*K hh%%dFE5A!$ rU)rrrNNNN)rrxr\rxrrxrzOptional[itertools.count[int]]rz.Optional[MutableMapping[str, CSEVariableType]]rz|jj|||yrk)rB__exit__)rgexc_typeexc_valexc_tbs rSrGzCodeGen.__exit__s   7F;rUrtr<rHr rIr rJr rurv)rzr{r|rrErGrrs@rSr?r?s1 &  %D D sAAAA AAc#K||}|dux}r t}|j}|j}|j}|j}||_||_||_|j |_ d||_||_||_||_|r |rJdyy#||_||_||_||_|r |rJdwwxYww)Nz$unexpected store inside swap_buffers)r*rRrSrTrWr) rglbcbsbdisallow_storesrRrSrTrWs rS swap_bufferszKernel.swap_bufferss :B Dj (? (!B ,,hh   ??$ F DJ"DL DKDHEEEv2 DJ"DL DKDHEEEv2sA/C 2B6)C *C  C ctrkrrs rSrz Kernel.load4rrUc|j} |j|_|j||||_S#||_wxYw)z+A load the depends on an index we have read)rRrSr)rgrdrrcs rS indirect_loadzKernel.indirect_load7s8  DJ99T5)DJDJs "8 Actrkrrs rSrzKernel.store_reductionArrUctrkrrs rSrz Kernel.storeD "!rUctrkrrs rSrzKernel.reductionI "!rUctrkrrs rSrz Kernel.scanRs "!rUctrkrrs rSrz Kernel.sort\rrrUctrkrrls rS var_rangeszKernel.var_rangeserrUct)z3 See [Note: Inductor bucketize op] rrs rSrzKernel.bucketizehs "!rUctrkrrls rSassert_functionzKernel.assert_functionws!!rUc vt|tr t|}t|tsJt||t|tsJ|t|tsJ|r|rd|d|d|d|d }|d|d|}n|r |d|}|}n |sJ|d|}|}|r d|d|d}|jd|d|dS) Nr z <= z) & (z < rz) | ~(z, "index out of bounds: z"))r2rkrxrHry)rgrrrmaskrY cond_prints rSindirect_assertzKernel.indirect_assert{s c; 'c(C#s#.T#Y.#} 5# 666} 5# 666 UugT#eC5E7! ?  %%a&:&:4&@A rUcH|jt| |||yrk)remove_kernel_local_buffersrrG)rgrHrIrJr s rSrGzKernel.__exit__s  ((* 7F3rUctjjsytfd|jD}t|jD]N}||j vs||j jvs+j||s>j|PD]}||j jvrw|j j|}t|trEtfd|jD}|r|j||j j||j#|y)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3tK|]/}|jvrj|j1ywrk) name_to_bufdefining_op_name)rrrs rSrz5Kernel.remove_kernel_local_buffers..s;& i+++  ! !# & 7 7 9& s58c3&K|]}|v ywrkr)rrnames_to_removes rSrz5Kernel.remove_kernel_local_buffers..sKaQ/1Krp)r7rrrrYrXrpr$can_buffer_be_removed_through_fusionr9rr2rrrremove_inplace_bufferrt remove_buffer)rgfused_node_namesrdrrcrrs @@rSrz"Kernel.remove_kernel_local_bufferss%GG%%  %& ..&   ,6<++ *DD222 7 77BB* ##D) *$ )Dtyy000ii//5c:.K3??KK..t4''++D1""4( )rUctjd|t|jj|<|j j |y)Nzremove_buffer(%r))r1rQrrprrrr9rs rSrzKernel.remove_buffers; %t,)0   &   &rUctjd|t|jj|<|j j |y)Nzremoving_inplace_buffer(%r))r1rQrrprrrr9rs rSrzKernel.remove_inplace_buffers9 /6*1 !!$'   &rUc t|ttfr|Dcgc]}|j|c}Stj j j|}t|jd}|Dcic]W}t|tjtjtjfr||jj!|Y}}t#||Scc}wcc}w)Nc|jSrkru)ss rSrmz(Kernel.rename_indexing..s !&&rU)r)r2listtuplerename_indexingr7rrrsorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZErprr/)rgrrsorted_symbols replacementss rSrzKernel.rename_indexings edE] +5:;D((+; ;  ))%0 2 28HI$  %%II)) tyy~~a    %..<  s C%;AC*ct|i|Srk)rk)rgrprs rSr6zKernel.create_cse_varsD+F++rUcZ|y|jj|jS)zC Returns arg name of a given input or output node. N)rprr)rgrs rSrzKernel.arg_names' <yy!!$--/22rU)NT)rpzOptional[KernelArgs]r`rrurv)rrErurr)rfr*rgOptional[IndentedBuffer]rhrrurrdrxrrrurkrdrxrrrfrkrurvrk rdrxrrrfrkr[r6rurv rrrrrr5rf+Union[CSEVariable, tuple[CSEVariable, ...]]rurrrrzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rtuple[CSEVariable, ...]rur rrrrrrrrrur)ruzdict[sympy.Symbol, sympy.Expr]rrkrrrrkrrrrrrrzOptional[CSEVariable]rurkrw) rzUnion[CSEVariable, str]rrrrr{z!Optional[Union[CSEVariable, str]]rurxr)rrrurxr<rKrtrdrxrurv)rz;Union[list[sympy.Expr], tuple[sympy.Expr, ...], sympy.Expr]rur)rpr rr rurk)rr@rur)#rzr{r|rNr~r\rOrrrrdrjrrmrrrrrrvrrryr}rrrErGrrrrr6rrrs@rSrMrMsM3FC9=I6=PT /( /HL /  /D&&(,'+ F F %F % F  FF8""SW"" *"3>"FO" " """& " ; " 5 ""'" " ( " !""'"(" "  " ! ""4804 " "C "& " $ "  "1 ". "  """37 V $VV V 0 V V<""&0"9="FJ" " "4%)N'' /P/ /.,3rUrMc8eZdZUdZded<dZded<dZded <y) rrz ClassVar[str]rNrrrrxops_name)rzr{r|rr~rrrrUrSrr s!"C"#'E 'HcrUrcb ddl}|j|jS#t$rYywxYw)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)rs rS jinja2_envr s?!!,,"   s " ..ceZdZdZe d d dZed dZe ddZddZe ddZ ddZ ddZ dd Z y )KernelTemplatezg Base class for defining kernel templates. Children classes: TritonTemplate, CUDATemplate c|jd}t|dkDr|ddDcgc] }d|z|z|zc}|dddj|Scc}w)NTrGrr) splitlinesrr)source num_indentsindents_spacinglinesrzs rSindent_except_firstz"KernelTemplate.indent_except_first sd!!$' u:>INqrAE&4<E!"Iwwu~sAct}|ytj|jd<ddlm} |j |S#|$r}Gdd|}|||d}~wwxYw)Nrr)TemplateSyntaxErrorc(eZdZdfd ZddZxZS)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErrorct||j|j|j|j ||_yrk)rrmessagelinenordfilenameoriginal_error)rgrr s rSrzRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__8 s>G$&..&--&++&//  +9D'rUcFd|jd}|d|jdz }t|jdr|jjj d}|dz }t d|jdz }tt||jdz}t||D]s}||jdz k(rN||dzd ||dz }t|jd s=|d d |jjdz zzd zz }c||dzd||dz }u|S)NzError in template at line  zError message: rz Context: rrrGz: --> columnz rz^ z: ) rrrrrsplitmaxminrrr)rg error_inforstartendres rSrzQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__A sA#=dkk]"!MJODLL>"DDJt22H= $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&uc!2 KA DKK!O3 *QveAhZr.J J #*4+>+>#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J !+QveAhZr.J J  K&%rU)rrrurvrw)rzr{r|rrrrs@rSDetailedTemplateSyntaxErrorr7 s 9&rUr)rrrfiltersrr from_string)renvrers rS_template_from_stringz$KernelTemplate._template_from_string+ sjl ;-;-O-O )*.# 8??6* *"! 8 &.A &>.a0a 7C! 8sAA!AA!ctjjt|tt fr.|Dcic]!}|j |j#c}n |j |jidfd }|Scc}w)Nc>j|}||S|Srk)r)rdr_get_dtype_reallookups rSrz1KernelTemplate._fake_get_dtype..get_dtypeb s'ZZ%F! "4( (rU)rdrxrur)r7rrr2rrr) fake_outsrrrrs @@rS_fake_get_dtypezKernelTemplate._fake_get_dtypeX sr''++ i$ /AJK#cllncmmo5KF((*I,?,?,ABF ) Ls&B c||_yrkrurs rSrzKernelTemplate.__init__j s  rUc|jS)a  entry point to override for templates to ensure a uid e.g. through a prefix the purpose of this is that every KernelTemplate/ExternKernelChoice is unique in the system, but reproducible e.g. restarting pytorch should yield the same id rurls rSuidzKernelTemplate.uidm syyrUc Xg}|j|fi|}|t|dk(r|dSy)z Maybe generates a new ChoiceCaller and returns it, or None if generation fails. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. NrGr)maybe_append_choicer)rgr temp_choicesrs rSchoice_or_nonezKernelTemplate.choice_or_nonex s> #% ))),A&A >c,/14? "rUc  |j|jdi|y#t$rQ}tj d|t |tj tjk|cYd}~Sd}~wwxYw)a% Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_infor) rr4rr1inforHgetEffectiveLevelrOINFO)rgchoicesrrs rSrz"KernelTemplate.maybe_append_choice sn  NN=4==262 3"  HHET 002W\\A   H s!$ A>AA93A>9A>c t)zM Generates a ChoiceCaller instance from the given arguments. r)rgrs rSr4zKernelTemplate.generate s "!rUN))rrxrrrrrurx)rrxrur )rzUnion[list[Buffer], Buffer]ruzCallable[[str], torch.dtype]rrw)rr ruzOptional[ChoiceCaller])rryrr ruzOptional[NotImplementedError])rr rur>)rzr{r|r}rrrrrrrrrr4rrUrSrr s >?"%8; *8*8X. %"  ,/ &."rUrc"eZdZdZdfd ZddZddZ d ddZ ddZddZ ddZ d ddZ dd Z dd Z dd Z dd Z d dd ZxZS)rcbt|ddlm}||_||_||_y)NrValueRangeAnalysis)rrrr vr_analysisrsparent_handler)rgrsrrr s rSrzCSEProxy.__init__ s+ /-/ ,rUc d |jgi t|ji}t}t }t t|}d ddk(r dk(r|j |jnydk(rT dk(rOtjjjjtjdj dn dvrt|}|i |i dvr Jd d  fd } tj | |S) Nmaskedrirg)rirgrs)rirgrct ttfr n }t ttfr,t dkDrt dttfr n } dz t|tr+dk(r|j ||_|j ||_tjjjtjj| }|jtjjstjj r)|Jt#tjj|||S)NrrGrgrrr)r2rrrrkrrr7rsrWr4rSrr rrrr) r var_dtype var_shapecsevarrprrrrdr output_idx output_shapes rSdo_csez!CSEProxy._default..do_cse s7 lT5M:Z(! lT5M: %)|Au >Z("  !OJ![)e#'AG77?'AGXX\\**  "" +F  ! !$f 5##??&&>> ,,,AHH,,fi@MrU)rzUnion[str, CSEVariable]rurk)_bound_variablerrr"r%r)rrr7 interpreterr1rrrrpytreetree_map)rgrdrprrf dtype_handler shape_handlershape_opdtype_oprrrrrrs ``` @@@@@rS_defaultzCSEProxy._default s@%%%d>#''e  L 0 0}d3H#T4V4L#T4V4L ' '+ ++ ( ( Tvu--rUcz ddlm}ddlm}ddlm}t tj|rtjSt tj|rtjStjj j|k(r|jjt |jjts$Jt!|jj|jjj# tjSt$j&rjt)||r^t+ fddDrtjS|rJd d}t-t/||}t1|j2||StjS) z If the variable comes from an FX node, we forward the bound we have already computed Else, if the variable when codegen'ing another op, we try to compute its bounds rr)TritonTemplateKernelrG)CUDATemplateKernelc3:K|]}|jvywrk)r)rrfx_nodes rSrz+CSEProxy._bound_variable.. sV11&Vs) set_indirectrrct|tr |jSt|tjr t |S|Srk)r2rkrrrrrs rS arg_to_boundz.CSEProxy._bound_variable..arg_to_bound s2a-88O5::.&q>)HrU)rr rur )rrselect_algorithmrcuda.cuda_kernelrr2r7rsrr=rr1rr\dictrHrr compute_all_boundsrrqrrrr) rgrdrprrrrr  arg_boundsr s @rSrzCSEProxy._bound_variable sP 0;8 ahh 4 5&&( ( ahh 2 3&&( (--,, >>T !dkk&@&@&Ldkk88$?  **B ?;;--11';;N;N;PQ Q  & &73Et+LV0UVV"**,,  : c,56J274++T2J? ?""$$rUcPt|trtj|}t|tjsJt ||f|j jdkr|rtj|tj|tj}|j jdk\r0tj|d}tj|||}n|}t!j"}|j t!j"k7rt|tj$r|j t!t& dz}t!|j|z|j|z}|j jdk\r"|j t!dt&z} || z}|j(j*j-|j(j.|||j0|j2}|j4j7|||} t9|ro|j jdk\ } t|tj$ xs|j j|k } |j(j;| || | | S)Nrrr)r2rrrrrHrrr2r9rrlongrltrWrr=NumberrrsrWr4rSrrrrr(r) rgrrrrstmr new_bounds neg_boundspos sympy_var assert_lower assert_uppers rSrzCSEProxy.indirect_indexing( s dC ==&D$ +?d4j$-??+ ::  a ggc3>>$ #CD::##q(QB))BS1C%,,.Jzz[0022z$ 7U!ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+c!1J++//** ##!iiii +C''99#tUK 5 ! # 0 0A 56L)$ ==   4'BL KK $ $YlL QrUc>|jj||||Srk)rsrrs rSrzCSEProxy.check_bounds_ s{{''dE5AArUc||jjjvr)tjjj |t |tjr|jj||S|jjj}||vr||S|jj||}|jdk(r|jxjdz c_ |Sr)rsrWrr7rXr9rrTMPrmrrrrU)rgrdrrouts rSrz CSEProxy.loadd s 4;;??55 5 HH & & * *4 0 udhh /;;,,T59 9kkoo11 ; t$ $kktU+ ==A  KK A %  rUcl||jjj|<|jjr{|tj j vr^|jjj|}|jD]%}||jjj|<'yyyrk) rsrWrr1r7rname_to_buffer get_output get_mutations)rgrdrfr other_names rS_update_store_cachezCSEProxy._update_store_cacheu s,1 ##D) ;; # #0F0F(F++**55d;C!//1 @ :? ++J7 @)G #rUc|jjj|||j|||tj j vr |jj||||yy)N)r[)rsrYr9r'r7rrrrrs rSrzCSEProxy.store| sc &&**40 <  $ $T5 1 qww.. . KK  dE5t  < /rUc|jjj||j|||tj j vr|jj|||Syrk)rsrYr9r'r7rrrrrs rSrzCSEProxy.store_reduction sZ &&**40   u- qww.. .;;..tUEB B /rUc||jxjdz c_|jj||||Sr)rsrVrrs rSrzCSEProxy.reduction s4 !!Q&!{{$$UI~uMMrUc<|jj|||Srk)rsrrs rSrz CSEProxy.scan s{{ F;;rUc>|jj||||Srk)rsrrs rSrz CSEProxy.sort s{{ CCrUc D|jj|||||||S)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )rsrrs rSrzCSEProxy.bucketize s1L{{$$         rU)rsz Kernel[Any]rzOpsHandler[Any])rdrxrpztuple[Any, ...]rzdict[str, Any]rur )rdrxrpr rr rurr) rrkrrrrrrrurrr)rdrxrfrkrurvrkrrrrrrr)rzr{r|rdrrrrrrr'rrrrrrrrs@rSrr s D-K.Z+%b 5 5%5 5  5  5nBB&0B9=BFJB B "@SW== *=3>=FO= =CNNN& N ; N 5 N <' <  <( < ! <D'D(D D  D ! D 4804N N CN & N $ N  N 1N .N  N rUr)rRrxrurv)NNNN)rrxr3rr4rr5rr6rr7Optional[CustomGraphModulePass]r8Optional[ConfigModule]rurv)rUnion[torch.device, str, None]ruzOrderedSet[BackendFeature])rr0rNr;rur)rrxruzOptional[SchedulingConstructor])FF)rrxrRrrSrrur)rrxrur.)rrxrur/rt)rSequence[sympy.Expr]rr1rr1rur)rrxrrrurv)rrxrur)rrxrpr rr rur)rr*rrrrrurv)rdrxrurrr)rur ) __future__rr`r dataclassesenumrrrOr2rrbrer^abcrrrrrtypingr r r r r rrrrtyping_extensionsrrrrtorch.fxtorch._prims_commonr torch.utilsrrtorch.utils._config_modulertorch.utils._ordered_setrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrr r!dtype_propagationr" ops_handlerr#r$shape_propagationr%utilsr&r'r(r)r*r+r,r-r.r/r0r1 virtualizedr2r3r4r5r6r7collections.abcr8r9r:r;custom_graph_passr<rr=r>r?r@rrBrrCrDrErFrrIrJrrHrrxrrr^_logginggetArtifactLoggerrzrM getLoggerr1rT dataclassrWrrrrrrrrr KernelArgTyperr~rr.r/r0r9r;rJrOrIrUrWrYcacherGrrrbfloat16rfloat16rr&float64int8int16rrruint16ruint64rrrrrcompile IGNORECASErlrgrirr INT_TO_FLOATrrvrxrrrrrrrkrrrrReductionCacheKeyr r?rMrrrrrs0rSr[s "    #   , ?)3/-GOOD:;:    QPBB$9>>$DD2- B$hy&9%:N%JK23sELL()JF~~//*E g!=  >/ /(  C  Td=ddN** ###   !!! <<<lIw8H,VW ,.).5"5"p:<6;DFAFDF AF8@D>B:>37B B,B/B!= B < B 8 B1B B4 &T &3 *33$3 *35C3 3U @E   " 8< ! V ``FU U$U U U; ;&7; ; , NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    u >:,' ' '' 'T: :!0:9D: :2aBaBH=N=$S1S1l"rzz";2==Q D;#%5z#D;N  - - - 6:`6 ;HH -  `6 ;HH /-  `6;HH /- `6$;HH /- %`60;HH /- 1`6< ;HH *)   =`6L ;HH (0   M`6X ;HH 66>   Y`6h ;HH 1  i`6r ;HH 2  s`6|;HH 1 }`6F;HH 2 G`6P;HH %8$  Q`6^ ;HH &%   _`6j;HH %8  k`6v ;HH &  w`6@;HH + A`6L%;HH 88 ) M`6X%;HH 88 ) Y`6d%;HH 8 )e`6n%;HH 8 )o`6z ;HH '  {`6D ;HH (  E`6N;HH c  O`6^,;HH ? 0_`6h,;HH ? 0i`6t&;HH 9 *u`6~ ;HH *  `6H);HH D -I`6R);HH D -S`6\);HH D -]`6f);HH D -g`6p(;HH C ,q`6z$1;HH L 5${`6D$1;HH L 5$E`6N$1;HH L 5$O`6X$1;HH L 5$Y`6b';HH B +c`6l(;HH C ,m`6v(;HH C ,w`62`F -#-">*Z  NNN ,JJZ #;#;L5 +; T  k5c!1223 5l'/=0 1l^ < <\3Wgo.\3~  G"G"TU ~U _9s e-