pi&7ddlmZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m Z mZddlmZmZddlmZdd lmZe rdd lmZmZe j4j7ed Ze j4j7ed Zeee e e jBfZ"eeege"fZ#ejHd GddZ%ejHd GddZ&ejHd GddZ' d-dZ(d.dZ)d/dZ*d0dZ+d1dZ, d2dZ- d3dZ.d4dZ/ d5dZ0 d5dZ1d6dZ2ejHGd d!Z3 d7d"Z4d8d#Z5Gd$d%eZ6 d9d&Z7 d:d'Z8ejHd Gd(d)Z9ejHd Gd*d+Z: d;d,Z;y)<) annotationsN)Enum)AnyCallableOptional TYPE_CHECKINGUnion)countersget_metrics_context)GraphPartitionMap InputType) OrderedSet)is_using_cudagraph_partition)SequenceSet perf_hintscudagraph_static_inputsT)frozenceZdZUdZded<y) FunctionIDz9Unique counter of a function wrapped in cudagraphify_implintidN__name__ __module__ __qualname____doc____annotations__a/opt/services/ai/voice_agent/venv/lib/python3.12/site-packages/torch/_inductor/cudagraph_utils.pyrrs ? Gr!rc:eZdZUdZded<ded<ded<ded<y ) PlaceholderInfoz A serializable version of torch.fx.Node that contains information pertinent to placeholder stack traces. We use these in logging and error messages related to cudagraphs, and will cache these results. strname Optional[str] stack_tracelist[PlaceholderInfo]usersmutating_use_stack_traceNrr r!r"r$r$%s  I  ++r!r$cNeZdZUdZded<ded<ded<ded <d ed <ded <y )WrappedFunctionz Represents a function that you want to record for CUDA graph replay, with a little more metadata so we can identify if we have an applicable CUDA graph in our CUDA graph tree for it. zCallable[..., Any]model Sequence[int]static_input_idxsrrztuple[torch.Tensor, ...] constantsSequence[PlaceholderInfo] placeholdersmutated_input_idxsNrr r!r"r-r-4s, $$N''++%%r!r-ct|jdk(r8tt|jjj ddS|jD]`}|j tjjjjk(s?|jj ddx}s^|cSy)Nrr() lenr*nextitermetagettargettorchopsatencopy_default)placeholder_nodeuser(s r"&get_mutating_use_stack_trace_from_noderCDs  ! !"a'D)//0166::=$OO%%# ::--55 5!hhll=$??{?""# r!c|jSN)r+)placeholder_infos r"get_mutating_use_stack_tracerGSs  4 44r!c|j}|jjdd}g}d}|jdk(r-|jDcgc] }t |}}t |}t||||Scc}w)Nr( placeholder)r&r9r:opr*to_placeholder_inforCr$)rAr&r(r*r+is r"rKrKWs  D"''++M4@K E#m+1A1G1GHA$Q'HH#I $   4e5M NN Is A7cr|jDcgc]}|jdk(st|c}Scc}w)NrI)nodesrJrK)graphnodes r"get_placeholder_inforQfs4.3kk &*TWW =UD!  s4 4c d|S)Nzskipping cudagraphs due to r )reasons r"format_default_skip_messagerTls ( 11r!cd}|D]}||}t|x}sntdt|d}|r|d|S|S)Nzmutated inputs (z instances). Found from : )rGrTr6)r3mutation_indicesr(idxrImsgs r"get_mutation_stack_tracer[psp"$K"3' 6{C C; C  & 3/01= C( 66 Jr!ctjjjjr3|j Dcgc]}||j vs |||s|}}n |j }tjd|j tjd||rt|j|SdScc}w)Nz'check mutation static input indices: %sz#check mutation mutation indices: %s) r< _inductorconfigtritoncudagraph_treesr4r0static_inputs_logdebugr[r3)funcinputsis_cuda_graph_recorded_tensorrYrXs r"check_for_mutationrfs  $$44..+ t---0= + +  22143I3IACST  !!2!24DE!+ s"B>cj|jD]$}|jjddx}s"|cSy)Nr()r*r9r:)rPrBr(s r"_get_use_stack_tracerhs:zz((,,}d; ;; ;  r!cJ|jtjddtr%|jtjdd|j tjdx}r8d|j d}t |x}rt|d|St|St|dk(r0tt|jjdk(ryd|jD}td d j|S) Nr9cpuz cpu device ()rWrcudac32K|]}t|ywrE)repr).0keys r" z:check_multiple_devices_or_any_cpu_nodes..sAscAszmultiple devices: z, )popr<devicerr:r&rhrTr6r7r8keystypejoin)device_node_mappingcpu_noderZr( keys_reprs r"'check_multiple_devices_or_any_cpu_nodesrzsELL0$7$% U 3T:&**5<<+>??x?X]]O1-.x8 8; 8.#6H /VW W*3//  A% )..01 2 7 76 AA&9&>&>&@AI &);DIIi?K $$44 !23 +=XCKAWCXX+,1 +BHH5 ' 6FGG>*a/ 't8[8Ys BBc|jr |jS|jD]}|js|jcSy)zM Gets the first non-empty stack trace of a placeholder or its users. N)r(r*)rIusers r"get_placeholder_stack_tracersH&&&!!$   ## #$ r!c$eZdZdZdZdZdZddZy)CheckInvariantStatusrc|jdk(ry|jdk(ry|jdk(ry|jd|jS)NCudagraphManagedIdxMismatchz-cudagraph managed tensor data pointer changedStaticInputIdxMismatchz!static input data pointer changed&ExpectedDeadIndicesBeforeGraphMismatchz+expected dead indices before graph are livez: )r&r)rs r"__str__zCheckInvariantStatus.__str__sK 995 5B YY2 26 YYB B@ii[4::,/ /r!Nrr%)rrrSUCCESSrrrrr r!r"rrs$G#$./*0r!rc t|t|k(rt|t|k(sJd|Dcgc]}|| }}|Dcgc]}|| }}|d}tt||D]t\}\} } t| tj sJ||} | j | k7s>|| } |d| jd| d| j dt| d }v|Scc}wcc}w)z} Logs the mismatch between input data pointers and recorded data pointers. This checks only idxs in target_idxs. zClength mismatch between inputs, recorded_data_ptr, and placeholdersz. z input name: z. data pointer changed from z to z. input stack trace:  ) r6 enumerateziprr<Tensordata_ptrr&r) r3rdrecorded_data_ptr target_idxsmismatchrL t_tensors t_data_ptrs error_msgtensorrindexrIs r"log_data_ptr_mismatchrs' v;#/0 0S[C DU5UM U%00q0I01<=A$Q'=K=*C I!*3y++F!G  FH&%,,///A ??  (&u-K+\+*:*:);<--5Jd6??;L:MN&&A+&N%OrS   1=s C- C2c>t|jdzdfd }tjjj j rLtjjj j kDrtj|yy)NrcddS)NzCUDAGraph supports dynamic shapes by recording a new graph for each distinct input size. Recording too many CUDAGraphs may lead to extra overhead. We have observed a0 distinct sizes. Please consider the following options for better performance: a) padding inputs to a few fixed number of shapes; or b) set torch._inductor.config.triton.cudagraph_skip_dynamic_graphs=True. Set torch._inductor.config.triton.cudagraph_dynamic_shape_warn_limit=None to silence this warning.r )num_cudagraphssr"warn_msgz4maybe_warning_due_to_dynamic_shape..warn_msgDs 00>/?@' ' r!TFr) r6rtr<r]r^r_"cudagraph_dynamic_shape_warn_limitrr)fn_cache new_int_keyrrs @r""maybe_warning_due_to_dynamic_shaper>st)A-N   %%HH  // ' ' J J K hj) r!c0eZdZUdZded<ded<ded<y) CudagraphCachedInfoz' Info needed to realign inputs r2r3list[Optional[str]] stack_tracesz list[str]cudagraph_fail_reasonsNrr r!r"rr[s,+%%%%r!rcDeZdZUdZded<ded<ded<ded<d ed <y ) CudagraphMetadataz. Metadata for recording a CUDA graph. r2r3OrderedSet[int]r0r4rrzdict[str, torch.Tensor]r1Nrr r!r"rrfs',+&&''%%&&r!rchg}t}t}t|jD]\}}||jvr|j |||j vr|j |||j |}ntd|jd|dgd}|j|g}|jD]4} | |j|j| $|jd6|jD cic]} | |j| } } t||||| Scc} w)z Convert the cudagraph metadata at the graph level to the graph partition level, given the graph partition info (i.e., mapping from partition input/output index to graph input/output index). N partition_ _placeholder_)r&r(r*r+)rrinput_index_mappingr0addr4r3r$rappendoutput_index_mappingrconstant_namesr1r) partition_mapmetadatapartition_placeholderspartition_static_input_idxspartition_mutated_input_idxspartition_input_idxgraph_input_idxrIpartition_stack_tracesgraph_output_idxr&partition_constantss r" get_partition_cudagraph_metadatarssp 3=<4>L 09))13,_ h88 8 ' + +,? @ h99 9 ( , ,-@ A  &"//@K*!-"2"2!3=AT@UV )- K %%k2'3* )>>0  ' " ) )(*?*?@P*Q R " ) )$ / 04A3O3O+/h  && #$   sD/)rA torch.fx.Noderr')rFr$rr')rArrr$)rOztorch.fx.Graphrr))rSr%rr%)r3r2rXz&Union[AbstractSet[int], Sequence[int]]rr%)rcr-rdlist[InputType]rezCallable[[torch.Tensor], bool]rr')rPrrr')rwz!dict[torch.device, torch.fx.Node]rr')rZr%rr) rztorch.fx.GraphModulerzOrderedSet[str]r4rr0r/rr')rIr$rr') r3r2rdrrzSequence[Optional[int]]rr/rrrr%)rz)dict[tuple[int, ...], Callable[..., Any]]rrrbool)rr rrrr)< __future__r dataclassesenumrtypingrrrrr r<torch._dynamo.utilsr r torch._inductor.utilsr r torch.utils._ordered_setrutilsrcollections.abcrr AbstractSet_logginggetArtifactLoggerrrralistrr OutputType ModelType dataclassrr$r-rCrGrKrQrTr[rfrhrzr|rrrrrrrrrrr r!r"rs"@@ =>//<00<H NN44 ' (5ell!234 5 d9o& 2 3 d#  $ d# , ,$ ,d# & &$ & #  5 O 2+< (  $B >T:TT8H:HH J    99#9(9% 9  90 0400+ /  #   >7 :d#&&$&d# ' '$ '3$333r!