
    pi>                   
   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$m%Z% ddl&Z&ddl'Z&ddl(Z&ddl)m*c m+Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZG ddlHmIZImJZJ ddlKmLZL ddlMmNZNmOZO ddlPmQZQ ddlRmFZF ddlSmTZT ddlUmVZV  e	j                  eX      ZYe&j                  j                  Z[e&j                  j                  Z\eZ]e"e]e&j                  j                  f   Z_ej                  j                  dd      Zb G d  d!e       Zc G d" d#e       Zd G d$ d%e       Ze e!d&      Zfe"e&j                  j                  j                  eif   Zj G d' d(      Zk ek       Zl	 d	 	 	 	 	 	 	 dd)Zm G d* d+      Zn G d, d-eo      Zpe"enepf   Zqdd.Zr G d/ d0      Zs G d1 d2e      Zt G d3 d4et      Zu G d5 d6et      Zv G d7 d8et      Zw G d9 d:et      Zx G d; d<et      Zyezed=f   Z{ G d> d?ey      Z| G d@ dAe|      Z} G dB dCe|      Z~ G dD dEe|      Z G dF dGey      Z G dH dIe      Z G dJ dKe      Z G dL dMe      Z G dN dOet      Z G dP dQet      Z G dR dSet      Z G dT dU      Z G dV dWe       Zej                   G dX dY             Zej                   G dZ d[e             Zej                   G d\ d]e             Zej                   G d^ d_e             Zdd`ZddaZ	 d	 	 	 	 	 	 	 	 	 ddcZedddddbf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeZ eD       Zdfedg<   	 	 	 	 	 	 	 	 	 	 	 	 ddhZ ee      j,                  diz  djz  Zg Zdkedl<   eddddbf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddmZ eGj4                  dbn      	 	 d	 	 	 	 	 	 	 	 	 	 	 ddo       Z	 	 d	 	 	 	 	 	 	 	 	 	 	 ddpZefdbdq	 	 	 	 	 	 	 	 	 ddrZefdbdq	 	 	 	 	 	 	 	 	 ddsZddtZ ej@                  du      ZddvZddwZddxZddyZddzZdd{Z G d| d}      Zdd~Z	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZ e&jV                         ddd	 	 	 	 	 	 	 	 	 dd       Z e&jZ                         dd       ZddZddZddZddZddZ eD       Zdfed<   	 d	 	 	 	 	 	 	 ddZddZddZy)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fx)guard_or_falsestatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)preserve_node_meta)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_type#TORCHINDUCTOR_PATTERN_MATCH_BACKENDinductorc                       e Zd ZU ded<   ddZy)SearchFnstr__name__c                     y N selfargskwargss      a/opt/services/ai/voice_agent/venv/lib/python3.12/site-packages/torch/_inductor/pattern_matcher.py__call__zSearchFn.__call__`           Nr:   r   r;   r   returnr   )r4   
__module____qualname____annotations__r=   r7   r?   r<   r2   r2   ]   s
    M=r?   r2   c                      e Zd ZddZy)	ReplaceFnc                     y r6   r7   r8   s      r<   r=   zReplaceFn.__call__d   r>   r?   Nr@   r4   rB   rC   r=   r7   r?   r<   rF   rF   c   s    =r?   rF   c                  $    e Zd Z	 	 	 	 	 	 	 	 ddZy)TraceFnc                     y r6   r7   )r9   fnr:   r;   s       r<   r=   zTraceFn.__call__h   s    "r?   N)rL   zUnion[SearchFn, ReplaceFn]r:   r   r;   r   rA   torch.fx.GraphModulerH   r7   r?   r<   rJ   rJ   g   s'    #,#58#DG#	#r?   rJ   Tc                      e Zd ZddZy)Multiplec                2    dt               vs| t        u sJ y y )NMULTIPLE)globalsrR   r9   s    r<   __init__zMultiple.__init__t   s    *dh.>>>.>*r?   NrA   None)r4   rB   rC   rU   r7   r?   r<   rP   rP   s   s    ?r?   rP   c                   ddl m}m} t        j                  j
                  dk(  ry| j                  dg       j                         }|j                   ||||j                               | j                  d |j                  j                         D               || d<   n0| j                  d |j                  j                         D               d|j                  v r|j                  d   | d<   y y )Nr   )
NodeSourceNodeSourceActionr,   	from_nodec              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr6   torchfxproxy_COPY_META_FIELDS.0kvs      r<   	<genexpr>z!_transfer_meta.<locals>.<genexpr>   6      
1EHHNN444 F
   79c              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr6   r]   rb   s      r<   rf   z!_transfer_meta.<locals>.<genexpr>   rg   rh   stack_trace)torch.fx.tracebackrY   rZ   r%   traceprovenance_tracking_levelgetcopyappendREPLACEupdatemetaitems)new_metaold_node	pass_namerY   rZ   new_from_nodes         r<   _transfer_metary   }   s     @
 ||--2 ["5::<Z)=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	

 %"*--"> &r?   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZ	ddZ
d dZd!dZd"dZ	 	 	 	 	 	 d#dZ	 	 d$	 	 	 	 	 	 	 	 	 d%dZ xZS )&Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]r:   dict[str, Any]r;   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphc                    t         |           || _        t        |xs g       | _        |xs i | _        g | _        i | _        || _        d | _	        y r6   )
superrU   r}   listr:   r;   r   r   r   r   )r9   r   r}   r:   r;   	__class__s        r<   rU   zMatch.__init__   sO     	$	l
!%r?   c                .    | j                   j                  S r6   )r   graphrT   s    r<   r   zMatch.graph   s    xx~~r?   c                2   | j                   rwt        | j                   j                               t        |j                   j                               z  D ]-  }| j                   |   |j                   |   k7  s#t        d|       | j                  j                  |j                         | j                  j                  |j                         | j                   j                  |j                          | j                  j                  |j                         y )Nzkwarg mismatch: {})	r;   r"   keysFailedMatchr:   extendr   rr   r   )r9   otherkeys      r<   r   zMatch.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW A;;s#u||C'88%&:C@@A 			$

%++&5<<(EMM*r?   c                f    | j                   rt        | j                         g| _         | S g | _         | S r6   )r:   tuplerT   s    r<   bundlezMatch.bundle   s.    *.))U499%&	 :<	r?   c                <    d| j                    d| j                   dS )NzMatch(..., , )r:   r;   rT   s    r<   __repr__zMatch.__repr__   s    TYYKr$++a88r?   c                    | j                   }t        | j                        D ]-  }|j                  r|j                  r|j                  |       / y r6   )r   reversedr   _erasedusers
erase_node)r9   r   ns      r<   erase_nodeszMatch.erase_nodes   s>    

$**% 	$A99QWW  #	$r?   c                    | j                   j                  D cg c]  }|| j                   j                  |   nd ! c}S c c}w r6   )r   outputspattern_to_node)r9   ps     r<   output_nodeszMatch.output_nodes   sF     XX%%
 -.MTXX%%a(tC
 	
 
s   $A c                B    t        d | j                         D              S )Nc              3  &   K   | ]	  }|s|  y wr6   r7   )rc   r   s     r<   rf   z$Match.output_node.<locals>.<genexpr>   s     8!aA8s   )nextr   rT   s    r<   output_nodezMatch.output_node   s    8t002888r?   c                \    t         j                  | | j                  j                  ||       y r6   )ReplacementPatternEntryreplace_with_graphr   r   )r9   r   r:   s      r<   r   zMatch.replace_with_graph   s$     	 22$((.."3T	
r?   c                   ddl m}m} t        |j                  |      r|j                  |j                  nt        j                         }dd}|5  |t        j                  t        |      } || j                        rzi | j                  d   j                  d   \  }	}
i |
}
t        | j                        | j                  }}dfd}t        j                   j"                  j%                  |||f|	|
f       t        j&                  j)                  |fd      } |||      }t        j&                  j)                  |d	       } |||      }t+        |j,                  j                        t+        |j,                  j                        k(  sJ t/        |j,                  j                  |j,                  j                        D ]0  \  }}d|j                  v s|j                  d   |j                  d<   2 n*t        j&                  j)                  |d
       } |||      }t+        | j                        dk(  r@|j,                  j                  D ]'  }t1        |j                  | j                  d   d       ) t2        j5                  | | j6                  j,                  ||       ddd       y# 1 sw Y   yxY w)a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVNc                H   t        |       dk7  ry| d   }d|j                  vry|j                  t        t        j
                  j                  j                  t        j
                  j                  j                  t        j
                  j                  j                  g      v S )Nr,   Fr   eager_input_vals)
lenrs   targetr"   r^   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r   nodes     r<   !should_propagate_eager_input_valszCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals  s~    5zQ8D!2;;*II**KKII**>>II**AA#  r?   )run_functional_passesr   c                Z    t        | t        j                  j                        r|| <   y y r6   )
isinstancer^   r_   Node)r   valnode_to_vals     r<   recordz(Match.replace_by_example.<locals>.record%  s$    !$6,/D) 7r?   c                    |    S r6   r7   )argr   s    r<   <lambda>z*Match.replace_by_example.<locals>.<lambda>-  s    +cBR r?   c                     | j                   d   S Nr   rs   r   s    r<   r   z*Match.replace_by_example.<locals>.<lambda>3  s    #((5/ r?   c                Z    d| j                   v r| j                   d   S | j                   d   S )Nr   example_valuer   r   s    r<   r   z*Match.replace_by_example.<locals>.<lambda>H  s.    ( !$ /2 r?   r,   replace_by_exampleru   rv   rw   )r   r   rA   bool)r   torch.fx.Noder   r   rA   rW   )torch._inductor.virtualizedr   r   r   	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyr   rs   r   r:   r;   r^   utils_pytreetree_mapr_   map_argr   r   zipry   r   r   r   )r9   replacement_fnr:   trace_fnr   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementrv   new_noder   r   s                       @r<   r   zMatch.replace_by_example   s    	? q{{K8Q[[=P KK'') 		  I	$,,4I 1< !)-A););<N)O&	;-o+0+;T[[L
0 ##,,Z6K8P  %xx//6RS )1(N%  %xx//6QR&'<lK 066<<=%%++B    +.)//55{7H7H7N7N+ &Hh *X]]:<DMM.=&89	  %xx//3  '~|D4::!#$**00 A"!"!%A"6 $66	II	 I	 I	s   FJ4CJ44J=)NN)
r   r   r}   r|   r:   zOptional[Sequence[Any]]r;   zOptional[dict[str, Any]]rA   rW   )rA   torch.fx.Graph)r   r{   rA   rW   )rA   r{   rA   r3   rV   )rA   zlist[Optional[torch.fx.Node]])rA   r   )r   r   r:   Sequence[Any]rA   rW   NT)
r   rF   r:   r   r   zOptional[TraceFn]r   r   rA   rW   )r4   rB   rC   __doc__rD   rU   propertyr   r   r   r   r   r   r   r   r   __classcell__r   s   @r<   r{   r{      s   	 
O44	55 )-+/&& & &	&
 )& 
&&  +
9$
9
!/
7D
	
 '+&*n!n n $	n
  $n 
nr?   r{   c                  4    e Zd ZU dZded<   ddZd	dZd
dZy)r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r3   format_stringc                f    || _         t        |      dkD  rt        d|       || _        || _        y )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr:   r;   )r9   r   r:   r;   s       r<   rU   zFailedMatch.__init__g  sB    * }#hivhwx  	r?   c                b     | j                   j                  | j                  i | j                  S r6   )r   formatr:   r;   rT   s    r<   __str__zFailedMatch.__str__r  s(    (t!!(($))Ct{{CCr?   c                     yNFr7   rT   s    r<   __bool__zFailedMatch.__bool__u      r?   N)r   r3   r:   r   r;   r   rA   rW   r   rA   r   )r4   rB   rC   r   rD   rU   r   r   r7   r?   r<   r   r   ]  s     	Dr?   r   c                    t        |       S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )r   )ms    r<   is_matchr   |  s    
 7Nr?   c                  d    e Zd ZU dZded<   ded<   ded<   ded	<   	 d	 	 	 	 	 	 	 ddZddZddZy
)r   zC
    Internal state needed while running PatternExpr._match().
    list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r   r   zlist[NodeOrConstant]exclusive_node_setNc               V    || _         |i n
t        |      | _        || _        g | _        y r6   )r   dictr   r   r   )r9   r   r   r   s       r<   rU   zMatchContext.__init__  s.     %4%<r$BW
"$r?   c                    || j                   v r)| j                   |   |k(  rt        | |      S t        d      S |j                  ||       }|| j                   vsJ |r|nd| j                   |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r{   r   _match)r9   r}   r   r   s       r<   matchzMatchContext.match  sy    d***##G,4T7++"#=>>NN4&d2222201tW%r?   c                    | j                   j                         D ci c]  \  }}|j                         r||| c}}S c c}}w r6   )r   rt   has_multiple_users)r9   r}   r   s      r<   filter_multi_user_patternsz'MatchContext.filter_multi_user_patterns  sL     "&!5!5!;!;!=
))+0@ TM
 	
 
s   Ar6   )r   r   r   z*Optional[dict[PatternExpr, torch.fx.Node]]r   r   rA   rW   )r}   r|   r   NodeOrConstantrA   MatchResult)rA   z dict[PatternExpr, torch.fx.Node])r4   rB   rC   r   rD   rU   r  r  r7   r?   r<   r   r     s^     )(??,,
 GK
%,
% D
%
 
% 

%

r?   r   c                  V    e Zd ZdZed	d       Zd
dZddZddZ	 	 	 	 	 	 ddZ	ddZ
y)r|   z+
    Base class for types of patterns.
    c                     y r6   r7   r9   r   r   s      r<   r  zPatternExpr._match  s    MPr?   c                    	 t        | g|j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wNr   )r   r   r  r   r9   r   es      r<   r  zPatternExpr.match  s<    	djj9??dKK 	H	s   '* 	?:??c                     yr   r7   rT   s    r<   r  zPatternExpr.has_multiple_users  r   r?   c                4    | j                   j                  dz   S )Nz())r   r4   rT   s    r<   r   zPatternExpr.__repr__  s    ~~&&--r?   c              #  L   K   | |j                   v r|j                   |     y y wr6   )r   r9   r   searcheds      r<   find_anchor_nodeszPatternExpr.find_anchor_nodes  s,      3&&&%%d++ 's   "$c                .    t        || j                        S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   r   )r9   r   s     r<   
pattern_eqzPatternExpr.pattern_eq  s     %00r?   Nr   r   r   r   rA   r
  r   r   rA   r
  r   r   r   r   r  OrderedSet[torch.fx.Node]rA   z.Generator[Optional[torch.fx.Node], None, None]r   r   rA   r   )r4   rB   rC   r   r   r  r  r  r   r  r  r7   r?   r<   r|   r|     sH     P P.,,+D,	7,1r?   r|   c                      e Zd ZdZddZy)Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    c                     t        || |g      S )N)r:   r{   r  s      r<   r  z
Arg._match  s    S$dV,,r?   Nr   r	  r   r   rA   r
  r4   rB   rC   r   r  r7   r?   r<   r!  r!    s    
-r?   r!  c                  (    e Zd ZdZddZddZddZy)	Ignoredz4
    Match an arg, but don't pass it to handler
    c                    t        ||       S r6   r#  r  s      r<   r  zIgnored._match  s    S$r?   c                     y)N*r7   rT   s    r<   r   zIgnored.__repr__  s    r?   c                     y)Nz	Ignored()r7   )r9   pps     r<   pretty_printzIgnored.pretty_print  s    r?   Nr$  r   r,  PatternPrettyPrinterrA   r3   )r4   rB   rC   r   r  r   r-  r7   r?   r<   r'  r'    s     r?   r'  c                  @     e Zd ZdZd fdZddZddZd	 fdZ xZS )

KeywordArgD
    Capture a kwarg which will become an input to the handler.
    c                0    t         |           || _        y r6   r   rU   namer9   r5  r   s     r<   rU   zKeywordArg.__init__      	r?   c                "    d| j                   dS )NzKeywordArg(r   r5  rT   s    r<   r   zKeywordArg.__repr__  s    TYYM++r?   c                6    t        || | j                  |i      S )Nr;   )r{   r5  r  s      r<   r  zKeywordArg._match  s    S$		4'899r?   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r6   typingcastr   r   r  r5  r9   r   r   s     r<   r  zKeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDr?   r5  r3   rA   rW   r   r$  r  	r4   rB   rC   r   rU   r   r  r  r   r   s   @r<   r1  r1    s#    ,:E Er?   r1  c                  L     e Zd ZU dZded<   d fdZd	dZd
dZd fdZ xZ	S )ExclusiveKeywordArgr2  r3   r5  c                0    t         |           || _        y r6   r4  r6  s     r<   rU   zExclusiveKeywordArg.__init__  r7  r?   c                "    d| j                   dS )NzExclusiveKeywordArg(r   r9  rT   s    r<   r   zExclusiveKeywordArg.__repr__	  s    %dii]!44r?   c                    ||j                   v rt        d      S |j                   j                  |       t        || | j                  |i      S )Nzexclusive arg appears twicer;  )r   r   rp   r{   r5  r  s      r<   r  zExclusiveKeywordArg._match  sH    3)))<==%%d+S$		4'899r?   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r6   r=  r@  s     r<   r  zExclusiveKeywordArg.pattern_eq  rA  r?   rB  r   r$  r  )
r4   rB   rC   r   rD   rU   r   r  r  r   r   s   @r<   rE  rE    s*     I5:E Er?   rE  c                       e Zd ZU dZded<   ded<   	 d	 	 	 	 	 d fdZeedd              ZddZ	dd	Z
dd
Z	 	 	 	 	 	 ddZddZddZd fdZ xZS )_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                Z   t         |           t        |      st        |t              r|gn
t        |      }|D ]O  t        t        j                  j                        s(|j                  fdj                         D               Q || _        t        |      | _        || _        y )Nc              3  6   K   | ]  }t        |        y wr6   )getattr)rc   overloadrL   s     r<   rf   z'_TargetExpr.__init__.<locals>.<genexpr>'  s     PX72x0Ps   )r   rU   callabler   r3   r   r^   _opsOpOverloadPacketr   	overloadsrL  r"   rM  r   )r9   rL  r   rL   r   s      @r<   rU   z_TargetExpr.__init__   s     	}
3(<se$s) 	QB"ejj99:

PPP	Q !#
r?   c                     y r6   r7   rT   s    r<   opz_TargetExpr.op-  s    r?   c                   | j                   d   }t        |t              s|j                  }t	        | j                         dkD  rd| dS | j                   d   t        t        |d       u rd| S | j                   d   t        t        |d       u rd| S t        | j                   d   t        j                  j                        rt        | j                   d         S |S )Nr   r,   [z, ...]torch.z	operator.)
rL  r   r3   r4   r   rP  r^   operatorrS  
OpOverload)r9   
first_reprs     r<   fns_reprz_TargetExpr.fns_repr1  s    XXa[
*c*#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;txx{##r?   c                    | j                   t        u rd}n"| j                   dk7  rd| j                    d}nd}| j                  j                   d| j	                          | dS )Nz
, MULTIPLEr,   r   r    ()r   rR   r   r4   r^  )r9   comma_userss     r<   r   z_TargetExpr.__repr__A  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr?   c                X    t        | j                  t              xs | j                  dkD  S )Nr,   )r   r   rP   rT   s    r<   r  z_TargetExpr.has_multiple_usersJ  s     $**h/A4::>Ar?   c                    t         r6   NotImplementedErrorr  s      r<   r  z_TargetExpr.find_anchor_nodesM  s
     "!r?   c                    t        |t        j                  j                        xr2 |j                  | j                  k(  xr t        |      | j                  v S r6   )r   r^   r_   r   rW  extract_targetrM  )r9   r   s     r<   
_match_fnsz_TargetExpr._match_fnsR  sD    tUXX]]+ 5477"5t$4	
r?   c                    | |j                   v xs6 | j                  t        u xs" t        |j                        | j                  k(  S r6   )r   r   rR   r   r  s      r<   _match_usersz_TargetExpr._match_usersY  s=    CKK -zzX%-4::$**,	
r?   c                    t        j                  t        |      }t        |   |      xrO | j
                  |j
                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S r6   )r>  r?  r   r   r  rW  rL  r   r@  s     r<   r  z_TargetExpr.pattern_eq`  sf    D%(Gu% *588#*EII%* 

ekk)		
r?   )r,   )rL  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]rA   rW   r   r   r  r   r   rA   r   )r   r   r   r   rA   r   r  )r4   rB   rC   r   rD   rU   r   r   rW  r^  r   r  r  ri  rk  r  r   r   s   @r<   rK  rK    s     
   UV4=Q	    LB""+D"	7"



 
r?   rK  .c                       e Zd ZdZdd	 	 	 	 	 	 	 	 	 d fdZe	 	 	 	 	 	 dd       Ze	 	 	 	 	 	 dd       ZddZddZ	dd	Z
	 	 	 	 	 	 dd
Zd fdZ xZS )_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r,   )_usersc               j   t         |   ||       t        |      | _        t	        |      | _        t        d t        j                  ||j                               D              r| j                  | _        n| j                  | _        | j                  | j                  | j
                        | _        y )Nc              3  R   K   | ]  }t        |t        t        t        f       ! y wr6   )r   r  r   r   rc   xs     r<   rf   z+_TargetArgsExpr.__init__.<locals>.<genexpr>|  s&      
 q4u-.
s   %')r   rU   r   r:   r  r;   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r9   rL  rp  r:   r;   r   s        r<   rU   z_TargetArgsExpr.__init__r  s     	f%$K	6l 
__T6==?;
 
  ..DL..DL $TYY Dr?   c                n    g | |j                         }t        |       g|j                         }||fS r6   )rx  r   r   )r:   r;   rx  specs       r<   r{  z_TargetArgsExpr.simple_flatten  s:     +4*&--/*D	*FKKM*t|r?   c                    t         t        t        t        t        t        idfdt        j                  | |ffd      }t        j                  |      \  }}||fS )Nc                    t        |       }j                  |      }|!t        j                   ||       fd      S | S )Nc                    t        |       v S r6   typert  type_mappings    r<   r   zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1g&= r?   is_leaf)r  rn   pytreer   )rt  cls
convert_fnconvert_typer  s      r<   r  z4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sG    q'C%))#.J% qM= 
 Hr?   c                    t        |       v S r6   r  r  s    r<   r   z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1g5 r?   r  )rt  r   rA   r   )r   r   r   r   r  r  r   tree_flatten)r:   r;   normalized_args_treeflatr~  r  r  s        @@r<   ry  z_TargetArgsExpr.pytree_flatten  s_    
 E%D*
		  &6N5 

 (()=>
dTzr?   c                   | j                         gt        t        | j                        | j                  j                         D cg c]  \  }}| d|  c}}}| j                  t        u r|j                  d       n-| j                  dk7  r|j                  d| j                          | j                  j                   ddj                  |       dS c c}}w )N=_users=MULTIPLEr,   _users=ra  r   r   )r^  mapreprr:   r;   rt   r   rR   rp   r   r4   join)r9   rd   re   r:   s       r<   r   z_TargetArgsExpr.__repr__  s    MMO
tyy!
 &*[[%6%6%89TQ1QCj9

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
        | j                         gfd| j                  D        | j                  j                         D cg c]  \  }}| dj	                  |        c}}}| j
                  t        u r|j                  d       n-| j
                  dk7  r|j                  d| j
                          d}| j                  j                   d|j                  |       dS c c}}w )	Nc              3  @   K   | ]  }j                  |        y wr6   )r-  )rc   rt  r,  s     r<   rf   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s     4Qbooa 4s   r  r  r,   r  r   ra  r   )r^  r:   r;   rt   r-  r   rR   rp   r   r4   r  )r9   r,  rd   re   r:   
joiner_strs    `    r<   r-  z_TargetArgsExpr.pretty_print  s    MMO
4$))4
 7;kk6G6G6IJda1R__Q'()J

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks    C'c                   | j                  |      r+t        |j                        t        | j                        k7  rt        d||       S | j	                  ||      st        d|       S |j                  }|j
                  }t        |      t        | j
                        k  rddlm} t        |j                        sJ  ||j                  |j                  |j
                        }|t        d||       S |\  }}t        |      t        | j                        k(  rDt        |      t        | j
                        k\  r#|D ci c]  }|| j
                  v s|||    }}n/t        d||       S |D ci c]  }|| j
                  v s|||    }}| j                  ||      \  }}	| j                  \  }
}|	|k7  rt        d|	|      S t        |      t        |
      k(  sJ t        ||       }t        t        j                         |
|      D ]  \  }}}t!        |t"              r3|j%                  ||      }t'        |      s|c S |j)                  |       Jt!        |t*        j,                  j.                        s||k7  stt        d||      c S  |j0                  j3                  |       |j                  |j4                  | <   |S c c}w c c}w )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})ri  r   r:   r   rk  r;   torch.fx.operator_schemasr  rR  r   rz  r|  r{   r   rv  countr   r|   r  r   r   r^   r_   r   r   rp   r   )r9   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr   r}   
child_nodechild_matchs                   r<   r  z_TargetArgsExpr._match  ss   t$DII#dii.(HGtTT  s+2D99		++w<#dkk**DDKK((();TYY*& *1"#KTSWXX!;wu:TYY/CLCDT4T6=RdkkAQq'!*}RGR&@$  /6Jdkk9Iq'!*}JGJ $UG <
I $ 5 5
I	!5y)LL:#j/111#t&))//*;Z&T 		"Aw
';/!ii<,&&%J6*:O"94 		 	
t++		$7 S Ks    KK0KKc              #    K   | |j                   v r|j                   |     y| j                  d   D ]  }t        |t              s|j	                  ||      D ]d  }t        |t
        j                  j                        s(|j                  D ].  }||vs| j                  |      s| |j                  |       0 f  yw)a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   r|  r   r|   r  r^   r_   r   r   ri  add)r9   r   r  r}   
other_noder   s         r<   r  z!_TargetArgsExpr.find_anchor_nodes  s      3&&&%%d++,,Q/ 		3G';/")";";C"J 3J%j%((--@  * 0 0 3x/#t4&*
 (T 2	33		3s   ACACC&Cc                   t        j                  t        |      }t        |   |      xrW | j
                  d   |j
                  d   k(  xr6 t        d t        | j
                  d   |j
                  d         D              S )Nr,   c              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr6   r   r|   r  rc   abs      r<   rf   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  6      Aq $.a#=Q16I   35r   )r>  r?  r   r   r  r|  allr   r@  s     r<   r  z_TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r?   )
rL  z/Union[torch.fx.node.Target, str, Sequence[Any]]r:   r   rp  zUnion[int, Multiple]r;   r   rA   rW   )r:   r   r;   zMapping[Any, Any]rA   z9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r   r.  r  r  r  )r4   rB   rC   r   rU   staticmethodr{  ry  r   r-  r  r  r  r   r   s   @r<   ro  ro  m  s     ()	E<E E %	E
 E 
E& %6	B  %6	B 8
?E1f33+D3	732	
 	
r?   ro  c                      e Zd ZdZdZy)CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr4   rB   rC   r   rW  r7   r?   r<   r  r    s     
Br?   r  c                      e Zd ZdZdZy)
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  r7   r?   r<   r  r  $       
Br?   r  c                      e Zd ZdZdZy)
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  r7   r?   r<   r  r  ,  r  r?   r  c                      e Zd ZdZddZy)_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    c                   | j                  |      st        d      S | j                  ||      st        d      S t        ||       }|j                  j                  |       |j                  |j                  | <   |j                  j                  |j                         |j                  j                  |j                         |S )Nfunction_mismatchmultiple_users)ri  r   rk  r{   r   rp   r   r   r:   r   r;   rr   )r9   r   r   r   s       r<   r  z_TargetExprVarArgs._match9  s    t$233  s+/00#t	t++		$	dii 	$r?   Nr  r%  r7   r?   r<   r  r  4  s    r?   r  c                      e Zd ZdZy)CallFunctionVarArgsr  Nr4   rB   rC   rW  r7   r?   r<   r  r  H  s    	Br?   r  c                      e Zd ZdZy)CallMethodVarArgsr  Nr  r7   r?   r<   r  r  L      	Br?   r  c                      e Zd ZdZy)CallModuleVarArgsr  Nr  r7   r?   r<   r  r  P  r  r?   r  c                  B     e Zd ZdZdd fdZddZd	dZd
 fdZ xZS )ListOfz$
    Matches a repeated pattern
    c                b    t         |           t        |t              sJ || _        || _        y r6   )r   rU   r   r|   r}   r   )r9   r}   r   r   s      r<   rU   zListOf.__init__Y  s,    ';///r?   c                N    | j                   j                   d| j                   dS Nra  r   )r   r4   r}   rT   s    r<   r   zListOf.__repr___  $    ..))*!DLL>;;r?   c                (   t        |t        t        f      rt        |      dk(  rt	        d      S t        ||       }|j                         }d}t        |      D ]  \  }}t        |j                  ||j                        }|j                  | j                  |      }	|j                         }t        |	      s| j                  st	        d||	      c S {d}|j                  |	j!                                 |st	        d      S |j!                         S )Nr   non_listFr  zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   r{   r  	enumerater   r   r   r  r}   r   r   r   r   )
r9   r   r   r   r   matchedr  r  	child_ctxr  s
             r<   r  zListOf._matchb  s    $u.#d)q.z**#t 88:&t_ 	+MAz$_J4D4DI $//$,,
CK'BBDOK(||&~q+FFGHH['')*	+ /00xxzr?   c                    t        j                  t        |      }t        |   |      xr@ | j
                  j	                  |j
                        xr | j                  |j                  k(  S r6   )r>  r?  r   r   r  r}   r   r@  s     r<   r  zListOf.pattern_eqz  sU    D%(Gu% .''6.-	
r?   F)r}   r|   r   r   rA   rW   r   )r   r   r   r   rA   r
  r  rC  r   s   @r<   r  r  T  s!    <0
 
r?   r  c                  ~     e Zd ZU ded<   d fdZedd       ZddZddZddZ		 	 	 	 	 	 ddZ
dd	Zd fd
Z xZS )MultiOutputPatternr   r   c                    t         |           t        |d   t              sJ t	        d |D              sJ |       t        |      | _        |d   j                  | _        y )Nr   c              3  H   K   | ]  }|d u xs t        |t                y wr6   )r   r|   rs  s     r<   rf   z.MultiOutputPattern.__init__.<locals>.<genexpr>  s#     Lq19:
1k ::Ls    ")r   rU   r   rK  r  r   r   rW  )r9   r   r   s     r<   rU   zMultiOutputPattern.__init__  sU    '!*k222LGLLUgULG}!*--r?   c                h    t        j                  t        | j                  d         }|j                  S Nr   )r>  r?  rK  r   rL  )r9   outputs     r<   rL  zMultiOutputPattern.fns  s&     [$,,q/:zzr?   c                N    | j                   j                   d| j                   dS r  )r   r4   r   rT   s    r<   r   zMultiOutputPattern.__repr__  r  r?   c                    | j                   D cg c]  }|j                  |       }}dd }| j                  j                   d|j	                  |       }| d}|S c c}w )Nz,
z  z([z
]))r   r-  r   r4   r  )r9   r,  rt  r:   r  str_outs         r<   r-  zMultiOutputPattern.pretty_print  sh    ,0LL9q"994&\
^^,,-R
0E/FGIT"	 :s   Ac                $   t        j                  t        | j                  d         }|j	                  ||      }t        |      s|S | j                  dd  D ]7  }|| j                  ||      }t        |      s|c S |j                  |       9 |S )Nr   r,   )r>  r?  rK  r   r  r   _match_from_anchorsr   )r9   r   r   r  r   r}   r  s          r<   r  zMultiOutputPattern._match  s    [$,,q/:IIfd#{H||AB' 	"G227C@KK(""HH[!	" r?   c                    t        |j                        }t        d      }|j                  |t	                     D ]3  }|j                  ||      }t        |      r|c S t        |      |_        5 |S )Nzno anchor found)r  r   r   r  r"   r  r   )r9   r}   r   priorr   r   s         r<   r  z&MultiOutputPattern._match_from_anchors  sl     S(()$%67--c:<@ 	.D		'4(A{"&u+C	. r?   c                    	 t        | j                  |j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wr  )r   r   r   r  r   r  s      r<   r  zMultiOutputPattern.match  s>    	DJJ?EEdDQQ 	H	s   03 	AAAAc                   t        j                  t        |      }t        |   |      xr] t        | j                        t        |j                        k(  xr0 t        d t        | j                  |j                        D              S )Nc              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr6   r  r  s      r<   rf   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  r  r  )	r>  r?  r   r   r  r   r   r  r   r@  s     r<   r  zMultiOutputPattern.pattern_eq  sm    D%(Gu% DLL!S%77 emm< 	
r?   )r   zSequence[Optional[PatternExpr]]rA   rW   )rA   z-Union[Callable[..., Any], str, Sequence[Any]]r   r.  r  )r}   r|   r   r   rA   r
  r  r  )r4   rB   rC   rD   rU   r   rL  r   r-  r  r  r  r  r   r   s   @r<   r  r    s]    ((   
< ")5		
 	
r?   r  c                  J     e Zd ZdZd fdZedd       ZddZd	 fdZ xZ	S )
RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    c                R    t         |           || _        |j                  | _        y r6   )r   rU   inner_patternrW  )r9   r  r   s     r<   rU   zRepeatedExpr.__init__  s#    *""r?   c                .    | j                   j                  S r6   )r  rL  rT   s    r<   rL  zRepeatedExpr.fns  s    !!%%%r?   c                   |j                  | j                  |      }t        |      s|S |j                  j	                  | j                         | j                  j                  |t                     D ]T  }t        | g|j                        j                  | j                  |      }t        |      s|c S |j                  |       V |S r  )
r  r  r   r   popr  r"   r   r   r   )r9   r   r   r   anchor_nodeanchor_ms         r<   r  zRepeatedExpr._match  s    IId(($/{H	
  --??Z\R 	K#TF$**=CC""KH H%HHX	 r?   c                    t        j                  t        |      }t        |   |      xr% | j
                  j	                  |j
                        S r6   )r>  r?  r   r   r  r  r@  s     r<   r  zRepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
r?   )r  rK  rA   rW   )rA   zSequence[FnsType]r  r  )
r4   rB   rC   r   rU   r   rL  r  r  r   r   s   @r<   r  r    s0    #
 & &"
 
r?   r  c                  Z    e Zd ZdZddZeej                  dd	d              Zd
dZ	ddZ
y)r/  z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    c                z    t         j                  j                  j                         | _        i | _        i | _        y r6   )r^   r_   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprT   s    r<   rU   zPatternPrettyPrinter.__init__  s*    224;= 8:r?   c                   t               }t        | d      sJ | j                  |      }|j                  D cg c]#  }|j                  |    d|j                  |    % }}|j                  | d|        dj                  |      S c c}w )zU
        Serializes obj to python code with obj written out to `output_name`
        r-  )r,  z = 
)r/  hasattrr-  r  r  rp   r  )objoutput_namer,  out_strr   r  s         r<   runzPatternPrettyPrinter.run  s     "#sN+++""b") --
 %%c*+3r/B/B3/G.HI
 

 	S	23yy  
s   (B
c                    t        |t              r0| j                  j                  |      x}r|S | j	                  |      S t        |d      r|j                  |       S t        |      S )Nr-  )r   ro  r  rn   memoizer  r-  r  )r9   r  memoized_names      r<   r-  z!PatternPrettyPrinter.pretty_print  sb    c?+ $ 8 8 < <S AA}A$$||C((3'##D))Cyr?   c                    |j                  |       }|j                         }dD ]  }|j                  |d      } | j                  j	                  |d       }|| j
                  |<   || j                  |<   |S )N)zaten.rZ  zprims.r`  )r-  r^  replacer  create_namer  r  )r9   r  obj_strobj_nameprefixtmp_names         r<   r  zPatternPrettyPrinter.memoize  s{    ""4(<<>3 	4F''3H	4 >>--h=(0  %%,c"r?   NrV   )r  )r  r|   r  r3   rA   r3   )r  r   rA   r3   )r  ro  rA   r3   )r4   rB   rC   r   rU   r  r   cacher  r-  r  r7   r?   r<   r/  r/    s6    ;
 __!  !$		r?   r/  c                      e Zd Z	 	 	 	 ddZy)_PassDictsTypec                     y r6   r7   )r9   rd   s     r<   __getitem__z_PassDictsType.__getitem__*  s     r?   N)rd    tuple[str, torch.fx.node.Target]rA   list[PatternEntry])r4   rB   rC   r  r7   r?   r<   r  r  )  s    !1!	!r?   r  c                  F    e Zd ZU ded<   ded<   ddZ	 	 d		 	 	 	 	 	 	 d
dZy)PatternEntryr|   r}   Callable[[Match], bool]extra_checkc                    t         r6   re  r9   r  r   r   s       r<   applyzPatternEntry.apply4  s    !!r?   Nc                   |Ht        | j                  d      sJ | j                  j                  D ]  }| j                  |||        y t	        |t
        t        f      rqt        | j                  d      sJ |r,|| j                  j                  |f   j                  d|        y || j                  j                  |f   j                  |        y t        j                  t        t           |      }|D ]  }| j                  |||        y )NrL  prependrW  r   )r  r}   rL  registerr   r  PatternMatcherPassrW  insertrp   r>  r?  r   r  )r9   
pass_dictsr   r  rL   rt  s         r<   r  zPatternEntry.register7  s     >4<<///ll&& ?j"g>?
T+=$>?4<<...DLLOOV45<<QEDLLOOV45<<TBXn%=zJJ :a9:r?   r  r{   r   r   r   r   rA   rW   r   )r   /Union[_PassDictsType, Sequence[_PassDictsType]]r   z!Union[torch.fx.node.Target, None]r  r   rA   rW   )r4   rB   rC   rD   r  r  r7   r?   r<   r  r  /  sF    ((" 59	:C: 2: 	:
 
:r?   r  c                       e Zd ZU ded<   ddZy)LoweringPatternEntryCallable[..., Any]handlerc                    t        j                  | j                        t        j                  | j                  |            }|j	                  |      5  |j                  |t        |j                        |j                        }|j                  j                  |j                         |j                  |       d d d        |j                  d   |u sJ |j                          y # 1 sw Y   -xY w)N)r   wrapsr&  r   inserting_beforer  r   r:   r;   rs   rr   replace_all_uses_withr   r   )r9   r  r   r   r&  r   s         r<   r  zLoweringPatternEntry.applyQ  s    /)//$,,/	0A0A$,,PU0VW##D) 	4--guUZZ7H%,,WK##DII.&&{3	4 {{2$&&&	4 	4s   A'C((C1Nr!  )r4   rB   rC   rD   r  r7   r?   r<   r$  r$  M  s    r?   r$  c                  $    e Zd ZU dZded<   ddZy)GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r%  r&  c                    |j                  |      5   | j                  |g|j                  i |j                   d d d        y # 1 sw Y   y xY wr6   )r*  r&  r:   r;   r  s       r<   r  zGraphPatternEntry.applyc  sE    ##D) 	=DLL<<u||<	= 	= 	=s   +AANr!  )r4   rB   rC   r   rD   r  r7   r?   r<   r-  r-  [  s      =r?   r-  c                  F    e Zd ZU ded<   e	 	 	 	 	 	 	 	 	 	 dd       ZddZy)r   zCallable[..., list[Any]]normalize_argsc                    G fddt         j                  j                        }| j                         }t	        |      dk(  r|d   }n|d   sJ t        |d   j                  j                        }|D cg c]9  }t        |t         j                  j                        r|j                  |      |f; }	}t        |	t        j                  d            d   }	 	 	 	 	 	 	 	 	 	 d	dj                  |      5  t        |t         j                  j                        sJ   ||      j                    }
t        |
t         j                  j                        r|
g}
d
d	 	 	 	 	 	 dfdt	        |      t	        |
      k(  rt#        ||
      D ]  \  }} ||        nt	        |      dk(  sJ  |d   |
       d d d        | j%                          y c c}w # 1 sw Y   xY w)Nc                  0     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNc                p   |j                   dv rt        | 	  |      S |j                  }| j	                  |      \  }}|j                   dk(  rt        |      sJ j                  |||      }t        |j                  |d       d|j                  v r|j                  d   |j                  d<   d|j                  v r}d|j                  vro|j                  d   |j                  d<   t        |j                  d   t        j                        r,d|j                  v sJ |j                  d   |j                  d<   |S |j                   dk(  rd	d
lm} t        | 9  |||      }t        |t        j                  j                         st#        d| d| d      j$                  J d }j$                  j'                         D ]  \  }	}
||
u s|	} n |Dt        |t(              sJ  |j$                  |      \  }}j$                  j+                  ||       j                  |      S t#        d|       )N)placeholderr  r  Interpreter_Replacerr   r   r   tensor_metaget_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )rW  r   run_noder   fetch_args_kwargs_from_envrR  r  ry   rs   r   r^   Tensortorch._higher_order_ops.utilsr9  r8  r_   GraphModulerf  owning_modulenamed_modulesr3   register_module)r9   r   r   r:   r;   resultr9  sub_gm
graph_namer   mod_r   r   s               r<   r;  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_nodex  s+   7777 7+D11#>>tDf77o-#F+++"00vFF"!'!%"8 *TYY6:>))DV:W$67		)e6;;.F-1YYu-=E*%dii&6E#0DII#==#=9==9QFKK6!M77j( #W-fdFCF%fehh.B.BC15fX=YZ`Yaabc  !..:::!%J"'"5"5"C"C"E "3!S=)*J!" "))&#666(C!//): ++;;JO >>*55)Jtf*=>>r?   r   r   rA   r   )r4   rB   rC   r  r  r8  r;  r   )r   r   s   @r<   Replacerr3  s  s    KKH2? 2?r?   rI  r,   r   )r   c                   | g}t        t        j                  j                            }|rc|j	                         }||vrK||vrGt        |d      r;|j                  |       ||j                  |<   |j                  |j                         |rby y )Nrs   )
r"   r^   r_   r   r  r  r  rs   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r<   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  sy     FE /1Giikw&;.V,KK$)2CHHX&LL!4!45 r?   c                    | j                   dk7  ry | j                  t        j                  k7  ry t	        | j
                        dk(  sJ | j
                  d   S )Nr  r#   r,   )rW  r   r[  getitemr   r:   r   s    r<   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sJ    77o-;;("2"22499~***yy|#r?   c           	        | |J y t        | t        j                  j                        sJ |#| j	                  d        j                  |        y t        |t        j                  j                        rd|j                  vr%|j                  j                  | j                         dD ]2  }|| j                  v s 	||| j                  |   t                     4 | j	                  |       j                  |        y t        | j                  j                               }|D ]#  } |      }|t        d       
|||          % j                  |        y )Nr   )	recomputeac_graph_idzPDeleted index from getitem, did you erase the index and not properly replace it?)r   r^   r_   r   r+  r   rs   rr   r"   r   r   r   AssertionError)oldnewrL  old_usesuseridxr:   r   rU  rQ  r  s         r<   r  z;ReplacementPatternEntry.replace_with_graph.<locals>.replace  sH    ;;&;!#uxx}}555;--d3$$S)c588==1CHH,1 %A #sxx/* #Xsxx/A:dCS --c2$$S):  		 01$ ,D'-C{,n  D#c(+,   %r?   )
r   r   rL  r3   rM  r3   rN  r  rA   rW   rH  )rZ  zUnion[torch.fx.Node, None]r[  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]rA   rW   )r^   r_   Interpreterr   r   r   r   r   r   r   indexminr[  
itemgetterr*  r?  r  r   r   )r  r   r   r:   rI  r   	last_noder   r   indicesr   rZ  r[  rU  rQ  r  s    ` `         @@@r<   r   z*ReplacementPatternEntry.replace_with_graphl  s
   7	?uxx++ 7	?r ))+|!$QI?"?a..445E &a/ Q#G 
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I. Z	6/1E1EFFF9(#45994@K+uxx}}5*m$E&/E&HE& E& E&N < C$44 #L+ > &HCC%& <(A---Q5uZ	6x 	o6Z	6 Z	6s   <>G>CG$$G-c           
         |j                   J | j                  |||j                    | j                  |j                  i |j                         y r6   )r   r   r0  r:   r;   r  s       r<   r  zReplacementPatternEntry.apply,  sN    &&222##D<u||<		
r?   N)
r  r{   r   r   r   z+Union[torch.fx.Graph, torch.fx.GraphModule]r:   zSequence[torch.fx.Node]rA   rW   r!  )r4   rB   rC   rD   r  r   r  r7   r?   r<   r   r   h  sV    ,,}}} G} &	}
 
} }~
r?   r   c                     yr   r7   )r  s    r<   _return_truerg  6  s    r?   c                F    t         j                  d| j                  |       y )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor4   )	search_fnr  s     r<   log_trace_failurerl  :  s    HHJ	r?   Fc                p   t         j                  |       |j                        }|s'|   j                  |rt	        |             yd       y||ryt        j                  dfd       t	        |      }|D ]&  |k(  s	|r yt        j                  dfd       ( |j                  |       y)aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     d  dS )NDuplicate pattern: z with no graphr7   )pattern_reprs   r<   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>[  s    ),~F r?   c                     d d  dS )Nro  z with duplicated match graph  r7   )	graph_strrp  s   r<   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>f  s    ),7TU^T__`a r?   )r/  r  rn   rp   r3   r^   _check)r}   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strrs  rp  s         @@r<   check_and_add_duplicate_patternry  B  s     (++G4L'++L9l#**3u:I EII}F	

 JM( 
		)a	

 }-r?   r7   c
           
         g t        j                         j                  j                         d f	d}
d	fd}t        u rt        j                         ryt        j                  d      5  |D cg c]*  }t        |t
        j                        xr |j                  , c}t         |      \  }}n}d}t        |t              r|n|gD ]D  }t        |t              st        ||r|j                   nd|j"                  |	      s< ddd       y t%        ||
|      }|j'                  |       |j(                  cddd       S c c}w # 1 sw Y   yxY w)
a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    c           
       	 t              }|D ]+  }|| j                  vst        d| d| j                          t        t        j                  j                  |D cg c]  }| j                  |    c}d             g }t        j                  j                  j                        }|J |5  t              D ]  \  }}t        |   t        j                        s%|r"t        |   j                        r
 ddd       yt        j                  |   j                         |   j!                         |   j                  |   j"                  |      |<   t%        j&                  |   j(                  |   j!                               D ]C  t        t        j*                        st-        fd|D              s3|j/                         E  }|s3|rdfd}	  ||z         }	g }t3        t5        t7        |      t7              z         |	j8                  j:                        D ]  \  }}|t7        |      k  r|j/                  |j<                         0|	j8                  j?                  |      5  |	j8                  jA                  ||t7        |      z
           }|jB                  |_        |jE                  |       |	j8                  jG                  |       ddd        ||z   }n
	        }	tI        |	|	      }| jK                         d
   }|J |jM                  |      }tO        |      r |      rz       | _(        t7        | j:                        dk(  rJ| jP                  j8                  j:                  D ]'  }tS        |jT                  | j:                  d
   d       ) 	 ddd       y	 ddd       yc c}w # t        $ r}
t1        |
       Y d}
~
ddd       yd}
~
ww xY w# 1 sw Y   xY w# t        $ r}
t1        |
       Y d}
~
ddd       yd}
~
ww xY w# 1 sw Y   yxY w)z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     | j                   d   S r   r   )r   s    r<   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE] r?   NF)dtypedevicerequires_gradc              3  :   K   | ]  }t        |k7          y wr6   )r   )rc   r  re   s     r<   rf   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s       ?>?1!q&9?s   c                 >     | t        |       t              z
  d   S r6   )r   )args_newr:   rk  s    r<   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_new  s#    ((3x=3t93L3N*OPPr?   )argnamesexclusive_arg_namesscalar_workaroundr   r,   r   r   T)r  r   rA   r   )+r   r;   r   r^   r_   r   _dynamor   detect_fake_moder  r   r=  r   r}  empty_stridedsizestrider~  rv  rw  shapeSymIntr  rp   rl  r   ranger   r   r   r   inserting_afterr5  r5  r+  r   fx_to_patternr   r  r   r   ry   rs   )r  r  r5  sym_argsr   r  gradspecific_patternr  specific_graphr  sym_arg_namesr5  r   r   specific_pattern_matchr   r:   re   argnames_staticr  r  
replace_fnr  r  rk  search_fn_patternr   s                    @@r<   check_fnz&register_replacement.<locals>.check_fn  s=    ( 	D5<<'"99A
/RWR^R^Q_a 	 HH089d#9;R
 (*MM''88>	$$$ Z	$]3 /4d1gu||4 0a ?$	Z	 Z	 $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM /%a63 ?CK? < %OOA.	//*  1#Q%)1-D)Q %'M*-c(mc$i78&,,22+ I; s8},)001C1CD$+11AA+N I'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KHI II   -x7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A"./K@V4W*2:t*D'u{{#q("44::@@ &%&VV%*[[^&3 sZ	 Z	t uZ	 Z	 :Z ( %))Q7$QZ	 Z	L%I I ( %))Q7$EZ	 Z	@%AZ	 Z	s   !O50P=&P=
B*P=5P=
%P=0O<BP=A(P/P= 	P	B;P=P=	P%O>1P=>PP=PP=	P:P5(P=5P::P==Qc                     D cg c]  }| j                  |       }}t        dt        |       dz         D ].  }d| | vr n%|j                  | j                  d|              0 | r
J d|        |S c c}w )Nr,   	tangents_zleftover kwargs: )r  r  r   rp   )r;   r5  r:   r  r  s       r<   r0  z,register_replacement.<locals>.normalize_args  s    -<=T

4 ==q#f+/* 	5A1#f,KK

Yqc?34	5 9.vj99z >s   A7Ffunctionalize_rng_opsN)rv  )r}   r  r0  r  r{   rA   r   )r;   r   rA   r~   )inspect	signature
parametersr   joint_fwd_bwdr^   is_inference_mode_enabledfunctorch_configpatchr   r=  r  gen_pattern_and_search_gmr   r  ry  r   ru  r   r  r}   )rk  r  example_inputsr   r   r  r  r  r  rv  r  r0  rt  r}   gmpattern_matcher_passr  r  s   `` ` ````       @@r<   register_replacementr  l  so   2 H)))4??DDFGOr rh =  **, 
		e	< "ES%
@AJq%,,';AOO;%
 $3!#KGR (GB %Z:J
	!  .0BC2 "BHH(66$3	 !5" " 
	! * )

 	$E" "%
" "s1    E%/E 4AE%9'E%!E%++E% E%%E.zOrderedSet[str]_serialized_patternsc                V   dd}t         j                         st        dt                |j                  }ddlm}  |j                  d      5  t        ||||      }d d d        t        j                  |       }	|t        vrd}
t        j                  |       nd	}
 |       }t        t         | d
z  |
      5 }|
dk(  r|j                  |       n|j                  d       |j                  |	       |j                  d       d d d        |S # 1 sw Y   xY w# 1 sw Y   |S xY w)Nc                    t        j                  d      } t        j                  d      j                  |       }g }t        t        j
                  j                        D ]^  }t        t        j
                  j                  |      }	 t        |t              r't        |t        t        f      r|j                  |       ` dj                  |      }d| d}| | S # t        $ r Y w xY w)Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirr^   	_inductorpattern_matcherrP  r   r  
issubclassr|   rK  rp   	TypeErrorr  )auto_generated_msgfile_templatepattern_matcher_importsr5  attrformatted_importss         r<   get_file_templatez-_serialize_pattern.<locals>.get_file_template:  s    %__
 !
 &'&
( 	  #%778 	D5??::DADdD)j;4/ ,2248	 %MM*ABPQbPccij!2 344  s   7C	C*)C*z0Could not find serialized patterns directory at r   r$   Fr  )r  wr  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r4   torch._functorchr%   r  gen_patternr/  r  r  r  openwrite)unique_namerk  r  r   r  r  pattern_namer  r}   serialized_pattern
write_moder  fs                r<   _serialize_patternr  3  s)   &5P #))+>?V>WX
 	
 %%L;				e	< ViCTUV .11'{1S//
  .
%'M	%<.(<<j	I QGGM"GGFO	"#	 N)V V Ns   D=ADDD(	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                   t        |      }dt        j                  v rt        | ||||      }
nT|j                  }t        j                  d|       }|rt        ||       st        j                  d|        t        ||       }
t        j                  |      D ]'  }t        |t              s|j                  !d |_        ) t         j#                  |||||
f       t%        |||||||||
|	
       y )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  rv  )r   osenvironr  r4   	importlibimport_moduler  ri  warningrP  r  	tree_iterr   r)   constantr  rp   r  )r  rk  r  r  r   r   r  r  r  rv  patr  r   r   s                 r<   gen_register_replacementr    s    >*N+ NH>O
 !))##<\NK
 ;/KKV a%/  c:&3<<+C
  CL   &&	NH.?E 'r?   r  c                r   g t        j                  |       j                  j                         }|i }g }d}|D ]4  }||v r|j	                  ||          |j	                  ||          |dz  }6  || |      }	t        |	t        t        t        t        j                  t        j                  f|||      |	fS )Nr   r,   )ignore_typesr  r  r  )r  r  r  r   rp   r  intfloatr   r^   r~  r}  )
rk  r  r   r  r  r  flat_inputs	input_idxargname	search_gms
             r<   r  r    s     A""9-88==?@H KI ''09:~i89NI K0IudELL%++F/ 3	
 		 	r?   c                &    t        | ||||      d   S r  )r  )rk  r  r   r  r  s        r<   r  r    s&     %>8->@S	 	r?   r  c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    c                n    t        |       sJ t        |       j                         d| _        | S )Nr}   r  r&  r  T)rR  r$  r  _inductor_lowering_functionr&  r  	pass_dictr}   r  s    r<   	decoratorz,register_lowering_pattern.<locals>.decorator  s;       g	

(9g(
..2+r?   r&  r%  rA   r%  r7   r}   r  r  r  r  s   ```` r<   register_lowering_patternr    s      r?   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    c                `    t        |       sJ t        |       j                         | S )Nr  r  )rR  r-  r  r  s    r<   r  z)register_graph_pattern.<locals>.decorator  s3       g	

(9g(
.r?   r  r7   r  s   ```` r<   register_graph_patternr    s      r?   c                B    |t        t        | j                              u S r6   )r   iterr   )r   r   s     r<   is_start_of_fx_graphr  '  s    4U[[)***r?   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    | j                   dk7  ry| t        j                  j                  j                  j
                  t        j                  j                  j                  j
                  fv S )Nr0   F)r  r^   r   r0   accumulate_grad_defaultresize_storage_bytes_)rW  s    r<   "fixme_incorrect_inductor_schema_opr  1  sW    	||z! 		++33		0088  r?   c                   t        | j                  t        j                  j                        r5t        | j                        s | j                  j                  j                  S t        | j                  t        j                  j                  j                        ry| j                  dk(  rAt        | j                        sJ t        j                  | j                  j                        rLy| j                  dk(  r<t        | j                  t               sJ t        j                  | j                        ry| j"                  j%                  d      d uS )NFr  Tr  out)r   r   r^   rS  r\  r  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedrW  rR  _mutation_op_researchr4   r3   r;   rn   rT  s    r<   is_mutation_opr  =  s    UZZ**0={{""---	U,,??RR
 ww/!$$$!!$++"6"67	M	!$++s+++!!$++.;;??5!--r?   c                    d| j                   v sJ d|j                   v sJ | j                   d   |j                   d   k(  S Nmutation_region_idr   )r  r  s     r<   same_mutation_regionsr  Q  sE    166)))166)))66&'1662F+GGGr?   c                ,   |}d|j                   vr3t        | |      s'|j                  }d|j                   vrt        | |      s'|j                   j                  dd      }||ur0|j                  }t        |      r|dz  }||j                   d<   ||ur0|S )Nr  r   r,   )rs   r  prevrn   r   r  )r   r   r   r  s       r<   get_mutation_region_idr  W  s    A
aff
,5I%QR5SFF aff
,5I%QR5S$8!<
4-FF!!#'9#$	 4-
 r?   c                V    dt        t        | j                              j                  vS r  )r   r  r   rs   r  s    r<   "should_compute_mutation_region_idsr  d  s!    tD,='>'C'CCCr?   c                h    d}| j                   D ]!  }t        |      r|dz  }||j                  d<   # y )Nr   r,   r  )r   r  rs   )r   r  nds      r<   compute_mutation_region_idsr  h  s=    kk ;"!#(:$%;r?   c                  B     e Zd Z	 d	 	 	 d fdZddZddZd	dZ xZS )
r  c                    t         |           t        t              | _        || _        t        t              | _        y r6   )r   rU   r   r   patternsrw   ru  )r9   rw   r   s     r<   rU   zPatternMatcherPass.__init__q  s:     	  	 # >I=Nr?   c                     | j                   |   S r6   )r  )r9   items     r<   r  zPatternMatcherPass.__getitem__  s    }}T""r?   c                   | j                   syt        |t        j                  j                        r|j
                  }nJt        |t        j                  j                        r|}|j                  }nt        dt        |             t        |      rt        |       t        j                  t        |      }d}g }d}| j                   D ]0  \  }}|dk(  rd}|j                  |j!                  ||d             2 |r"|j                  |j!                  dd             | j"                  | j"                  nd}	t        |t        j                  j                        sJ t%        ||	      5  t'        t(        j*                  j-                  |      d	      D ]  }
t/        |
      }|
j0                  dk(  r|
j0                  |f| j                   vr9t3        |
d
      rG| j                   |
j0                  |f   D ]>  }|
j4                  r t|j6                  j9                  |
      }t;        |      r,t=        t?        tA        ||jB                                    dk7  rdtD        jF                  jI                  d      |
jJ                  k(  r-tL        jO                  d|
|
jP                  ||j6                         t;        |      stS        |jU                  |            s|dz  }|jW                  |||
       tX        tZ           dxx   dz  cc<   tX        tZ           dxx   t=        |jB                        z  cc<   A  	 d d d        |S # 1 sw Y   |S xY w)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)rW  r   sort)rW  r  r  )reverse)allow_cpu_inputsr,   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %spattern_matcher_countpattern_matcher_nodes).r  r   r^   r_   r?  r   Graphr@  r   r  r  r  r   r   r  rp   
find_nodesrw   r    sortedrv  rw  from_iterablerh  rW  r.   r   r}   r  r   r   r"   r  r   r  r  rn   r5  ri  r  r:   r   r  r  r   backend)r9   r  r   get_mutation_region_id_partialr  r   has_call_modulerW  r   rw   r   entryr   s                r<   r  zPatternMatcherPass.apply  s   }}b%((../HHEEHHNN+E$$B\]abd]e\fg  .e4'.)2):):"E*
& -- 	QJB]""&U--F-OP		Q
 LL))])GH&*nn&@DNNFW	"ehh22333#B	2 !	Sy<<UCTR  S'-77m+(= 
 9PUV!]]DGGV+<= SE||++D1A !&s+I177'ST 
 !zz~~&IJdiiWL$		1emmT{~e6G6G6J'K
Aud3 )*ABaGB )*ABc!''lRB)S S!	SD E!	SD s   .EMMA!MMc                8    | j                   j                          y r6   )r  clearrT   s    r<   r!  zPatternMatcherPass.clear  s    r?   r6   )rw   Optional[str]rA   rW   )r  r  rA   r  )r  +Union[torch.fx.GraphModule, torch.fx.Graph]rA   r  rV   )r4   rB   rC   rU   r  r  r!  r   r   s   @r<   r  r  p  s2     $(O O 
O #?Br?   r  c                     t         r6   re  r   s     r<   _not_implementedr%    s    
r?   c                  	
 |xs i }|j                         D ci c]  \  }}||
 c}}
t        
      t        |      k(  sJ 	 d	 	 	 	 	 d
fdt        j                         	 G 	fddt        j
                  j                        }t        | t        j
                  j                        sJ  ||       j                         }t        |t              st        t        j                  |            S |S c c}}w )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    ||n}t        | t        t        f      r| v rt        |          S t	        |       |v r
t               S t        | t              rt        d | D              r| r
t               S | S )Nc              3  <   K   | ]  }t        |t                y wr6   )r   r'  )rc   ys     r<   rf   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s     &I!z!W'=&Is   )r   r  r  r1  r  r'  r   r  )rt  ignore_types_overridecurrent_ignore_typesr  inv_scalar_workarounds      r<   process_argz"fx_to_pattern.<locals>.process_arg  sy     &;%F!L 	 a%&10E+E3A6777**9a3&Iq&I#Ia9r?   c                  l     e Zd ZeZeZeZ	 	 	 	 	 	 	 	 dfdZ	 	 	 	 	 	 	 	 dfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterc                    t              }|t              k  r|   }n1r|j                  d      sJ |}nt        j                  dd|      }|}|v rt        |      S t        |      S )Ntangentz_\d+$r`  )r   r   
startswithresubrE  r1  )	r9   r   r:   r;   r   r5  r  argnumr  s	         r<   r5  z,fx_to_pattern.<locals>.Converter.placeholder  su     VA3x= {((333"f5***400!$''r?   c                b   	}|t         j                  k(  rt        d D              f	 	 	 	 	 d	fd}|}t        j                  |||f      \  }}t
        v r>|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}t        |g|i |S c c}w c c}}w )Nc              3  2   K   | ]  }|t         us|  y wr6   )r  )rc   ts     r<   rf   zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      Q1C<Qs   c                     | |      S r6   r7   )rt  r*  r-  s     r<   process_arg_fn_implzKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q*?@@r?   rt  rN   r*  zOptional[Sequence[type[Any]]]rA   zUnion[T, KeywordArg, Ignored])r[  rS  r   r  r   r   rt   r  )
r9   r   r:   r;   process_arg_fnr:  r  rd   r  r-  s
           r<   r  z.fx_to_pattern.<locals>.Converter.call_function  s     )N))) LQ Q#/Q LAA+HA
 3A "5!??>D&>JLD&|#378aq)88;A<<>J41a!^A..JJ8888 9Js   B&B+c                l   t         |   |      }|j                  dk(  rzt        |t              rj|j
                  d   }t        |t              sJ t        |      t        |      k(  sJ t        ||      D ]  \  }}t        |j                        |_	        ! |S t        |j                        |_	        |S )Nr  r   )
r   r;  rW  r   r   r:   r   r   r   r   )r9   r   rvr:   rr   r   s         r<   r;  z)fx_to_pattern.<locals>.Converter.run_node"  s    !!$BttxJr5$9vvay!$
3332w#d)+++!"dm -FAs!#))nAG- I qww<Ir?   )r   r3   r:   r   r;   Mapping[str, Any]rA   z&Union[ExclusiveKeywordArg, KeywordArg])r   r3   r:   r   r;   r@  rA   r|   )r   r   rA   r   )r4   rB   rC   r%  r  r  r8  r5  r  r;  r   )r   r  r5  r  r  r-  s   @r<   	Converterr/    sr    &&#	(	(  	( &		(
 4	((	9	9  	9 &		9
 	96
	 
	r?   rA  r6   r;  )rt   r   rv  r  r^   r_   r_  r   r?  r  r|   r  r  tree_leaves)r  r  r  r  r  rd   re   rA  r}   r5  r,  r-  s    `` `    @@@r<   r  r    s     */R.?.E.E.GHdaQTH$%->)???? FJ%B	& __F> >EHH(( >@ b%((..///m!Gg{+!&"4"4W"=>>No Is   DT)r   get_decomp_fnc               R   t               5  t               5  | |       n	t               } t        | |d      | }ddd       ddd       ddlm} |r, |j                         |j                  j                          j                          |S # 1 sw Y   WxY w# 1 sw Y   [xY w)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder,   remove_noop_ops)	r   r!   r-   r   fx_passes.post_gradrH  r   eliminate_dead_code	recompile)rL   r:   r   rC  decompositionsr  rH  s          r<   r   r   5  s     
"	# E%7%9 E,8MO>Q>S 	 >WRf=tD	E E 5!
$$&LLNIE E E Es!   B%BBB	BB&c           
        d	 	 	 	 	 	 	 	 dfd}t         j                  j                  d      5   t        | d |t	               dd      |  ddd       sJ ddlm}  |j                         dd	lm	} t               }t        t         j                  j                  j                  j                  t!        d
      t!        d            }t#        ||t$              j'                  |j(                         |j+                  j                         t         j,                  j                  j/                         j                  _        j                  j3                          j5                          S # 1 sw Y   *xY w)z=Build a normalized training graph, for use with fx_to_patternNc                <    rJ t        |       t        | |fi |S r6   )clone_graphr(   )joint_graphinputsr;   r  s      r<   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graphT  s(     v% f???r?   c                    t        |       S r6   )r'   )gr  s     r<   r   zjoint_fwd_bwd.<locals>.<lambda>_  s    + r?   TF)partition_fnrL  keep_inference_input_mutations
enable_logr,   rG  )pointless_viewr   r  )r}   r&  r  )rP  rM   rQ  r   r;   r   rA   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])r^   _guardstracingr&   r-   rI  rH  r   fx_passes.joint_graphrX  r  r  r   atenviewr  r1  r-  rg  r  r  r  r_   CodeGen_codegenrJ  rK  )rL   r:   rR  rH  rX  matcher_passr}   r  s          @r<   r  r  O  sJ    *.B@)@3@@LO@	:@ 
		t	$ 	
++.0+/	
 	 I24BHH5%'L		##Z%6
68JG \h|$$%rxx  ..0BHHHH  "LLNI? s   E77Fc                    g }t         j                  j                  | j                  | j                  f|j
                         |S r6   )r^   r_   r   r:   r;   rp   )r   r:   s     r<   r  r  ~  s1    )+D	HHaffahh'5Kr?   c                T   t        t        | j                              }t        t        j
                  j                            }t        t               }d }|r|j                         }t        |      D cg c]	  }||vs| }}|r||d      j                  |       n^|j                  |       |r|j                  |ur|j                  |       |}|j                  t        |j                  |d                   |r|s!t        |      t        | j                        k(  sJ y c c}w )Nr(  r7   )r   r   r   r"   r^   r_   r   r   r  r  rp   r  r   r   r   )r   pendingreadywaitingcursorr   rt  waiting_fors           r<   stable_topological_sortrh    s     8EKK()G uxx}}%'E $G F
{{}"'+@Q%q@@ KO$++D1IIdO&++T1d#F NN8GKKb$9:;   3u:U[[)99999 As   4	D%>D%c                j     t         j                  t        j                         d fd              }|S )z0Wrapper around lazy init functions in fx_passes/c                 J   t         t           j                         } t        j                  j                  d       5  t               5  t               5          }d d d        d d d        d d d        | t         t        <   S # 1 sw Y   (xY w# 1 sw Y   ,xY w# 1 sw Y   0xY wr6   )r   r  ro   r^   rY  rZ  r   r*   )counters_refrC  rL   s     r<   	lazy_initz%init_once_fakemode.<locals>.lazy_init  s      (--/]]""4( 	*@*B 	NDT 	TF	 	 	 )	 	 	 	 	 	s;   BBBB"BB
BB	BB")rA   r   )r   r  r)  )rL   rl  s   ` r<   init_once_fakemoderm    s4     ____R	  	 r?   c                     d fd}|S )z2Function for extra_check to put pass behind a flagc                $    t        t              S r6   )rP  r%   )r  r5  s    r<   
flag_checkzconfig_flag.<locals>.flag_check  s    vt$$r?   )r  r{   rA   r   r7   )r5  rp  s   ` r<   config_flagrq    s    % r?   c                L     G d dt               } ||       j                         S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphc                J   t         |   |      }t        |t        j                  j
                        rn|j                  j                  j                  |j                         | j                  j                  j                  |j                  d       |j                  _        |S r6   )r   r;  r   r^   r_   Proxyr   rs   rr   	new_graph_graph_namespacer  r5  )r9   rv   r   r   s      r<   r;  z'clone_graph.<locals>.CopyGraph.run_node  sq    w'1H(EHHNN3""))(--8%)^^%D%D%P%PMM4&" Or?   )rv   r   rA   r   )r4   rB   rC   r;  r   r   s   @r<   	CopyGraphrt    s    	 	r?   ry  )r+   	transform)input_graphry  s     r<   rO  rO    s$    K  [!++--r?   _seen_patternsc                    t        | j                        |kD  r| j                  |   S |y | j                  j                  |      S r6   )r   r:   r;   rn   )r   
arg_number
kwarg_names      r<   get_arg_valuer    s?     499~
"yy$$		{{z**r?   c           	        |g}t        |t        j                  j                        r7|j	                  |j                         D cg c]  }t        ||       c}       | D cg c]  }|j                  |v s| c}S c c}w c c}w r6   )r   r^   rS  rT  r   rU  rP  r   )r   rL   rL  rQ  r   s        r<   filter_nodesr    sf    $C"ejj112

",,.IhGB)IJ"9TdkkS&8D99 J9s   A>#B7Bc                    | j                   dk(  rPt        | j                  t              sJ t	        | j
                  j                  | j                        j                  S | j                  S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )rW  r   r   r3   r   r   r@  r   rT  s    r<   rh  rh    sO    
 ww-$++s+++114;;?III;;r?   )r`  )ru   r   rv   r   rw   r3   rA   rW   )r   r
  rA   zTypeIs[Match]r  )rk  r%  r  r   rA   rW   r  )
r}   r|   r   zOptional[torch.fx.Graph]ru  zdict[str, list[Optional[str]]]rv  r   rA   r   )rk  r2   r  rF   r  Iterable[Any]r   rJ   r   r"  r  r  r  )Union[dict[str, Union[float, int]], None]r  Sequence[str]r  zUnion[PatternExpr, None]rv  r   rA   r   )r  r3   rk  r2   r  r   r   rJ   r  r  rA   r|   )r  r3   rk  r2   r  rF   r  r  r   rJ   r   r"  r  r  r  r  r  r  rv  r   rA   rW   )Nr7   )rk  r2   r  r   r   rJ   r  r  r  r  rA   z(tuple[PatternExpr, torch.fx.GraphModule])rk  r2   r  r   r   rJ   r  r  r  r  rA   r|   )
r}   r|   r  r  r  r  r  r   rA   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r   r   r   rA   r   )rW  ztorch._ops.OpOverloadrA   r   rm  )r  r   r  r   rA   r   )r   r   r   r   rA   r  )r   r   rA   r   )r   r   rA   rW   )r:   r   r;   r   rA   r   )r7   r7   Nr7   )r  r#  r  zSequence[type[Any]]r  r  r  r  r  r  rA   r|   )
rL   r%  r:   r   r   r   rC  zOptional[Callable[..., Any]]rA   rM   )rL   r%  r:   r   rA   rM   )r   r   rA   zlist[torch.fx.node.Argument])rL   r%  rA   zCallable[[], Any])r5  r3   rA   zCallable[[Match], Any])r{  rM   rA   rM   r6   )r   r   r~  r  r  r"  rA   r   )r   zIterable[torch.fx.Node]rL   r   rA   r   )r   r   rA   ztorch.fx.node.Target)r   
__future__r   r   dataclassesr   r  r  rv  loggingr[  r  r3  r  r>  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   pathlibr   r   r   r   r   r   r   r   typing_extensionsr   r   r^   torch._guardstorch.fxtorch.utils._pytreer   r   r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   r   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    rk   r!   torch.utils._ordered_setr"   
_functorchr%   r  _functorch.aot_autogradr&   r'   _functorch.partitionersr(   _subclassesr)   r*   r_   r+   r`  decompositionr-   loweringr.   	getLoggerr4   ri  r   r\  primsConstantr   r	  r  rn   r  r2   rF   rJ   rN   r   Targetr3   FnsTyperP   rR   ry   r{   r   r   r
  r   r   r|   r!  r'  r1  rE  rK  r   _SimpleSpecro  r  r  r  r  r  r  r  r  r  r  r/  r  	dataclassr  r$  r-  r   rg  rl  ry  r  r  rD   r  __file__parentr  r  r  r  r  r  r  r  r  compiler  r  r  r  r  r  r  r  r%  r  no_gradr   enable_gradr  r  rh  rm  rq  rO  r|  r  r  rh  r7   r?   r<   <module>r     s  !F #         	 	   # # N N  N N N *    $ $ ; ( 0 @ 6 W + I K 1 / 3 C 7 4   . ; g!yy~~		x./
**..>

K>x >> >#h # CL $$c)
*? ? : IK??(5?BE?	?8A AH, 8 E;&''
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHol
k l
^?    (, * * ,
[ ,
^F
 F
R#
; #
L4 4n!X ! : : :: 
< 
 
 	= 	= 	= J
l J
 J
Z "	''#' 2' 	'
 
'` ,8CG)+26!AAA "A 	A
 @A )A AA 'A 0A A 
AH )3 o 4LLL "L 	L
 AL L^ x.//+=@UU     $ ,8CG)+!555 5 "	5
 5 @5 )5 A5 '5 5 
5p e4
 DH)+!  A	
 ' . 5L DH)+				!		 		 A			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ "**VW	.(H
D;V Vr )+ CG)+e3e%e e A	e
 'e eP 
 #'26
  	
 0  2 + +\!:H&. #-, . GK+
+%(+6C++:r?   