
    g3fi                     @   U d Z ddlmZ ddlZddlZddlZddlmZmZm	Z	 ddl
mZmZmZ ddlmZ ddlZddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ erddlm Z  ddl!m"Z"  ejF                  e$      Z% ejL                         Z'de(d<   ddZ) G d de      Z*y)z2A tracer that runs evaluators over completed runs.    )annotationsN)FutureThreadPoolExecutorwait)TYPE_CHECKINGAnycast)UUID)EvaluationResultEvaluationResults)	langchain)run_copy)
BaseTracer)tracing_v2_enabled)_get_executor)Sequence)Runz)weakref.WeakSet[EvaluatorCallbackHandler]_TRACERSc                 R    t        t              D ]  } | | j                           y)zWait for all tracers to finish.N)listr   wait_for_futures)tracers    _/var/www/auto_recruiter/arenv/lib/python3.12/site-packages/langchain_core/tracers/evaluation.pywait_for_all_evaluatorsr      s'    x. &##%&    c                  H    e Zd ZU dZdZded<   dZded<   	 ded	<   	 d
Zded<   	 dZded<   	  e	j                         Zded<   	 dZded<   	 dZded<   	 ded<   ded<   	 	 	 	 	 d 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d! fdZd"dZe	 	 	 	 d#d       Z	 d$	 	 	 	 	 	 	 d%dZd&dZd'dZ xZS )(EvaluatorCallbackHandlerzTracer that runs a run evaluator whenever a run is persisted.

    Attributes:
        client : Client
            The LangSmith client instance used for evaluating the runs.
    evaluator_callback_handlerstrnameNUUID | None
example_idzlangsmith.Clientclient  Sequence[langsmith.RunEvaluator]
evaluatorszThreadPoolExecutor | Noneexecutorzweakref.WeakSet[Future]futuresTboolskip_unfinished
str | Noneproject_namez-dict[tuple[str, str], list[EvaluationResult]]logged_eval_resultszthreading.Locklockc                    t           di | t        |t              rt	        |      n| _        |xs t        j                          _        | _	        |t                _        n7|dkD  r+t        |       _        t        j                    fd       nd _        t        j                  t         d              _        | _        | _        i  _        t+        j,                          _        t0        j3                          y)a(  Create an EvaluatorCallbackHandler.

        Args:
            evaluators : Sequence[RunEvaluator]
                The run evaluators to apply to all top level runs.
            client : LangSmith Client, optional
                The LangSmith client instance to use for evaluating the runs.
                If not specified, a new instance will be created.
            example_id : Union[UUID, str], optional
                The example ID to be associated with the runs.
            skip_unfinished: bool, optional
                Whether to skip unfinished runs.
            project_name : str, optional
                The LangSmith project name to be organize eval chain runs under.
            max_concurrency : int, optional
                The maximum number of concurrent evaluators to run.
        Nr   )max_workersc                 P    t        d j                        j                  d      S )Nr   T)r   )r	   r'   shutdownselfs   r   <lambda>z3EvaluatorCallbackHandler.__init__.<locals>.<lambda>g   s"    14==AJJPTJU r   r$   )super__init__
isinstancer   r
   r"   langchain_tracer
get_clientr#   r&   r   r'   r   weakreffinalizeWeakSetr   r(   r*   r,   r-   	threadingLockr.   r   add)	r4   r&   r#   r"   r*   r,   max_concurrencykwargs	__class__s	   `       r   r7   z!EvaluatorCallbackHandler.__init__@   s    6 	"6" *:s ;D 	 = 0 ; ; =$")ODMq .?KDMU
 !DMvd|46.(#% NN$	Tr   c           	     `   	 | j                   | j                  j                  ||      }|g}t        | j                   dg| j                        5 }|j                  r%| j                  j                  |j                        nd}|j                  ||      }| j                  |||j                  r|j                  j                  nd      }ddd       t        |j                        }| j                  5  D ]O  }	t        t!        |	d|j                              }
| j"                  j%                  |
|fg       j'                  |	       Q 	 ddd       y# 1 sw Y   xY w# t        $ r7 t        j                  d|j                  |j                  j                          w xY w# 1 sw Y   yxY w)zEvaluate the run in the project.

        Args:
            run: The run to be evaluated.
            evaluator: The evaluator to use for evaluating the run.
        Neval)r,   tagsr#   )example)source_run_idzError evaluating run %s with %starget_run_id)r,   r#   evaluate_runr   reference_example_idread_example_log_evaluation_feedback
latest_runid	Exceptionlogger	exceptionrC   __name__r   r.   getattrr-   
setdefaultappend)r4   run	evaluatoreval_resulteval_resultscbreference_exampleevaluation_resultr"   resrun_ids              r   _evaluate_in_projectz-EvaluatorCallbackHandler._evaluate_in_projectr   s   	  ("kk66sIF +}#!..fXdkk  // KK,,S-E-EF "
 %.$:$:- %; %!
  $<<%68mm"--"2"2  =  2 112
YY 	# WS/366BC((33VZ4H"MTT	 	5 $  	1##,,
 		 	s3   AE! A=EE! 6AF$EE! !A F!$F-c                    t        | t              r| g}|S t        | t              rd| v r| d   }|S dt        |        d}t	        |      )NresultszInvalid evaluation result type z1. Expected EvaluationResult or EvaluationResults.)r8   r   dicttype	TypeError)rb   results_msgs      r   _select_eval_resultsz-EvaluatorCallbackHandler._select_eval_results   sl     g/0yH  &9+?y)H 	 2$w- AC C  C. r   c                   | j                  |      }|D ]  }i }|j                  ri |j                  |}t        |dd       }||j                  }| j                  j                  ||j                  |j                  |j                  |j                  |j                  ||j                  xs |t        j                  j                  j                  	        |S )NrI   )scorevaluecomment
correctionsource_inforH   feedback_source_type)rh   evaluator_inforT   rO   r#   create_feedbackkeyrj   rk   rl   rm   rH   	langsmithschemasFeedbackSourceTypeMODEL)r4   evaluator_responserW   rH   rb   r^   source_info_run_id_s           r   rM   z1EvaluatorCallbackHandler._log_evaluation_feedback   s     ++,>? 	C+-L!!E#"4"4EEc?D9G&&KK''iiii>>(!//@=%.%6%6%I%I%O%O ( 
	$ r   c                   | j                   r-|j                  s!t        j                  d|j                         yt        |      }| j                  |_        | j                  D ]a  }| j                  | j                  ||       "| j                  j                  | j                  j                  | j                  ||             c y)z_Run the evaluator on the run.

        Args:
            run: The run to be evaluated.
        zSkipping unfinished run %sN)r*   outputsrQ   debugrO   r   r"   rK   r&   r'   r`   r(   r@   submit)r4   rW   run_rX   s       r   _persist_runz%EvaluatorCallbackHandler._persist_run   s     LL5svv>}$(OO! 	I}}$))$	:  MM(()B)BD)T		r   c                .    t        | j                         y)z!Wait for all futures to complete.N)r   r(   r3   s    r   r   z)EvaluatorCallbackHandler.wait_for_futures   s    T\\r   )NNTr&   N)r&   r%   r#   zlangsmith.Client | Noner"   zUUID | str | Noner*   r)   r,   r+   rA   z
int | NonerB   r   returnNone)rW   r   rX   zlangsmith.RunEvaluatorr   r   )rb   $EvaluationResult | EvaluationResultsr   list[EvaluationResult])N)rw   r   rW   r   rH   r!   r   r   )rW   r   r   r   r   r   )rS   
__module____qualname____doc__r    __annotations__r"   r&   r'   r;   r=   r(   r*   r,   r7   r`   staticmethodrh   rM   r   r   __classcell__)rC   s   @r   r   r   &   sL    -D#,"J"2E35J058*.H'.C'6w'8G$8A OT H#L*#JFF

 +/(, $#/&*040 (0 &	0
 0 !0 $0 0 
0d*X 5	 & &*	@  #	
 
 6&r   r   r   )+r   
__future__r   loggingr>   r;   concurrent.futuresr   r   r   typingr   r   r	   uuidr
   rs   langsmith.evaluation.evaluatorr   r   langchain_core.tracersr   r9   langchain_core.tracers._compatr   langchain_core.tracers.baser   langchain_core.tracers.contextr    langchain_core.tracers.langchainr   collections.abcr   langchain_core.tracers.schemasr   	getLoggerrS   rQ   r=   r   r   r   r   r$   r   r   <module>r      sz    8 "    ? ? + +   N @ 3 2 = :(2			8	$6Egoo6G
3 G&xz xr   