
    3fi3                   @   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZ d dlmZmZ d dlmZmZmZ d dlmZ d dlmZmZmZmZmZmZ d d	lm Z m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d)dZ+ ejX                  e-      Z.dZ/dZ0dZ1dZ2d*dZ3	 	 	 	 	 	 	 	 d+dZ4d,dZ5e G d d             Z6e G d d             Z7d-dZ8 G d d      Z9 G d d      Z: G d d       Z; G d! d"      Z< G d# d$      Z= G d% d&e#      Z> G d' d(e$      Z?y).    )annotationsN)ThreadPoolExecutor)	dataclassfield)datetimetimezone)EmptyFullQueue)MappingProxyType)AnyAsyncIteratorCallableDictListUnion)AgentActionAgentFinish)AsyncCallbackHandlerBaseCallbackHandler)BaseMessage)	LLMResultguard_importc                     t        d      t        dd      t        d      t        d      t        d      t        d      t        d	      t        d
      t        d      t        d      t        d      fS )z4Import `google-cloud-bigquery` and its dependencies.zgoogle.cloud.bigqueryzgoogle.authzgoogle-auth)pip_namez$google.api_core.gapic_v1.client_infozFgoogle.cloud.bigquery_storage_v1.services.big_query_write.async_clientzgoogle.cloud.exceptionsz@google.cloud.bigquery_storage_v1.services.big_query_write.clientzgoogle.cloud.storagezgoogle.cloud.bigquery.schemaz&google.cloud.bigquery_storage_v1.typeszgoogle.api_core.exceptionspyarrowr        t/var/www/auto_recruiter/arenv/lib/python3.12/site-packages/langchain_google_community/callbacks/bigquery_callback.pyimport_google_cloud_bigqueryr!      sy     	,-]];;<T	
 	./N	
 	+,34=>12Y r            zlangchain-bq-agent-analyticsc                   t        | t              r!|dk7  rt        |       |kD  r
| d| dz   dfS | dfS t        | t              r8d}i }| j	                         D ]  \  }}t        ||      \  }}|rd}|||<    ||fS t        | t        t        f      rBd}g }| D ]&  }	t        |	|      \  }}|rd}|j                  |       (  t        |       |      |fS | dfS )z:Recursively truncates string values within a dict or list.N...[TRUNCATED]TF)

isinstancestrlendictitems_recursive_smart_truncatelisttupleappendtype)
objmax_lentruncated_anynew_dictkvvaltruncnew_listis
             r    r-   r-   7   s   #sb=SX/x=#33T99Ez	C	IIK 	DAq21g>JC $HQK		
 &&	C$	' 	!A21g>JC $OOC 		!
 tCy"M11:r   c                   d$fd}d$fd}d$fd}d$fd}d$fd}t        i dj                  dj                  dj                  d	j                  d
|dj                  dj                  dj
                  dj                  dj                  dj
                  d|d|dj
                  d|d|      dddddiddiddd%fdd&fdd'fd d'fd!g }| D ]  }	 |	      }
|
s|j                  |
         t        |      t        |       k7  rt        j                  d"       y#j                  |      S )(z/Converts a BigQuery schema to a PyArrow schema.c                 *     j                  dd       S )Nustz	timestamp	pa_modules   r    _pyarrow_datetimez;_bigquery_schema_to_arrow_schema.<locals>._pyarrow_datetimeX   s    ""4D"11r   c                 (     j                  dd      S )N&   	   )
decimal128rC   s   r    _pyarrow_numericz:_bigquery_schema_to_arrow_schema.<locals>._pyarrow_numeric[   s    ##B**r   c                 (     j                  dd      S )NL   rG   )
decimal256rC   s   r    _pyarrow_bignumericz=_bigquery_schema_to_arrow_schema.<locals>._pyarrow_bignumeric^   s    ##B++r   c                 &     j                  d      S )Nr>   )time64rC   s   r    _pyarrow_timez7_bigquery_schema_to_arrow_schema.<locals>._pyarrow_timea   s    %%r   c                 *     j                  dd      S )Nr>   UTCr?   rA   rC   s   r    _pyarrow_timestampz<_bigquery_schema_to_arrow_schema.<locals>._pyarrow_timestampd   s    ""4E"22r   BOOLBOOLEANBYTESDATEDATETIMEFLOATFLOAT64	GEOGRAPHYINT64INTEGERJSONNUMERIC
BIGNUMERICSTRINGTIME	TIMESTAMPs   google:sqlType:geographys   {"encoding": "WKT"})   ARROW:extension:names   ARROW:extension:metadatare   s   google:sqlType:datetime   google:sqlType:json)r\   rY   r_   )RECORDSTRUCTc                &    j                  |       S N)get)bigquery_scalar_BQ_TO_ARROW_SCALARSs    r    _bigquery_to_arrow_scalarszD_bigquery_schema_to_arrow_schema.<locals>._bigquery_to_arrow_scalars   s    #''88r   c                V    |       }|rsj                  | j                  r| j                  j                         nd      }| j                  j                         dk7  }j	                  | j
                  |||      S t        j                  d| j
                  | j                         y )N REQUIRED)nullablemetadataz=Could not determine Arrow type for field '%s' with type '%s'.)rk   
field_typeuppermoder   nameloggerwarning)bigquery_field
arrow_typers   rr   &_BQ_FIELD_TYPE_TO_ARROW_FIELD_METADATA_bigquery_to_arrow_data_typerD   s       r    _bigquery_to_arrow_fieldzB_bigquery_schema_to_arrow_schema.<locals>._bigquery_to_arrow_field   s    6~F
=AA5C5N5N))//1TVH &**002j@H??##Z(X #   	K%%	

 r   c                    g }| j                   D ]J  } |      }|r|j                  |       t        j                  d| j                  |j                          y  j                  |      S )Nz@Failed to convert STRUCT/RECORD field '%s' due to subfield '%s'.)fieldsr0   rx   ry   rw   struct)r   arrow_fieldssubfieldarrow_subfieldr~   rD   s       r    #_bigquery_to_arrow_struct_data_typezM_bigquery_schema_to_arrow_schema.<locals>._bigquery_to_arrow_struct_data_type   sk     
	H5h?N##N3VJJMM
 
	 --r   c                H   | j                   dk(  rM j                  | j                  | j                  | j                              }|r	j                  |      S d S | j                  r| j                  j                         nd}|v r |       S  |      }|r |       S y )NREPEATED)r   rp   )rv   SchemaFieldrw   rt   r   list_ru   )
r   innerfield_type_upperconstructor_STRUCT_TYPESr}   rn   r   bq_schema_clsrD   s
       r    r}   zF_bigquery_schema_to_arrow_schema.<locals>._bigquery_to_arrow_data_type   s    ::#0))JJ 0 0 * E
 .39??5)<<7<7G7G5++113R},6u==01AB= r   z3Failed to convert schema due to one or more fields.Nreturnr   )rl   r)   r   zCallable[[], Any] | None)rz   r   r   r   )r   r   r   r   )r   bool_binarydate32float64stringint64r0   r*   rx   errorschema)bq_schema_listr   rD   rE   rJ   rN   rQ   rT   r   rz   r   r|   rm   r   r}   r~   rn   r   s    ``        @@@@@@@r     _bigquery_schema_to_arrow_schemar   R   s   2+,&3 ,	
IOO	
y	
 Y%%	
 I$$		

 )	
 Y&&	
 y((	
 ))	
 Y__	
 y	
 I$$	
 '	
 -	
 i&&	
 M	
  +!	
. &A)?
 -.HI(*@A.* )M9".   L( '(8&'
 <C//JKL))r   c                l   | j                  dddd      | j                  dddd	      | j                  d
ddd      | j                  dddd      | j                  dddd      | j                  dddd      | j                  dddd      | j                  dddd      | j                  dddd      | j                  dddd      | j                  ddd| j                  ddd      | j                  d dd      | j                  d!dd      | j                  d"d#d      | j                  d$dd      | j                  d%dd      gd&'      | j                  d(ddd)      | j                  d*ddd+      | j                  d,ddd-      | j                  d.ddd/      | j                  d0d1dd2      gS )3z1Returns the BigQuery schema for the events table.rB   rd   rq   z*The UTC timestamp when the event occurred.)rv   description
event_typerb   NULLABLEzThe category of the event.agentzThe name of the agent.
session_idz1A unique identifier for the conversation session.invocation_idz&A unique identifier for a single turn.user_idzThe identifier of the end-user.trace_idzOpenTelemetry trace ID.span_idzOpenTelemetry span ID.parent_span_idzOpenTelemetry parent span ID.contentr_   z!The primary payload of the event.content_partsrg   r   	mime_type)rv   uritext
part_indexr^   part_attributesstorage_modez9For multi-modal events, contains a list of content parts.)rv   r   r   
attributeszArbitrary key-value pairs.
latency_mszLatency measurements.statuszThe outcome of the event.error_messagezDetailed error message.is_truncatedrV   z)Flag indicating if content was truncated.)r   )bigquery_modules    r    _get_bigquery_events_schemar      s    	##D	 	$ 	
 	##4	 	$ 	
 	##XJ<T 	$ 	
 	##K	 	$ 	
 	##@	 	$ 	
 	##9	 	$ 	
 	##1	 	$ 	
 	##0	 	$ 	
 	##7	 	$ 	
 	##;	 	$ 	
 	##++K
+S++E8*+M++FH:+N++L)*+U++%xj ,   ++NH:+V	 T 	$ 	
  	##4	 	$ 	
 	##/	 	$ 	
 	##3	 	$ 	
 	##1	 	$ 	
 	##C	 	$ 	
Eh hr   c                  F    e Zd ZU dZded<   dZded<   dZded<   d	Zded
<   y)RetryConfig   intmax_retries      ?floatinitial_delayg       @
multiplier      $@	max_delayN)__name__
__module____qualname__r   __annotations__r   r   r   r   r   r    r   r   7  s*    KM5JIur   r   c                      e Zd ZU dZded<   dZded<   dZded<   dZd	ed
<   dZded<    e	d       Z
ded<   dZded<    e	e      Zded<   dZd	ed<   dZded<   dZded<   dZd	ed<   dZded<   dZded <   y)!BigQueryLoggerConfigTboolenabledNzlist[str] | Noneevent_allowlistevent_denylisti  r   max_content_lengthagent_events_v2r)   table_idc                 
    g dS )N)r   r   r   r   r   r   r    <lambda>zBigQueryLoggerConfig.<lambda>G  s     B r   )default_factoryz	list[str]clustering_fieldslog_multi_modal_contentr   retry_config   
batch_sizer   r   batch_flush_intervalr   shutdown_timeouti'  queue_max_size
str | Nonegcs_bucket_nameconnection_id)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   ?  s    GT(,O%,'+N$+((%Hc%#(B$y  %)T( %k BL+BJ"%%%"e"NC"&OZ& $M:$r   r   c                   ddl }|D ci c]  }|j                  g  }}| D ]  }|D ]  }|j                  |j                        }|j                  |j                        j                  }d}|rd|v r
|d   dk(  rd}|j                  |j                        j
                  }	|j                  j                  |	      }
|j                  j                  |	      }|r|:t        |t        t        f      r	 t        j                  |      }nt        |t         t"        f      rt        |t"              r	 |j%                  d      }d}t        |t               rJ|j)                         }|j+                  d      r)|j-                  d	      r	 t        j.                  |       d}|sv	 t        j                  |      }n_	 t        j                  |      }nHt        |t        t        f      r2|
s0|s.|,t        |t         t"        f      s	 t        j                  |      }||j                     j1                  |         |j2                  j5                  ||
      S c c}w # t        t        f$ r t!        |      }Y bw xY w# t&        $ r t!        |      }Y Qw xY w# t        t        f$ r Y w xY w# t        t        f$ r t!        |      }Y w xY w# t        t        f$ r t!        |      }Y w xY w# t        t        f$ r t!        |      }Y w xY w)z3Prepares a PyArrow RecordBatch from a list of rows.r   NFre   rf   Tutf-8){[)}]r   )r   rw   rk   r   rs   r1   types	is_structis_listr(   r+   r.   jsondumps	TypeError
ValueErrorr)   bytesdecodeUnicodeDecodeErrorstrip
startswithendswithloadsr0   RecordBatchfrom_pydict)rowsarrow_schemapaschema_fielddatarowvaluefield_metadatais_jsonarrow_field_typer   r   is_already_jsonstrippeds                 r    _prepare_arrow_batchr   S  s   GST|\..2TDT ;2( :	2LGGL--.E)//0A0ABKKNG"9^"K"#:;-. #G+11,2C2CDII**+;<Ihh&&'78G$!%$6/$(JJu$5E $EC<8%eU33(-W(= +0%eS1',{{}H'22:>8CTCT *D!)$(JJu$56:O  /3(,

5(9/$(JJu$5E ED$<07$ZU|-L+ $

5 1 ""#**51u:	2;2x >>%%d<%@@{ U, !*:6 /$'JE/ $6 3(+E
3 )3I'> !)$(!) %.z#: 3(+E
3
 !*:6 /$'JE/ &z2 + #E
+sq   I4I2I=J"J.9KK.I:9I:=JJJ+*J+.K
KK+*K+.L
Lc                  f    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	dZd
dZddZd
dZddZdddZ	y)_AsyncBatchProcessorzJInternal. Handles asynchronous batching and writing of events to BigQuery.c
                    || _         || _        || _        || _        || _        || _        t        j                  |      | _        d | _	        d| _
        || _        |	| _        y )NmaxsizeF)write_clientr   write_streamr   flush_intervalr   asyncior   _queue_worker_task	_shutdownbq_storage_typesservice_unavailable_exception
selfr  r   r  r   r  r   r   r  r  s
             r    __init__z_AsyncBatchProcessor.__init__  sg     )(($,(5<]]"6
 26 0-J*r   c                v   K   | j                   )t        j                  | j                               | _         y y wrj   )r	  r  create_task_batch_writerr  s    r    startz_AsyncBatchProcessor.start  s3     $ ' 3 3D4F4F4H ID %s   79c                   K   	 | j                   j                  |       y # t        j                  $ r t        j                  d       Y y w xY ww)N(BigQuery log queue full, dropping event.)r  
put_nowaitr  	QueueFullrx   ry   r  r   s     r    r0   z_AsyncBatchProcessor.append  sA     	GKK""3'   	GNNEF	Gs%   A  A(AA
AAc                  K   | j                   r| j                  j                         s_g }	 | j                   r+	 |j                  | j                  j	                                nP|j                  t        j                  | j                  j                         | j                         d {          | j                  j                          t        |      | j                  k  r]	 |j                  | j                  j	                                | j                  j                          t        |      | j                  k  r]|r| j                  |       d {    | j                   sB| j                  j                         s^y y # t
        j                  $ r Y y w xY w7 # t
        j                  $ r Y ww xY w7 d# t
        j                  $ r Y t
        j                  $ r t         j#                  d       Y y t$        $ rx}t'        |t(              rdt+        |      v rY d }~y t         j-                  d|d       	 t        j.                  d       d {  7   n# t(        $ r Y Y d }~y w xY wY d }~d }~ww xY ww)NtimeoutzBatch writer task cancelled.zEvent loop is closedError in batch writer: %sTexc_infor   )r
  r  emptyr0   
get_nowaitr  
QueueEmptywait_forrk   r  	task_doner*   r   _write_rows_with_retryTimeoutErrorCancelledErrorrx   info	Exceptionr(   RuntimeErrorr)   r   sleepr  batches      r    r  z"_AsyncBatchProcessor._batch_writer  s    ..(9(9(;E#>>T[[%;%;%=> LL%.. KKOO-t7J7J 
 %%'%j4??2T[[%;%;%=>--/ %j4??2 55e<<<1 ..(9(9(;(; #--  #--  ='' )) :; a.3ISQRV3S8!dK!--***#  +s  *JG  )F
 #AG  +F#,:G  'AF% *G  G  F>G  J,JJ
F G  JF  G  %F;8G  :F;;G   I?J'I?=J?I?I:$J)I:I!II! I:!	I1*I:+J0I11I:4J:I??Jc                "  K   d}| j                   j                  }	 t        || j                        }| j                  j	                         j                         }|j	                         j                         }| j                  j                  | j                  t              |j                  j                  _        |j                  j                  _        || j                   j$                  k  r*	 dfd}| j&                  j)                   |              d {   }	|	2 3 d {   }
t+        |
dd       }t+        |dd       }|s&|dk7  s,t+        |d	d
      }t         j-                  d||       |t.        t0        t2        fv r| j5                  |      d|j7                         v rt         j#                  d|        y t         j#                  d|       t+        |
dg       }|r|D ]  }t         j#                  d|        t         j#                  d|        y y y # t        $ r"}t         j#                  d|d       Y d }~y d }~ww xY w7 /7 '6 y # | j4                  $ r}|dz  }|| j                   j$                  kD  r3t         j#                  d| j                   j$                  dz   |       Y d }~y t9        |dt;        j:                         z   z  | j                   j<                        }t         j-                  d|||       t?        j@                  |       d {  7   || j                   jB                  z  }Y d }~n1d }~wt        $ r"}t         j#                  d|d       Y d }~y d }~ww xY w|| j                   j$                  k  rrw)Nr   r  r   -Failed to prepare Arrow batch (Data Loss): %sTr  c                   K     y wrj   r   )reqs   r    req_iterz=_AsyncBatchProcessor._write_rows_with_retry.<locals>.req_iter   s     Is   
r   codemessageUnknown error-BigQuery Write API returned error code %s: %szschema mismatchzBigQuery Schema Mismatch: %sz Non-retryable BigQuery error: %s
row_errorszRow error details: %szRow content causing error: %sr   8BigQuery Batch Dropped after %s attempts. Last error: %sBBigQuery write failed (Attempt %s), retrying in %.2fs... Error: %s8Unexpected BigQuery Write API error (Dropping batch): %s)r   zAsyncIterator[Any])"r   r   r   r   	serialize
to_pybytesr  AppendRowsRequestr  _DEFAULT_TRACE_ID
arrow_rowswriter_schemaserialized_schemar   serialized_record_batchr)  rx   r   r   r  append_rowsgetattrry   _GRPC_DEADLINE_EXCEEDED_GRPC_INTERNAL_GRPC_UNAVAILABLEr  lowerminrandomr   r  r+  r   )r  r   attemptdelayarrow_batchrC  serialized_batchr.  r4  	responsesresponser   
error_coder   r9  	row_error
sleep_timer3  s                    @r    r%  z+_AsyncBatchProcessor._write_rows_with_retry  sO    !!//	.tT5F5FGK $ 1 1 ; ; = H H J*446AAC''99!..9J : C >OCNN((::JCNN7 **666A #'"3"3"?"?
"KK	&/ # #(#Hgt<E!(!=J!jAo(/y/(RK&)
 &3*-* 
 #'"D"D]"SS0M4G4G4II &$BM!" # !'$F!" .5X|R-P
#-5? %Y	(.5Li(X%Y &-Ld S"K 7  	LL?T   		 L#i> 55 61T..:::LLR))559
  Q0143D3D3N3N
   mmJ///**555 N!  
 y **666s   NB=H N5)I II %I)I*I-I I A1I NAI N	I
(I NI

NI II NM/$AM0N5A(ML M<NM/M*%N*M//Nc                  K   d| _         t        j                  d       | j                  r+	 t	        j
                  | j                  |       d {    y y 7 # t        j                  $ r2 t        j                  d       | j                  j                          Y y t        $ r }t        j                  d|       Y d }~y d }~ww xY ww)NT/BatchProcessor shutting down, draining queue...r  z5BatchProcessor shutdown timed out, cancelling worker.(Error during BatchProcessor shutdown: %s)r
  rx   r(  r	  r  r#  r&  ry   cancelr)  r   r  r  r.  s      r    shutdownz_AsyncBatchProcessor.shutdownA  s     EFL&&t'8'8'JJJ J'' +VW!!((* LGKKLsM   )C$A AA CA ACCC%C ;C CCN)r  r   r   r   r  r)   r   r   r  r   r   r   r   r   r  r   r  r   r   Noner   zdict[str, Any]r   r]  r   list[dict[str, Any]]r   r]  g      @r  r   r   r]  
r   r   r   __doc__r  r  r0   r  r%  r[  r   r   r    r   r     s    TKK K 	K
 K K "K K K (+K4JG&PTl
Lr   r   c                  D    e Zd Z	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 ddZddZy)_GCSOffloaderc                n     ||      | _         | j                   j                  |      | _        || _        y Nproject)clientbucketexecutor)r  
project_idbucket_namerm  storage_client_clss        r    r  z_GCSOffloader.__init__O  s-     )<kk((5 r   c           	        K   t        j                         }|j                  | j                  t	        j
                  | j                  |||             d {   S 7 wrj   )r  get_running_looprun_in_executorrm  	functoolspartial_upload_sync)r  r   content_typepathloops        r    upload_contentz_GCSOffloader.upload_contentZ  sT      '')))MMd//|TJ
 
 	
 
s   AAAAc                    | j                   j                  |      }|j                  ||       d| j                   j                   d| S )N)rw  zgs:///)rl  blobupload_from_stringrw   )r  r   rw  rx  r}  s        r    rv  z_GCSOffloader._upload_syncc  sG    {{%<@t{{''($00r   N)rn  r)   ro  r)   rm  r   rp  r   )r   zbytes | strrw  r)   rx  r)   r   r)   )r   r   r   r  rz  rv  r   r   r    rf  rf  N  sW    	!	! 	! %		!
  	!

/2
:=
	
1r   rf  c                  j    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	dZd
dZddZd
dZddZdddZ	y)_BatchProcessorzHInternal. Synchronous version of `_AsyncBatchProcessor` using threading.c
                    || _         || _        || _        || _        || _        || _        || _        || _        |	| _        d| _	        t        | j                        | _        d | _        y )NFr  )r  r   r  r   r  r   r   r  r  r
  r   r  r	  r  s
             r    r  z_BatchProcessor.__init__q  sl     )(($,(, 0-J*-24;N;N-O59r   c                    | j                   At        j                  | j                  d      | _         | j                   j	                          yy)z$Starts the background worker thread.NT)targetdaemon)r	  	threadingThreadr  r  r  s    r    r  z_BatchProcessor.start  sA    $ ) 0 08J8JSW XD##% %r   c                    	 | j                   j                  |       y# t        $ r t        j	                  d       Y yw xY w)z'Adds a row to the queue for processing.r  N)r  r  r
   rx   ry   r  s     r    r0   z_BatchProcessor.append  s8    	GKK""3' 	GNNEF	Gs    ??c                   | j                   r| j                  j                         s<g }	 | j                   r+	 |j                  | j                  j	                                n5|j                  | j                  j                  | j                               | j                  j                          t        |      | j                  k  r]	 |j                  | j                  j	                                | j                  j                          t        |      | j                  k  r]|r| j                  |       | j                   s| j                  j                         s;yy# t
        $ r Y yw xY w# t
        $ r Y Yw xY w# t
        $ r Y t        $ r7}t        j                  d|d       t        j                   d       Y d}~d}~ww xY w)z;The background thread's main loop for batching and writing.r  r  Tr  r   N)r
  r  r   r0   r!  r	   rk   r  r$  r*   r   r%  r)  rx   r   timer+  r,  s      r    r  z_BatchProcessor._batch_writer  s[   ..(9(9(;E>>T[[%;%;%=> LL9L9L!MN%%'%j4??2T[[%;%;%=>--/ %j4??2 //6) ..(9(9(;(; !  ! 
   8!dK

1sl   F )E% !A(F 
AE4 F &F %	E1.F 0E11F 4	F =F ?F  F 	GG-G		Gc                   d}| j                   j                  }	 t        || j                        }| j                  j	                         j                         }|j	                         j                         }| j                  j                  | j                  t              }||j                  j                  _        ||j                  j                  _        || j                   j$                  k  r	 | j&                  j)                  t+        |g            }	|	D ]n  }
t-        |
dd      }t-        |dd      }|s |dk7  s&t-        |d	d
      }t         j/                  d||       |t0        t2        t4        fv r| j7                  |       y yyy# t        $ r"}t         j#                  d|d       Y d}~yd}~ww xY w# | j6                  $ r}|dz  }|| j                   j$                  kD  r3t         j#                  d| j                   j$                  dz   |       Y d}~yt9        |dt;        j:                         z   z  | j                   j<                        }t         j/                  d|||       t?        j@                  |       || j                   jB                  z  }Y d}~n1d}~wt        $ r"}t         j#                  d|d       Y d}~yd}~ww xY w|| j                   j$                  k  ra)z,Writes a batch to BigQuery with retry logic.r   r0  r1  Tr  Nr   r5  r6  r7  r8  r   r:  r;  r<  )"r   r   r   r   r=  r>  r  r?  r  r@  rA  rB  rC  r   rD  r)  rx   r   r   r  rE  iterrF  ry   rG  rH  rI  r  rK  rL  r   r  r+  r   )r  r   rM  rN  rO  rC  rP  r3  r.  rQ  rR  r   rS  r   rU  s                  r    r%  z&_BatchProcessor._write_rows_with_retry  s   !!//	.tT5F5FGK $ 1 1 ; ; = H H J*446AAC''99!..9J : C >OCNN((::JCNN7 **6661 --99$u+F	 ) #H#Hgt<E!(!=J!jAo(/y/(RK&)
 &3*-* 
 #'"D"D]"SS #%#& - 7  	LL?T   		: 55 61T..:::LLR))559
  Q0143D3D3N3N
   

:&**555 N!  
 Y **666sX   B=F 1AF: 8F: >A	F: F: 	F7F22F7:K	AJA>JK)KKc                H   d| _         t        j                  d       | j                  rN	 | j                  j	                  |       | j                  j                         rt        j                  d       yyy# t        $ r }t        j                  d|       Y d}~yd}~ww xY w)zShuts down the worker thread.TrW  r  z"BatchProcessor shutdown timed out.rX  N)	r
  rx   r(  r	  joinis_alivery   r)  r   rZ  s      r    r[  z_BatchProcessor.shutdown  s    EFL!!&&w&7$$--/NN#GH 0 
  LGKKLs   AA8 8	B!BB!N)r  r   r   r   r  r)   r   r   r  r   r   r   r   r   r  r   r  r   r   r]  r\  r^  r_  ra  rb  rc  r   r   r    r  r  n  s    R:: : 	:
 : : ": : : (+: 
:2&G:EN
Lr   r  c                  H    e Zd ZdZ	 	 d	 	 	 	 	 	 	 	 	 ddZddZ	 	 	 	 d	dZy)
_LangChainContentParserzGInternal. Parses LangChain content (including Multi-Modal) for logging.Nc                X    || _         || _        || _        || _        || _        d| _        y Ni   	offloaderr   r   
max_lengthr   inline_text_limitr  r  r   r   r  r   s         r    r  z _LangChainContentParser.__init__  s0     # $*!*r   c                    | j                   dk7  r,t        |      | j                   kD  r|d | j                    dz   dfS |dfS Nr&   r'   TFr  r*   r  r   s     r    	_truncatez!_LangChainContentParser._truncate  D    ??b SY%@)$//*-==tCCU{r   c                  K   g }g }d}t        |t              r|g}nt        |t              r|}nt        |      g}t        |      D ]  \  }}|dddddd}t        |t              rt	        |j                  d            }	| j                  r|	| j                  kD  rt        j                         j                          d| j                   d| j                   d	| d
z   }
	 | j                  j                  |d|
       d{   }d|d<   ||d<   d|i}| j                  r| j                  |d<   ||d<   |dd dz   |d<   n| j%                  |      \  }}|rd}||d<   |j'                  |       n\t        |t(              rK|j+                  d      }|dk(  r|j+                  dd      }t	        |j                  d            }	| j                  r|	| j                  kD  rt        j                         j                          d| j                   d| j                   d	| d
z   }
	 | j                  j                  |d|
       d{   }d|d<   ||d<   d|i}| j                  r| j                  |d<   ||d<   |dd dz   |d<   nG| j%                  |      \  }}|rd}||d<   |j'                  |       n|dk(  r|j+                  di       }t        |t(              r|j+                  d      n|}d|d<   |r3|j-                  d      r!| j                  r	 |j/                  dd      \  }}|j/                  d      d   j/                  d      d    }d dl}|j3                  |      }t5        j6                  |      xs d!}t        j                         j                          d| j                   d| j                   d	| | z   }
| j                  j                  |||
       d{   }d|d<   ||d<   d|i}| j                  r| j                  |d<   ||d<   ||d<   d"|d<   nd%|d<   n|r||d<   d&|d<   d'|d<   |j'                  d(       n~|d)k(  ryd*|d<   d+|j+                  d,       |d<   t9        j:                  |j+                  d-      |j+                  d,      d.      |d/<   |j'                  d0|j+                  d,       d1       |j'                  |        d2j=                  |      }|||fS 7 # t        $ rN}t         j#                  d|       | j%                  |      \  }}|rd}||d<   |j'                  |       Y d}~d}~ww xY w7 # t        $ rN}t         j#                  d|       | j%                  |      \  }}|rd}||d<   |j'                  |       Y d}~d}~ww xY w7 # t        $ r&}t         j#                  d#|       d$|d<   Y d}~d}~ww xY ww)3z;Parses LangChain Message Content (string or list of dicts).F
text/plainN{}INLINEr   r   r   r   r   r   r   r|  _p.txtGCS_REFERENCEr   r   
authorizer
object_ref   ... [OFFLOADED]r   !Failed to offload text to GCS: %sTr1   rp   	image_urlurl
image/jpegr   data:,r   :;r   .bin[MEDIA OFFLOADED])Failed to offload base64 image to GCS: %s[UPLOAD FAILED][BASE64 IMAGE]EXTERNAL_URI[IMAGE URL][IMAGE]tool_useapplication/jsonTool Call: rw   idtool_idrw   r   [TOOL: r    | )r(   r)   r.   	enumerater*   encoder  r  r   nowdater   r   rz  r   r)  rx   ry   r  r0   r+   rk   r   splitbase64	b64decode	mimetypesguess_extensionr   r   r  r  r   r   summary_textr   	raw_partsidxpart	part_datatext_lenrx  r   r  r.  cleanr9   	part_typetext_valimg_url_objr  headerencodedr   r  r   extfull_summarys                              r    parse_message_contentz-_LangChainContentParser.parse_message_content  s6      gs# 	I&IWI"9- E	,IC!)#' (I $$t{{734>>h1G1G&G&lln1134Admm_AF<<.3%t4D3$(NN$A$A ,%  5D	.1+.	%(&+S\
--7;7I7IJ|42<	,/,0#J9J,J	&) $(>>$#7LE5'+(-If% ''. D$' HHV,	&#xx3H"8??7#;<H~~(T5K5K*K"*,,."5"5"7!8$--J#||nBse48 7(,(E(E (,) #C 9HIn5/2Ie,*/J#11;?;M;M
< 86@Il308#AR0RIf- (,~~h'?u +/L,1	&)$++E2+-"&((;";K &k48 $.(  .:Ik*s~~g6>>F25))C2C,2LL,=a,@,F,Fs,KA,N	 -'-'7'7'@&/&?&?	&J&Tf*2,,.*=*=*?)@$--PQ'R'+||nBseC5$A(" -1NN,I,I$()T-" '" =L	. 936	% 0.3S\
#'#5#5?C?Q?QJ|$<:D	, 79B	+ 64G	& 1 1AIf-+.	%(4B	.1,9	&) ''	2*,-?Ik**5dhhv6F5G(HIf%37::$(HHTNDHHV<LM4I/0 !'''$((62B1C1(EF  +KE	,N zz,/]L88m % 3'JAN'+~~d';u +/L,1	&)$++E2234#  ) 7"NN+NPQR+/>>(+CLE5$/305If-(//667H'" $- F &$OQR!" 5F	& 1	Fs   C W(# S<S9=S<C&W(( UU	=UBW(CV6*V3+<V6'CW(9S<<	UAU	W(UW(U	V0"AV+&W(+V00W(3V66	W%?W W( W%%W(i N  N
r  z_GCSOffloader | Noner   r)   r   r)   r  r   r   r   r   r)   r   ztuple[str, bool]r   z"Union[str, List[Union[str, Dict]]]r   ztuple[str, list[dict], bool]r   r   r   rd  r  r  r  r   r   r    r  r    s_    Q  $(+'+ + 	+
 + "+
Y99Y9	%Y9r   r  c                  H    e Zd ZdZ	 	 d	 	 	 	 	 	 	 	 	 ddZddZ	 	 	 	 d	dZy)
_SyncLangChainContentParserzInternal. A purely synchronous parser that re-implements the parsing logic
    without using asyncio. It uses a synchronous GCS offloader.
    Nc                X    || _         || _        || _        || _        || _        d| _        y r  r  r  s         r    r  z$_SyncLangChainContentParser.__init__  s0     # $*!*r   c                    | j                   dk7  r,t        |      | j                   kD  r|d | j                    dz   dfS |dfS r  r  r  s     r    r  z%_SyncLangChainContentParser._truncate  r  r   c                   g }g }d}t        |t              r|g}nt        |t              r|}nt        |      g}t        |      D ]  \  }}|dddddd}t        |t              r	t	        |j                  d            }	| j                  r|	| j                  kD  rt        j                         j                          d| j                   d| j                   d	| d
z   }
	 | j                  j                  |d|
      }|j                  d||dd dz   d       d|i}| j                  r| j                  |d<   ||d<   n| j'                  |      \  }}|rd}||d<   |j)                  |       n`t        |t*              rO|j-                  d      }|dk(  r|j-                  dd      }t	        |j                  d            }	| j                  r|	| j                  kD  rt        j                         j                          d| j                   d| j                   d	| d
z   }
	 | j                  j                  |d|
      }|j                  d||dd dz   d       d|i}| j                  r| j                  |d<   ||d<   nM| j'                  |      \  }}|rd}||d<   |j)                  |       n|dk(  r|j-                  di       }t        |t*              r|j-                  d      n|}d|d<   |r-|j/                  d      r| j                  r	 |j1                  dd      \  }}|j1                  d      d   j1                  d      d    }d dl}|j5                  |      }t7        j8                  |      xs d!}t        j                         j                          d| j                   d| j                   d	| | z   }
| j                  j                  |||
      }|j                  d||d"d#       d|i}| j                  r| j                  |d<   ||d<   nd&|d<   n|r|j                  |d'd(d)       |j)                  d*       n|d+k(  r|j                  d,d-|j-                  d.       t;        j<                  |j-                  d/      |j-                  d.      d0      d1       |j)                  d2|j-                  d.       d3       |j)                  |        d4j?                  |      }|||fS # t         $ rN}t"        j%                  d|       | j'                  |      \  }}|rd}||d<   |j)                  |       Y d}~|d}~ww xY w# t         $ rN}t"        j%                  d|       | j'                  |      \  }}|rd}||d<   |j)                  |       Y d}~d}~ww xY w# t         $ r&}t"        j%                  d$|       d%|d<   Y d}~d}~ww xY w)5z/Synchronously parses LangChain Message Content.Fr  Nr  r  r  r   r|  r  r  r  r  r  )r   r   r   r   r  r  r  Tr   r1   rp   r  r  r  r   r  r  r   r  r  r   r  r  )r   r   r   r   r  r  r  r  r  )r   r   r   r  r  r  r  rw   r  r  )r   r   r   r  r   r  ) r(   r)   r.   r  r*   r  r  r  r   r  r  r   r   rv  updater   r)  rx   ry   r  r0   r+   rk   r   r  r  r  r  r  r   r   r  r  s                              r    r  z1_SyncLangChainContentParser.parse_message_content  s     gs# 	I&IWI"9- R	,IC!)#' (I $$t{{734>>h1G1G&G&lln1134Admm_AF<<.3%t4D3"nn99$dS!((0?'*(,Tc
5F(F ',S\
--7;7I7IJ|42<	,/ $(>>$#7LE5'+(-If% ''.D$' HHV,	&#xx3H"8??7#;<H~~(T5K5K*K"*,,."5"5"7!8$--J#||nBse48 7"&.."="= (,#C &,,4C+.,4TcN=N,N!" +0J#11;?;M;M
< 86@Il3 (,~~h'?u +/L,1	&)$++E2+-"&((;";K &k48 $.( 
 .:Ik*s~~g6>>F25))C2C,2LL,=a,@,F,Fs,KA,N	 -'-'7'7'@&/&?&?	&J&Tf*2,,.*=*=*?)@$--PQ'R'+||nBseC5$A(" '+nn&A&A$	SW&X ) 0 08G/25>0C	%&!" /4S\
#'#5#5?C?Q?QJ|$<:D	, 7 1AIf-!(('*0>(5 !''	2*,$$);&1$((62B1C$D/3zz/3xx~,0HHV,<!"0	 !'''$((62B1C1(EF  +eR	,h zz,/]L88q % 3'JAN'+~~d';u +/L,1	&)$++E223L  ) 7"NN+NPQR+/>>(+CLE5$/305If-(//667^ $- F &$OQR!" 5F	& 1	FsM   !AS9$AUDV-9	UAUU	V*AV%%V*-	W6WWr  r  r  r  r  r   r   r    r  r    sa      $(+'+ + 	+
 + "+ 
e99e9	%e9r   r  c                      e Zd ZdZ	 	 d	 	 	 	 	 	 	 	 	 d fdZddZd dZ	 	 	 	 	 	 d!	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"dZddZd#dZ	d$d	Z
ddd
	 	 	 	 	 	 	 	 	 	 	 	 	 d%dZddd
	 	 	 	 	 	 	 	 	 	 	 	 	 d&dZdd	 	 	 	 	 	 	 	 	 d'dZdd	 	 	 	 	 	 	 	 	 d(dZdd	 	 	 	 	 	 	 	 	 	 	 d)dZdd	 	 	 	 	 	 	 	 	 d*dZdd	 	 	 	 	 	 	 	 	 	 	 d+dZdd	 	 	 	 	 	 	 	 	 d,dZdd	 	 	 	 	 	 	 	 	 d(dZdd	 	 	 	 	 	 	 	 	 	 	 d-dZdd	 	 	 	 	 	 	 	 	 d.dZdd	 	 	 	 	 	 	 	 	 d(dZdd	 	 	 	 	 	 	 	 	 d/dZdd	 	 	 	 	 	 	 	 	 d0dZdd	 	 	 	 	 	 	 	 	 d1dZdd	 	 	 	 	 	 	 	 	 d(dZddZ xZS )2AsyncBigQueryCallbackHandlerACallback handler for logging LangChain events to Google BigQuery.Nc                   t         |           t               \  | _        | _        | _        | _        | _        | _        | _	        | _
        | _        | _        | _        || _        || _        |xs
 t!               | _        |r|| j"                  _        d| _        d| _        t+        j,                         | _        d | _        d | _        d | _        t7        d      | _        d | _        d | _        y NFr   )max_workers)superr  r!   bigquerygoogle_authgapic_client_infoasync_clientcloud_exceptionssync_write_client_modulestorage	bq_schemar  api_core_exceptionsr   rn  
dataset_idr   configr   _started_is_shutting_downr  Lock_setup_lockrk  r  async_batch_processorr   	_executorr  _arrow_schemar  rn  r  r   r  	__class__s        r    r  z%AsyncBigQueryCallbackHandler.__init__  s     	 )*	
M"!)LN!$G %$6 4 6#+DKK #',)0!%BF"-?A-N/3"&r   c                    K    j                   ry  j                  4 d {     j                   r	 d d d       d {    y t        j                         }|j	                   j
                   fd       d {    _         j                   d j                   d j                  j                   t         j                        |j	                   j
                   fd       d {    |j	                   j
                   fd       d {   \  }} j                  j                  t              } j                   j#                  ||       _        d j                   d j                   d	 j                  j                   d
}t'         j(                   j*                        } j                  j,                  rOt/         j                   j                  j,                   j
                   j0                  j2                         _        t7         j$                  || j                  j8                   j                  j:                   j                  j<                   j                  j>                   j@                   jB                  jD                  	       _#         jF                  jI                          d {    d _         d d d       d {    y 7 7 7 p7 7 7 +7 # 1 d {  7  sw Y   y xY ww)Nc                 P     j                   j                   j                        S rh  )r  Clientrn  r  s   r    r   z>AsyncBigQueryCallbackHandler._ensure_started.<locals>.<lambda>  s    (<(<T__(<(U r   .c                 (    j                         S rj   )_ensure_table_exists)full_table_idr   r  s   r    r   z>AsyncBigQueryCallbackHandler._ensure_started.<locals>.<lambda>  s    (A(A-QW(X r   c                 >     j                   j                  dg      S )N.https://www.googleapis.com/auth/cloud-platformscopes)r  defaultr  s   r    r   z>AsyncBigQueryCallbackHandler._ensure_started.<locals>.<lambda>  s#    ((00LM 1  r   
user_agentcredentialsclient_info	projects/
/datasets//tables/	/_defaultT)%r  r  r  rr  rs  r  rk  rn  r  r  r   r   r  r  
ClientInfor@  r  BigQueryWriteAsyncClientr  r   r  r   r   rf  r  r  r  r   r   r   r   r   r  r  ServiceUnavailabler  r  )	r  ry  creds_r  r  r   r  r   s	   `      @@r    _ensure_startedz,AsyncBigQueryCallbackHandler._ensure_started  s    ==## 9	! 9	!}}9	! 9	! 9	! ++-D $ 4 4 U! DK
 ??#1T__$5Qt{{7K7K6LM  1?F&& X   "11 HE1 00;;, < K !% 1 1 J J!{ !K !D DOO,Jt6G H++../y: 
 <L {{**!.OOKK//NNLL''	" *>!!&&00((**%%((;;
*D& ,,22444 DMs9	! 9	! 9	! 9	!
N 5q9	! 9	! 9	! 9	!s   LK&LK9L K)L6K9<K,=A4K91K/2&K9K2F0K9	K5
K9L K7!L)L,K9/K92K95K97L9L?L LLc                   | j                   t        d      	 | j                   j                  |       y # | j                  j                  $ r | j
                  j                  ||      }| j
                  j                  | j
                  j                  j                  d      |_
        | j                  j                  |_        | j                   j                  |       Y y w xY wNz#BigQuery client is not initialized.r   rB   )type_r   rk  r   	get_tabler  NotFoundr  TableTimePartitioningTimePartitioningTypeDAYtime_partitioningr  r   create_tabler  r   r   tbls       r    r  z1AsyncBigQueryCallbackHandler._ensure_table_exists      ;;BCC	*KK!!(+$$-- 	*--%%hv%>C$(MM$B$Bmm88<<K %C %C! %)KK$A$AC!KK$$S)	*   5 B,C$#C$c	                h  K   | j                   j                  sy | j                          d {    |xs i }|j                  d      }	|j                  d      }
|j                  d      }t	        |      }t	        |      }t        | j                  ||| j                   j                  | j                   j                        }d}g }d}d }	 t        |t              rbd|v r^g }|d   D ]Q  }|j                  d      }|j                  |       d {   \  }}}|rd	}|j                  |       |r|d
z  }||z  }S |}nt        |t              rMd|v rI|d   D ]@  }|j                  |       d {   \  }}}|rd	}|j                  |       |r|d
z  }||z  }B nZt        |t              r|j                  |       d {   \  }}}n,t        t	        |      | j                   j                        \  }}i dt#        j$                  t&        j(                        d|d|d|	dt	        |      d|
d|d|d|rt	        |      nd dd|id| j                   j*                  r|ng d|d|rd|ind d|s|rdndd|xs |d|}| j,                  t/        d      | j,                  j1                  |       d {    y 7 7 7 [7 # t        $ rU}d| }t        j!                  d||       t        t	        |      | j                   j                        \  }}g }Y d }~3d }~ww xY w7 rwNr   r   r   )r   rp   Fmessagesr   Tr  promptszFailed to parse content: z%s for run_id %srB   r   r   r   r   r   summaryr   r   r   total_msr   ERROROKr   r   #Batch processor is not initialized.)r  r   r  rk   r)   r  r  r   r   r(   r+   r  extendr-   r)  rx   ry   r   r  r   utcr   r  r   r0   r  r   run_idr   parent_run_idr   r   latencyrs   r   r   r   r   r   parserr  r   r   parsing_error	all_partsmsgmsg_contentsptp_strr.  r   s                               r    _logz!AsyncBigQueryCallbackHandler._log  s     {{""""$$$>r\\,/
,,y)W%v;f+(NNKK**++33
 -	'4(Z7-B	":. &C"%'')"4K$*$@$@$MMGAq!'+$$Q'#$- A%L& !*GT*yG/C$Y/ &E$*$@$@$GGGAq!'+!((+#$- A%L& GS)
 !66w??	 ! 
 .GL$++"@"@.*l
hll3
*
 U
 *	

 S[
 w
 
 w
 Mc-0t
 	<0
 {{22 +
 *
 7:w/
  -gT!
" U3m#
$ L%
( %%-BCC((//444y 	%< N H @  	7s;MNN-}fE)BGdkk<<*&L, M	< 	5s   +L2KBL2AK KAK .K	/AK ?K 5K 5CL2=L0>L2K 	K K 	L-A
L("L2(L--L2c                  K   | j                   ry 	 d| _         | j                  r7| j                  j                  | j                  j                         d {    | j
                  j                  d       d| _         y 7 (# d| _         w xY wwNT)waitF)r  r  r[  r  r   r  r  s    r    r[  z%AsyncBigQueryCallbackHandler.shutdownb  sz     !!	+%)D"))0099$++:V:VWWWNN###.%*D" X &+D"s/   BAB A? B 7B?B 	B

Bc                B   K   | j                          d {    | S 7 wrj   )r  r  s    r    
__aenter__z'AsyncBigQueryCallbackHandler.__aenter__m  s"     ""$$$ 	%s   c                @   K   | j                          d {    y 7 wrj   r[  )r  argss     r    	__aexit__z&AsyncBigQueryCallbackHandler.__aexit__q       mmo   r9  tagsc          
        K   |j                  di       j                  d      xs |j                  d      }| j                  d|d|i|||d|j                  d             d {    y 7 w	Nkwargsmodelrw   LLM_REQUESTr/  rQ  rU  rs   r   r9  r   rs   rk   rD  r  
serializedr/  r8  r9  rQ  rT  
model_names           r    on_llm_startz)AsyncBigQueryCallbackHandler.on_llm_startv  sr       ^^Hb155g>X*..QWBX
ii(' $z:ZZ
+  
 	
 	
s   A#A-%A+&A-c          
     6  K   |j                  di       j                  d      xs |j                  d      }|D 	cg c]  }|D ]  }	|	j                           }
}}	| j                  d|d|
i|||d|j                  d             d {    y c c}	}w 7 w	NrT  rU  rw   rV  r.  rW  rs   rX  rk   
model_dumprD  r  r[  r.  r8  r9  rQ  rT  r\  subm	flat_msgss              r    on_chat_model_startz0AsyncBigQueryCallbackHandler.on_chat_model_start  s       ^^Hb155g>X*..QWBX
/7EEAQ\\^E^E	Eii+' $z:ZZ
+  
 	
 	
 F	
s   :BB1BBBr9  c          
     6  K   |j                   r,|j                   d   r|j                   d   d   j                  }nd}|j                  r|j                  j                  d      nd }| j	                  d|||d|i|j                  d             d {    y 7 wNr   rp   token_usageLLM_RESPONSEusagers   rX  generationsr   
llm_outputrk   rD  r  rR  r8  r9  rT  	resp_textrl  s          r    
on_llm_endz'AsyncBigQueryCallbackHandler.on_llm_end  s      H$8$8$; ,,Q/277II:B:M:M##''6SWii''ZZ
+  
 	
 	
s   BBBBc          	     |   K   | j                  d|t        |      ||j                  d             d {    y 7 wN	LLM_ERRORrs   r   r9  rs   rD  r)   rk   r  r   r8  r9  rT  s        r    on_llm_errorz)AsyncBigQueryCallbackHandler.on_llm_error  s?      iie*'ZZ
+  
 	
 	
   2<:<c          	        K   | j                  d|t        j                  |t              ||j	                  d             d {    y 7 wNCHAIN_STARTr  rs   r   r9  rs   rD  r   r   r)   rk   r  r[  inputsr8  r9  rT  s         r    on_chain_startz+AsyncBigQueryCallbackHandler.on_chain_start  sF      iiJJvs3'ZZ
+  
 	
 	
   AAA
Ac          	        K   | j                  d|t        j                  |t              ||j	                  d             d {    y 7 wN	CHAIN_ENDr~  rs   r  r  r  outputsr8  r9  rT  s        r    on_chain_endz)AsyncBigQueryCallbackHandler.on_chain_end  sF      iiJJw4'ZZ
+  
 	
 	
r  c          	     j   K   | j                  d||||j                  d             d {    y 7 wNTOOL_STARTINGrs   r  rD  rk   r  r[  	input_strr8  r9  rT  s         r    on_tool_startz*AsyncBigQueryCallbackHandler.on_tool_start  s;      ii'ZZ
+  
 	
 	
   )313c          	     j   K   | j                  d||||j                  d             d {    y 7 wNTOOL_COMPLETEDrs   r  r  r  outputr8  r9  rT  s        r    on_tool_endz(AsyncBigQueryCallbackHandler.on_tool_end  s;      ii'ZZ
+  
 	
 	
r  c          	     |   K   | j                  d|t        |      ||j                  d             d {    y 7 wN
TOOL_ERRORrs   rv  rw  rx  s        r    on_tool_errorz*AsyncBigQueryCallbackHandler.on_tool_error  s?      iie*'ZZ
+  
 	
 	
rz  c          	     j   K   | j                  d||||j                  d             d {    y 7 wNRETRIEVER_STARTrs   r  r  r  r[  queryr8  r9  rT  s         r    on_retriever_startz/AsyncBigQueryCallbackHandler.on_retriever_start  s;      ii'ZZ
+  
 	
 	
r  c          	        K   |D cg c]  }|j                          }}| j                  d|t        j                  |t              ||j                  d             d {    y c c}w 7 
wNRETRIEVER_ENDr~  rs   r  ra  rD  r   r   r)   rk   r  	documentsr8  r9  rT  docdocss          r    on_retriever_endz-AsyncBigQueryCallbackHandler.on_retriever_end)  sj      -66S 66iiJJtS1'ZZ
+  
 	
 	
 7	
s   A.A'AA.!A,"A.c          	     |   K   | j                  d|t        |      ||j                  d             d {    y 7 wNRETRIEVER_ERRORrs   rv  rw  rx  s        r    on_retriever_errorz/AsyncBigQueryCallbackHandler.on_retriever_error:  s?      iie*'ZZ
+  
 	
 	
rz  c          	     j   K   | j                  d||||j                  d             d {    y 7 wNTEXTrs   r  r  r  r   r8  r9  rT  s        r    on_textz$AsyncBigQueryCallbackHandler.on_textJ  s;      ii'ZZ
+  
 	
 	
r  c          
        K   | j                  d|t        j                  |j                  t	        |j
                        dt              ||j                  d             d {    y 7 wNAGENT_ACTION)toolinputr~  rs   r  rD  r   r   r  r)   
tool_inputrk   r  actionr8  r9  rT  s        r    on_agent_actionz,AsyncBigQueryCallbackHandler.on_agent_actionZ  s`      iiJJs63D3D/EFPS (ZZ
+  
 	
 	
s   A"A,$A*%A,c          	        K   | j                  d|t        j                  d|j                  it              ||j                  d             d {    y 7 wNAGENT_FINISHr  r~  rs   r  rD  r   r   return_valuesr)   rk   r  finishr8  r9  rT  s        r    on_agent_finishz,AsyncBigQueryCallbackHandler.on_agent_finishl  sQ      iiJJ&*>*>?M'ZZ
+  
 	
 	
s   AAAAc          	     |   K   | j                  d|t        |      ||j                  d             d {    y 7 wNCHAIN_ERRORrs   rv  rw  rx  s        r    on_chain_errorz+AsyncBigQueryCallbackHandler.on_chain_error|  s?      iie*'ZZ
+  
 	
 	
rz  c                @   K   | j                          d {    y 7 wrj   rK  r  s    r    closez"AsyncBigQueryCallbackHandler.close  rN  rO  NN
rn  r)   r  r)   r   r   r  zBigQueryLoggerConfig | Noner   r]  r\  r   r)   r   z	List[Any]r   r   NNNNNNr   r)   r8  	uuid.UUIDr   r   r9  uuid.UUID | Noner   dict | Noner   r   r:  z
int | Noners   r  r   r]  )r   z'AsyncBigQueryCallbackHandler')rL  r   r   r]  r[  Dict[str, Any]r/  z	List[str]r8  r  r9  r  rQ  List[str] | NonerT  r   r   r]  r[  r  r.  zList[List[BaseMessage]]r8  r  r9  r  rQ  r  rT  r   r   r]  
rR  r   r8  r  r9  r  rT  r   r   r]  
r   BaseExceptionr8  r  r9  r  rT  r   r   r]  r[  r  r  r  r8  r  r9  r  rT  r   r   r]  )
r  r  r8  r  r9  r  rT  r   r   r]  r[  r  r  r)   r8  r  r9  r  rT  r   r   r]  )
r  r)   r8  r  r9  r  rT  r   r   r]  r[  r  r  r)   r8  r  r9  r  rT  r   r   r]  
r  r   r8  r  r9  r  rT  r   r   r]  
r   r)   r8  r  r9  r  rT  r   r   r]  
r  r   r8  r  r9  r  rT  r   r   r]  
r  r   r8  r  r9  r  rT  r   r   r]  )r   r   r   rd  r  r  r  rD  r[  rI  rM  r]  rf  rr  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  __classcell__r   s   @r    r  r    s   K  $.2%'%' %' 	%'
 ,%' 
%'N<!|*" *."& " $i5i5 i5 	i5
 (i5  i5 i5 i5 i5 
i5V	+ +/!%
"
 

 
 (
 
 
 

4 +/!%
"
 *

 
 (
 
 
 

6 +/

 	

 (
 
 

6 +/

 	

 (
 
 

, +/
"
 

 
 (
 
 

, +/

 	

 (
 
 

, +/
"
 

 
 (
 
 

, +/

 	

 (
 
 

* +/

 	

 (
 
 

, +/
"
 

 
 (
 
 

, +/

 	

 (
 
 

, +/

 	

 (
 
 

* +/

 	

 (
 
 

* +/

 	

 (
 
 

. +/

 	

 (
 
 

* +/

 	

 (
 
 

 r   r  c                      e Zd ZdZ	 	 d	 	 	 	 	 	 	 	 	 d fdZddZddZ	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dZddZddd	 	 	 	 	 	 	 	 	 	 	 	 	 d!d	Z	ddd	 	 	 	 	 	 	 	 	 	 	 	 	 d"d
Z
dd	 	 	 	 	 	 	 	 	 d#dZdd	 	 	 	 	 	 	 	 	 	 	 d$dZdd	 	 	 	 	 	 	 	 	 d%dZdd	 	 	 	 	 	 	 	 	 d&dZdd	 	 	 	 	 	 	 	 	 	 	 d'dZdd	 	 	 	 	 	 	 	 	 d(dZdd	 	 	 	 	 	 	 	 	 d&dZdd	 	 	 	 	 	 	 	 	 d)dZdd	 	 	 	 	 	 	 	 	 d*dZdd	 	 	 	 	 	 	 	 	 d+dZdd	 	 	 	 	 	 	 	 	 	 	 d,dZdd	 	 	 	 	 	 	 	 	 d-dZdd	 	 	 	 	 	 	 	 	 d&dZdd	 	 	 	 	 	 	 	 	 d&dZddZ xZS ).BigQueryCallbackHandlerr  Nc                   t         |           t               \  | _        | _        | _        | _        | _        | _        | _	        | _
        | _        | _        | _        || _        || _        |xs
 t!               | _        |r|| j"                  _        d| _        d| _        t+        j,                         | _        d | _        d | _        d | _        t7        d      | _        d | _        d | _        y r  )r  r  r!   r  r  r  r  r  r  r  r  r  r  r   rn  r  r   r  r   r  r  r  r  r  rk  r  batch_processorr   r  r  r  r  s        r    r  z BigQueryCallbackHandler.__init__  s     	 )*	
M"!)LN!$G %$6 4 6#+DKK #',+4>>+;!%7;-?A-N/3"&r   c                \   | j                   ry | j                  5  | j                   r
	 d d d        y | j                  j                  | j                        | _        | j                   d| j                   d| j                  j                   }t        | j                        }| j                  ||       | j                  j                  dg      \  }}| j                  j                  t              }| j                   j#                  ||      | _        d| j                   d| j                   d	| j                  j                   d
}t'        || j(                  | j*                        }| j                  j,                  rOt/        | j                  | j                  j,                  | j0                  | j2                  j                        | _        t7        | j$                  ||| j                  j8                  | j                  j:                  | j                  j<                  | j                  j>                  | j@                  | jB                  jD                  	      | _#        | jF                  tI        d      | jF                  jK                          d| _         d d d        y # 1 sw Y   y xY w)Nri  r  r	  r
  r  r  r  r  r  r  r4  T)&r  r  r  r  rn  rk  r  r  r   r   r  r  r  r  r  r@  r  BigQueryWriteClientr  r   r  r   r   rf  r  r  r  r  r   r   r   r   r  r  r  r  r   r  )r  r  r   r  r  r  r  r   s           r    r  z'BigQueryCallbackHandler._ensure_started  sO   == 4	!}}4	! 4	! --..t.GDK ??#1T__$5Qt{{7K7K6LM  1?F%%mV<''//HI 0 HE1 00;;, < K !% = = Q Q!{ !R !D DOO,Jt6G H++../y: 
 <L {{**!.OOKK//NNLL''	" $3!!&&00((**%%((;;
$D  ##+ !FGG  &&( DMi4	! 4	! 4	!s   J"I(J""J+c                   | j                   t        d      	 | j                   j                  |       y # | j                  j                  $ r | j
                  j                  ||      }| j
                  j                  | j
                  j                  j                  d      |_
        | j                  j                  |_        | j                   j                  |       Y y w xY wr  r  r(  s       r    r  z,BigQueryCallbackHandler._ensure_table_exists  r*  r+  c	                   | j                   j                  sy | j                          |xs i }|j                  d      }	|j                  d      }
|j                  d      }t	        |      }t	        |      }t        | j                  ||| j                   j                  | j                   j                        }d}g }d}d }	 t        |t              rZd|v rVg }|d   D ]I  }|j                  d      }|j                  |      \  }}}|rd	}|j                  |       |r|d
z  }||z  }K |}nt        |t              rEd|v rA|d   D ]8  }|j                  |      \  }}}|rd	}|j                  |       |r|d
z  }||z  }: nRt        |t              r|j                  |      \  }}}n,t        t	        |      | j                   j                        \  }}i dt#        j$                  t&        j(                        d|d|d|	dt	        |      d|
d|d|d|rt	        |      nd dd|id| j                   j*                  r|ng d|d|rd|ind d|s|rdndd|xs |d|}| j,                  t/        d      | j,                  j1                  |       y # t        $ rU}d| }t        j!                  d||       t        t	        |      | j                   j                        \  }}g }Y d }~d }~ww xY wr-  )r  r   r  rk   r)   r  r  r   r   r(   r+   r  r5  r-   r)  rx   ry   r   r  r   r6  r   r  r   r0   r7  s                               r    rD  zBigQueryCallbackHandler._log  sy    {{"">r\\,/
,,y)W%v;f+,NNKK**++33
 +	'4(Z7-B	":. &C"%'')"4K$::;GGAq!'+$$Q'#$- A%L& !*GT*yG/C$Y/ &E$::5AGAq!'+!((+#$- A%L& GS)
 009	 !  .GL$++"@"@.*l
hll3
*
 U
 *	

 S[
 w
 
 w
 Mc-0t
 	<0
 {{22 +
 *
 7:w/
  -gT!
" U3m#
$ L%
( 'BCC##C(=  	7s;MNN-}fE)BGdkk<<*&L, M	s   DJ 	K7"A
K22K7c                   | j                   ry 	 d| _         | j                  r/| j                  j                  | j                  j                         | j
                  j                  d       d| _         y # d| _         w xY wrF  )r  r  r[  r  r   r  r  s    r    r[  z BigQueryCallbackHandler.shutdownn  sh    !!	+%)D"##$$--dkk.J.JKNN###.%*D"UD"s   AA5 5	A>rP  c          
         |j                  di       j                  d      xs |j                  d      }| j                  d|d|i|||d|j                  d             y rS  rY  rZ  s           r    r]  z$BigQueryCallbackHandler.on_llm_starty  sd      ^^Hb155g>X*..QWBX
		(' $z:ZZ
+ 	 	
r   c          
        |j                  di       j                  d      xs |j                  d      }|D 	cg c]  }|D ]  }	|	j                           }
}}	| j                  d|d|
i|||d|j                  d             y c c}	}w r_  r`  rb  s              r    rf  z+BigQueryCallbackHandler.on_chat_model_start  s      ^^Hb155g>X*..QWBX
/7EEAQ\\^E^E	E		+' $z:ZZ
+ 	 	
 Fs   Brg  c          
        |j                   r,|j                   d   r|j                   d   d   j                  }nd}|j                  r|j                  j                  d      nd }| j	                  d|||d|i|j                  d             y ri  rm  rp  s          r    rr  z"BigQueryCallbackHandler.on_llm_end  s     H$8$8$; ,,Q/277II:B:M:M##''6SW		''ZZ
+ 	 	
r   c          	         | j                  d|t        j                  |t              ||j	                  d             y r|  r  r  s         r    r  z&BigQueryCallbackHandler.on_chain_start  s8     			JJvs3'ZZ
+ 	 	
r   c          	         | j                  d|t        j                  |t              ||j	                  d             y r  r  r  s        r    r  z$BigQueryCallbackHandler.on_chain_end  s8     			JJw4'ZZ
+ 	 	
r   c          	     `    | j                  d|t        |      ||j                  d             y r  rw  rx  s        r    r  z&BigQueryCallbackHandler.on_chain_error  s1     			e*'ZZ
+ 	 	
r   c          	     N    | j                  d||||j                  d             y r  r  r  s         r    r  z%BigQueryCallbackHandler.on_tool_start  s-     			'ZZ
+ 	 	
r   c          	     `    | j                  d|t        |      ||j                  d             y r  rw  r  s        r    r  z#BigQueryCallbackHandler.on_tool_end  s1     			K'ZZ
+ 	 	
r   c          	     `    | j                  d|t        |      ||j                  d             y r  rw  rx  s        r    r  z%BigQueryCallbackHandler.on_tool_error
  s1     			e*'ZZ
+ 	 	
r   c          	     N    | j                  d||||j                  d             y r  r  r  s        r    r  zBigQueryCallbackHandler.on_text  s-     			'ZZ
+ 	 	
r   c          
         | j                  d|t        j                  |j                  t	        |j
                        dt              ||j                  d             y r  r  r  s        r    r  z'BigQueryCallbackHandler.on_agent_action*  sR     			JJs63D3D/EFPS (ZZ
+ 	 	
r   c          	         | j                  d|t        j                  d|j                  it              ||j                  d             y r  r  r  s        r    r  z'BigQueryCallbackHandler.on_agent_finish<  sC     			JJ&*>*>?M'ZZ
+ 	 	
r   c          	     N    | j                  d||||j                  d             y r  r  r  s         r    r  z*BigQueryCallbackHandler.on_retriever_startL  s-     			'ZZ
+ 	 	
r   c          	         |D cg c]  }|j                          }}| j                  d|t        j                  |t              ||j                  d             y c c}w r  r  r  s          r    r  z(BigQueryCallbackHandler.on_retriever_end]  sW     -66S 66		JJtS1'ZZ
+ 	 	
 7s   Ac          	     `    | j                  d|t        |      ||j                  d             y r  rw  rx  s        r    r  z*BigQueryCallbackHandler.on_retriever_errorn  s1     			e*'ZZ
+ 	 	
r   c          	     `    | j                  d|t        |      ||j                  d             y rt  rw  rx  s        r    ry  z$BigQueryCallbackHandler.on_llm_error~  s1     			e*'ZZ
+ 	 	
r   c                $    | j                          y rj   rK  r  s    r    r  zBigQueryCallbackHandler.close  s    r   r  r  r\  r  r  r  r  r  r  r  )
r  zUnion[Dict[str, Any], Any]r8  r  r9  r  rT  r   r   r]  r  r  )
r  r   r8  r  r9  r  rT  r   r   r]  r  r  r  r  r  )r   r   r   rd  r  r  r  rD  r[  r]  rf  rr  r  r  r  r  r  r  r  r  r  r  r  r  ry  r  r  r  s   @r    r  r    s   K  $.2%'%' %' 	%'
 ,%' 
%'N7!r*" *."& " $g)g) g) 	g)
 (g)  g) g) g) g) 
g)R	+" +/!%
"
 

 
 (
 
 
 

4 +/!%
"
 *

 
 (
 
 
 

4 +/

 	

 (
 
 

8 +/
"
 

 
 (
 
 

, +/
+
 	

 (
 
 

* +/

 	

 (
 
 

, +/
"
 

 
 (
 
 

, +/

 	

 (
 
 

* +/

 	

 (
 
 

* +/

 	

 (
 
 

* +/

 	

 (
 
 

. +/

 	

 (
 
 

, +/
"
 

 
 (
 
 

, +/

 	

 (
 
 

, +/

 	

 (
 
 

* +/

 	

 (
 
 

 r   r  r   )r2   r   r3   r   r   ztuple[Any, bool])r   	list[Any]r   r   rD   r   r   r   )r   r   r   r  )r   r`  r   r   r   r   )@
__future__r   r  rt  r   loggingr  rL  r  r  uuidconcurrent.futuresr   dataclassesr   r   r   r   queuer	   r
   r   r   r   typingr   r   r   r   r   r   langchain_core.agentsr   r   langchain_core.callbacksr   r   langchain_core.messagesr   langchain_core.outputsr   langchain_core.utilsr   r!   	getLoggerr   rx   rG  rH  rI  r@  r-   r   r   r   r   r   r   rf  r  r  r  r  r  r   r   r    <module>r     sM   "          1 ( ' $ $ " B B : N / , -* 
		8	$  2 6p*p*.1p*>Ap*p*fjd    % % %&AARoL oLd1 1@WL WLtp9 p9f9 9NK#7 Kfz1 zr   