
    riu                        d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z: d dl;m<Z< d dl=m>Z? d dl=m@Z@mAZA d dlBmCZC d dlDmEZEmFZFmGZGmHZHmIZImJZJ d dlKmLZL d dlMmNZNmOZO d dlPmQZQ ddZRd ZSe
j                  j                  ddd de	j                  g      d        ZWe
j                  j                  dd gdfd dgdfg d dfg      d!        ZXd" ZYd# ZZe
j                  j                  d$      d%        Z\e
j                  j                  d$      d&        Z]e
j                  j                  d$      d'        Z^d( Z_e
j                  j                  d)g d* e	j                  g d+g d,g d-g d.g      fg d/g d0fg      d1        Zae
j                  j                  d)g d2 e	j                  g d.g d.g d3g d3g d4g      fg d5g d6fg      d7        Zbd8 Zcd9 Zdd: Zed; Zfe
j                  j                  d<eN      e
j                  j                  d=eO      d>               Zgd? Zhe
j                  j                  d@g dA      dB        ZidC ZjdD Zke
j                  j                  dE e	j                  g dF       e	j                  g dF      dGdHf e	j                  g dF       e	j                  g dI      dGdJf e	j                  g dF       e	j                  g dK      dGdLf e	j                  g dI       e	j                  g dF      dGdMfg      dN        Zle
j                  j                  dO e	j                  g dP       e	j                  g dQ      dGdRfg      dS        ZmdT Zne
j                  j                  dUddg      dV        ZodW Zpe
j                  j                  dXdYdZid[dZid\dZd]d^d_d]dZdZd]d^d`d]g      da        Zqe
j                  j                  dbd^d^d]d^fe	j                  dZd]e	j                  fd`dZd]d`fe	j                  e	j                  d]e	j                  fe	j                  e	j                  fg      dc        Zse
j                  j                  dbd^d^d]d^fe	j                  dZd]dZfe	j                  ddd]ddfe	j                  e	j                  d]e	j                  fe	j                  e	j                  fg      de        Ztdf Zudg Zve
j                  j                  dd de	j                  g      e
j                  j                  dhd gd gfg      e
j                  j                  die% ee&dj      e/e0g      dk                      Zwe
j                  j                  dhd gd gfg      e
j                  j                  die% ee&dj      e/e0g      dl               Zxdm Zydn Zzdo Z{dp Z|e
j                  j                  dqdrdsg      dt        Z}du Z~e
j                  j                  dvg dw      dx        Zdy Zdz Zd{ Ze
j                  j                  d|g d}fd~dgdfgddg      d        Zd Ze
j                  j                  dg d      d        Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze
j                  j                  d$      d        Zd Zd Zd Zd Zd Zd Zd Ze
j                  j                  dddg      d        Ze
j                  j                  d$      d        Ze
j                  j                  d$      d        Ze
j                  j                  d$      e
j                  j                  dddde	j                  e	j                  fg      d               Ze
j                  j                  ddg      e
j                  j                  dvg d      e
j                  j                  dd de	j                  g      d                      Ze
j                  j                  dvg d      d        Ze
j                  j                  dd de	j                  g      d        Zd Zd Ze
j                  j                  dd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Zd Zd Zd Zd Ze
j                  j                  d ejP                  dgd gdgd gg      df ejP                  d gdgdgdgg      df ejP                  g dg dg dg      dfg      d        Zd Zd Zd Zd Zd Zd Zd ZdÄ Ze
j                  j                  de	jd                  e	jf                  e	jh                  g      dĄ        Ze
j                  j                  de	jd                  e	jf                  e	jh                  g      dń        Ze
j                  j                  dhg dg dfg ddd gd dgdd ggfg d g dƢg dg dǢgfg      dȄ        ZdɄ Zdʄ Zd˄ Zd̄ Zd̈́ Zd΄ Zdτ Ze
j                  j                  dg dѢg dҢfg dӢg dҢfg dҢg dӢfg      dԄ        Ze
j                  j                  die)e% ee&ddj      e.e/e0e g      e
j                  j                  dg d֢      dׄ               Ze
j                  j                  d e	j                  d dg       e	j                  dd g      dZf e	j                  d dg       e	j                  d dg      d^f e	j                  d dg       e	j                  d d g      dZf e	j                  d d g       e	j                  d d g      d^fg      dل        Ze
j                  j                  e
j                  j                  d e+e%e	j                  ۫       e+e&de	j                  ܫ       e+e/e	j                  ۫       e+e0e	j                  ۫      g      d݄               Zdބ Zd߄ Zd Zd Zd Zd Ze
j                  j                  dg dg dddfg dg dddfg dg dddfg dg dddfg dg dddfg dddddgddddgddddggddfg dg dgg dg dgddfg dg dg dg dgd dgdfg dg dg dg dgd gdfg	      d        Zd Ze
j                  j                  d eA             d         Ze
j                  j                  de e*e4e5g      e
j                  j                  dddg      e
j                  j                  dddg      e
j                  j                  d eA             d                             Ze
j                  j                  de e*e4e5g      e
j                  j                  dddg      e
j                  j                  d eA             d                      Zy(      N)partial)chainpermutationsproduct)linalgsparse)hamming)	bernoulli)datasetssvm)config_context)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_brier_scored2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifierdevice)get_namespace)yield_namespace_device_dtype_combinations)MockDataFrame)_array_api_for_testsassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    | t        j                         } | j                  }| j                  }|r||dk     ||dk     }}|j                  \  }}t        j                  |      }t        d      }|j                  |       ||   ||   }}t        |dz        }t
        j                  j                  d      }t
        j                  ||j                  |d|z        f   }t        j                  ddd      }	|	j!                  |d| |d|       j#                  ||d       }
|r	|
ddd	f   }
|	j%                  ||d       }||d }|||
fS )
zMake some classification predictions on a toy dataset using an SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger:   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                g/var/www/auto_recruiter/arenv/lib/python3.12/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrb   G   sN    $$&AAQx1q51GGIz
		)A
R
 CKKNQ41qAy1}D ))


"C
a9cJ&6778A ''t!
DC771Ud8QuX.<<QtuXFL $AqD)[[45"FtuXF6<''    c            
         t        j                         } t        | d      \  }}}ddddddd	d
ddddddddddddddddddd}t        ||t	        j
                  t        | j                              | j                  d      }|j                         |j                         k(  sJ |D ]u  }|dk(  r#t        ||   t              sJ ||   ||   k(  r)J ||   j                         ||   j                         k(  sJ ||   D ]  }t        ||   |   ||   |           w t        |d   d   t              sJ t        |d   d   t              sJ t        |d   d   t              sJ t        |d   d   t              sJ y ) NFrT   rU   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   )ri   rg   rh   rj   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyzweighted avgT)labelstarget_namesoutput_dictru   rq   rg   rt   rj   )r   rD   rb   r   rH   rI   lenrw   keys
isinstancefloatr3   rK   )irisr`   r_   _expected_reportreportkeymetrics           ra   ,test_classification_report_dictionary_outputr   w   s   D'UCFFA
 -)*	
 -*+	
 -)+	
 ++'	
 '++(	
5 OD #yyT../0&&F ;;=O002222 W*fSk5111#;/#"6666#;##%)=)B)B)DDDD)#. W#OC$8$@&+fBUVWW oh/<eDDDok2;?GGGoh/	:C@@@ok29=sCCCrc   zero_divisionwarnrC   c                 .   g dg d}}t        j                  d      5 }t        j                  dd       t        ||| d       | d	k(  r3t	        |      d
kD  sJ |D ]  }d}|t        |j                        v rJ  n|rJ d d d        y # 1 sw Y   y xY w)Nabc)r   r   dTrecordalwaysz.+Use `zero_division`)message)r   rx   r   rC   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsfilterwarningsr   ry   strr   )r   r`   r_   r   itemmsgs         ra   0test_classification_report_zero_division_warningr      s    $oFF		 	 	-  	2IJF-T	
 F"v;?"? 0Oc$,,////0 :  s   AB:BBzlabels, show_micro_avgTr   rC   r<   c                 h    ddgddg}}t        ||| d      }|rd|v sJ d|vsJ yd|v sJ d|vsJ y)a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   rC   T)rv   rx   z	micro avgru   Nr   )rv   show_micro_avgr`   r_   r   s        ra   1test_classification_report_labels_subset_supersetr      s`     VaVFF"66&dSFf$$$'''V###&(((rc   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |t        j                  |            dk(  sJ t        | t        j                  |             dk(  sJ t        | t        j                  | j
                              dk(  sJ t        |t        j                  | j
                              dk(  sJ y )Nr   rC   rC   rC   r   rC   r   r   rC         ?rC   r   )rH   arrayr   logical_notzerosrG   y1y2s     ra   .test_multilabel_accuracy_score_subset_accuracyr      s    	9i(	)B	9i(	)B"b!S((("b!Q&&&"b!Q&&&"bnnR01Q666"bnnR01Q666"bhhrxx01Q666"bhhrxx01Q666rc   c            	      @   t        d      \  } }}t        | |d       \  }}}}t        |ddgd       t        |ddgd       t        |d	d
gd       t        |ddg       i ddifD ]  }t	        j
                         5  t	        j                  d       t        | |fi |}t        |dd       t        | |fi |}	t        |	dd       t        | |fi |}
t        |
d
d       t        t        | |fddi|d|z  |	z  d|z  |	z   z  d       d d d         y # 1 sw Y   xY w)NTrU   averageg\(\?g333333?r<   g)\(?g(\?皙?gRQ?   r   rU   errorbeta      )rb   r!   r4   r5   r   r   simplefilterr"   r#   r   r3   r   )r`   r_   r~   rZ   rfskwargspsrsfss              ra   %test_precision_recall_f1_score_binaryr      sG   't4FFA 1NJAq!Qa$q1a$q1a$q1q2r(#
 	8,- $$& 	!!'* :6:B%b$2ff77B%b$2&&3F3B%b$2FF==f=R"$r	B7	 		 	s   <BDD	z1ignore::sklearn.exceptions.UndefinedMetricWarningc                     dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddgd      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddgt	        d            k(  sJ t        ddgddgt	        d            t        j                  t        ddgddgd            k(  sJ y )	N      ?rC   r   r           infg     j@)r"   r#   r   r   r|   pytestapprox rc   ra   +test_precision_recall_f_binary_single_classr     s;   
 /1a&1a&1111,1v1v....(Aq6Aq6****+q!fq!f15555/2r(RH5555,Bx"b2222(B8b"X....+r2hRuU|DDDDBx"be=RHr2hS1B   rc   c                  >   g d} g d}t        | t        j                  d            }t        |t        j                  d            }| |f||fg}t        |      D ]  \  }\  } }t	        | |g dd       }t        g d|       t	        | |g dd      }t        t        j                  g d      |       d	D ]5  }|d
k(  r|dk(  rt        t	        | |g d|      t	        | |d |             7  dD ]  }t        j                  t              5  t	        ||t        j                  d      |       d d d        t        j                  t              5  t	        ||t        j                  dd      |       d d d         t        j                  g dg dg      } t        j                  g dg dg      }t        | |d
ddg      \  }}	}
}t        t        j                  ||	|
g      t        j                  g d             y # 1 sw Y   xY w# 1 sw Y   (xY w)N)rC      r   r<   )rC   rC   r   r<   r   classes)r   rC   r<   r   r   rv   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   rC   r   r   rC   rC   rC   r   rC   r   rv   )      ?rC   竪?)r*   rH   rI   	enumerater#   r4   meanr3   r   raises
ValueErrorr   r!   )r`   r_   
y_true_bin
y_pred_binrE   iactualr   rZ   r   r   r~   s               ra   $test_precision_recall_f_extra_labelsr   &  s    FF		!=J		!=JVz:67D( FFff_dS!";VD ff_gV!"''*C"DfM 8 	G)#QVVOWUVVD'J	( 7 ]]:& 	WZ		!gV	W]]:& 	JryyQ/?	 	 XXy),-FXXy),-F0	1a&JAq!Q !Q+RXX6G-HI	W 	W	 	s   #H$HH	H	c                     g d} g d}t        | t        j                  d            }t        |t        j                  d            }| |f||fg}t        |      D ]  \  }\  } }t	        t
        | |ddg      }t	        t
        | |d       }t        dd	g |d 
             t        d |d
             t        d |d
             t        d |d
             dD ]  } ||
       ||
      k7  rJ   y )N)rC   rC   r<   r   )rC   r   r   r   r   r   rC   r   rv   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r*   rH   rI   r   r   r#   r4   r3   )	r`   r_   r   r   rE   r   	recall_13
recall_allr   s	            ra   &test_precision_recall_f_ignored_labelsr   T  s     FF		!=J		!=JVz:67D( MFFL&&!QH	\66$G
!3*i.EFOYw-GH3Yz5RSGYw%?@ 6 	MGW-G1LLLL	MMrc   c            	          t        j                  g dg dg dg dg dg dg      } t        j                  g dg dg dg d	g d
g dg      }d}t        j                  t        |      5  t        | |d       ddd       y# 1 sw Y   yxY w)z:Test multiclass-multiouptut for `average_precision_score`.)r<   r<   rC   rC   r<   r   r   rC   r<   rC   r<   r   rC   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr<   	pos_labelN)rH   r   r   r   r   r   )r`   y_scoreerr_msgs      ra   -test_average_precision_score_non_binary_classr   k  s    XX	
	F hh	
	G ?G	z	1 >1=> > >s   ,BBzy_true, y_scorer   r   rC   r<   r   r   r   r   )r   r   r   r   rC   rC   rC   rC   rC   rC   rC   )r   r   r   r   r   333333?r   rm   rm   rC   rC   c                 &    t        | |      dk(  sJ y)a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    rC   Nr   r`   r   s     ra   -test_average_precision_score_duplicate_valuesr     s    8 #673q888rc   )r<   r<   rC   rC   r   )r   r   r   )r   r   r   r   )r   r   r   c                 &    t        | |      dk7  sJ y )Nr   r   r   s     ra   (test_average_precision_score_tied_valuesr     s    : #673s:::rc   c                      d} t        j                  t        |       5  t        g dg ddd       d d d        y # 1 sw Y   y xY w)NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   rC   r<   r<   r<   r   r   r   )r   warnsUserWarningr!   r   s    ra   (test_precision_recall_f_unused_pos_labelr    sB    
	  
k	- 
'yAw	

 
 
s	   ;Ac            	          t        d      \  } }}d } || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}       y c c}w c c}w )NTr   c                 <   t        | |      }t        |ddgddgg       |j                         \  }}}}||z  ||z  z
  }t        j                  ||z   ||z   z  ||z   z  ||z   z        }|dk(  rdn||z  }	t        | |      }
t        |
|	d       t        |
dd       y )	N   r         r   r<   decimal=
ףp=?)r   r5   flattenrH   sqrtr   r4   )r`   r_   cmtpfpfntnnumdentrue_mccmccs              ra   testz*test_confusion_matrix_binary.<locals>.test  s    ff-2Q!R12BB2gRggrBw27+rBw727CDq1cCi/!#x;!#tQ7rc   rb   r   r`   r_   r~   r  rW   s        ra   test_confusion_matrix_binaryr    sR    't4FFA8 	&	!Q#a&	!F#;qCF#;<	!#;
   AA
c            	          t        d      \  } }}d } || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}       y c c}w c c}w )NTr   c                 P    t        | |      }t        |ddgddggddgddggg       y )Nr
  r	  r   r  r    r5   )r`   r_   r  s      ra   r  z5test_multilabel_confusion_matrix_binary.<locals>.test  s7    (82"a1b' 2b!Wq"g4FGHrc   r  r  s        ra   'test_multilabel_confusion_matrix_binaryr!    sS    't4FFAI 	&	!Q#a&	!F#;qCF#;<	!#;r  c            	          t        d      \  } }}dd} || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}d       y c c}w c c}w )NFr   c           	      P   t        | |      }t        |ddgddggddgddggd	d
gddggg       |rg dng d}t        | ||      }t        |ddgddggd	d
gddggddgddggg       |rg dng d}t        | ||      }t        |ddgddggd	d
gddggddgddggddgddggg       y )N/   r   r      &   r      r      r   r<      )021r   r<   rC   r   )r*  r+  r,  3)r   r<   rC   r   rp   r   r   )r`   r_   string_typer  rv   s        ra   r  z9test_multilabel_confusion_matrix_multiclass.<locals>.test  s   (82q'Ar7#r1gAw%72r(QG9LM	

 %0Y(G2q'Ar7#r2hB%8B7RG:LM	

 *5%,(Ga1b'"bAr7#a2q'"a1a&!		
rc   T)r/  )Fr  r  s        ra   +test_multilabel_confusion_matrix_multiclassr0    sT    'u5FFA
6 	&	!Q#a&	!F#;qCF#;N	!#;s
   AA
csc_containercsr_containerc                    t        j                  g dg dg dg      }t        j                  g dg dg dg      } ||      } ||      } | |      } | |      }t        j                  g d      }dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]!  }|D ]  }t        ||      }t        ||	        # t        ||d      }t        |dd	gddggddgd	dggd	dgd
d	ggg       t        ||d
d	g      }t        |d	d
gdd	ggdd	gddggg       t        ||d
d	gd      }t        |d	d	gddggddgd	d	ggd	dgdd	ggg       t        |||d      }t        |d
d	gd
d
ggddgd	dggd	dgdd	ggg       y )Nr   r   rC   r   rC   rC   r   r   r   r   )r<   rC   r   rC   r   r<   T
samplewiser   )rv   r7  )sample_weightr7  r   r   )rH   r   r    r5   )r1  r2  r`   r_   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr8  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  ra   +test_multilabel_confusion_matrix_multilabelrB    s   
 XXy)Y78FXXy)Y78Fv&Jv&Jv&Jv&J HHY'MAA1a&1a&!1QFQF3CDGZ,EZ,E ,
 	,J,ZDBr7+	,, 
%VV	EBraVaV,1v1v.>!Q!Q@PQR 
%VVQF	CBraVaV,1v1v.>?@ 
%VVQFt	TBraVaV,1v1v.>!Q!Q@PQR 
%m
B raVaV,1v1v.>!Q!Q@PQRrc   c            	         t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        j                  t        d      5  t        | |d	d
g       d d d        t        j                  t        d      5  t        | |g dg dg dg       d d d        d}t        j                  t        |      5  t        | |dg       d d d        d}t        j                  t        |      5  t        | |dg       d d d        t        j                  t        d      5  t        g dg dd       d d d        d}t        j                  t        |      5  t        g dg dgg dg dg       d d d        y # 1 sw Y   $xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w# 1 sw Y   y xY w)Nr   r4  r5  r   r   r   inconsistent numbers of samplesr   rC   r<   r8  z)Sample weights must be 1D array or scalarrC   r<   r   )r<   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr6  z'multiclass-multioutput is not supported)r<   rC   r   rC   r   r<   )rH   r   r   r   r   r    )r`   r_   r   s      ra   'test_multilabel_confusion_matrix_errorsrH  C  s   XXy)Y78FXXy)Y78F 
z)J	K J#FF1a&IJ	z)T	U 
#F9i*K	

 7G	z	1 A#FFB4@A6G	z	1 @#FFA3?@ 
z)=	> K#IyTJK 8G	z	1 T#Y	$:Y	<RST T+J J
 
A A@ @K K
T TsH   FF	F$>F01F<)GFF!$F-0F9<GGz%normalize, cm_dtype, expected_results))truer   TUU?)predr   rJ  )allr   geq?)Nr   r<   c                     g ddz  }t        t        t        g d             }t        |||       }t	        ||       |j
                  j                  |k(  sJ y )Nr   r   	normalize)listr   r   r   r2   dtypekind)rO  cm_dtypeexpected_resultsy_testr_   r  s         ra   test_confusion_matrix_normalizerV  a  sP     ]F%i012F	&&I	>BB()88==H$$$rc   c                     g d} g d}t        | |d      }|j                         t        j                  d      k(  sJ t	        j
                         5  t	        j                  dt               t        | |d      }d d d        j                         t        j                  d      k(  sJ t	        j
                         5  t	        j                  dt               t        || d       d d d        y # 1 sw Y   wxY w# 1 sw Y   y xY w)	N)r   r   r   r   rC   rC   rC   rC   )r   r   r   r   r   r   r   r   rI  rN         @r   rK  r   )r   sumr   r   r   r   r   RuntimeWarning)rU  r_   cm_truecm_preds       ra   ,test_confusion_matrix_normalize_single_classr]  r  s    %F%Fvv@G;;=FMM#.... 
	 	 	" Eg~6"66VDE ;;=FMM#....		 	 	" ;g~66:; ;E E; ;s   )C2 )C>2C;>Dc                      g d} g d}t        j                  t        d      5  t        ||        ddd       y# 1 sw Y   yxY w)z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r  r  r   )rU  r_   s     ra   "test_confusion_matrix_single_labelr`    s:    FF	k)F	G )() ) )s	   :Azparams, warn_msg)rC   rC   rC   r   r   r   r`   r_   z?`positive_likelihood_ratio` is ill-defined and set to `np.nan`.)r   r   r   r   r   r   zdNo samples were predicted for the positive class and `positive_likelihood_ratio` is set to `np.nan`.r   r   r   rC   rC   rC   z?`negative_likelihood_ratio` is ill-defined and set to `np.nan`.z9No samples of the positive class are present in `y_true`.c                 z    t        j                  t        |      5  t        di |  d d d        y # 1 sw Y   y xY wNr   r   )r   r  r  r   )paramswarn_msgs     ra   test_likelihood_ratios_warningsrg    s3    X 
k	2 *)&)* * *   1:zparams, err_msg)r   rC   r   rC   r   rC   rC   r   r   r<   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                 z    t        j                  t        |      5  t        di |  d d d        y # 1 sw Y   y xY wrd  )r   r   r   r   )re  r   s     ra   test_likelihood_ratios_errorsrk    s2    $ 
z	1 *)&)* * *rh  c                     t        j                  dgdz  dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        | |      \  }}t        |d       t        |d	       t        | |       \  }}t	        |t         j
                  dz         t        |t        j                  d      d
       t        j                  dgdz  dgdz  z         }t        | ||      \  }}t        |d       t        |d       y )NrC   r   r   r
  r<   
   r	  g?g_B{	%?g-q=)rtolr      r   r   rE  gUUUUUU@gqq?)rH   r   r   r2   r5   nanr   )r`   r_   posnegr8  s        ra   test_likelihood_ratiosrs    s     XXqcAgb()FXXqcAgb(A3723F&vv6HCC!C! 'vv6HCsBFFQJ'C!51
 HHcURZ3%!)34M&vv]SHCC C!rc   raise_warningc                     t        j                  ddg      }t        j                  ddg      }d}t        j                  t        |      5  t        |||        ddd       y# 1 sw Y   yxY w)zaTest that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
    param is set.rC   r   zI`raise_warning` was deprecated in version 1.7 and will be removed in 1.9.r   )rt  N)rH   r   r   r  FutureWarningr   )rt  r`   r_   r   s       ra   0test_likelihood_ratios_raise_warning_deprecationrw    s`     XXq!fFXXq!fF
UC	m3	/ MmLM M M   A$$A-c                  h   t        j                  g d      } t        j                  g d      }t        | |d      \  }}|t        j                  d      k(  sJ t        j                  g d      } t        j                  g d      }t        | |d      \  }}|t        j                  d      k(  sJ y)zTest that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
    LR- when `replace_undefined_by=1` is set.r5  r   rC   replace_undefined_byr   r   N)rH   r   r   r   r   )r`   r_   positive_likelihood_ratior~   negative_likelihood_ratios        ra   1test_likelihood_ratios_replace_undefined_by_worstr~    s    
 XXi FXXi F#:Q$ q %c(:::: XXi FXXi F#:Q$ A  %c(::::rc   r{  LR+r   LR-g      )r  r  r   rp  rX  c                     t        j                  ddg      }t        j                  ddg      }d}t        j                  t        |      5  t        |||        ddd       y# 1 sw Y   yxY w)zTest that class_likelihood_ratios raises a `ValueError` if the input dict for
    `replace_undefined_by` is in the wrong format or contains impossible values.rC   r   zGThe dictionary passed as `replace_undefined_by` needs to be in the formr   rz  N)rH   r   r   r   r   r   )r{  r`   r_   r   s       ra   6test_likelihood_ratios_wrong_dict_replace_undefined_byr    s`     XXq!fFXXq!fF
SC	z	- 
F1E	

 
 
rx  zreplace_undefined_by, expectedc                    t        j                  g d      }t        j                  g d      }t        |||       \  }}t        j                  |      rt        j                  |      sJ y|t	        j
                  |      k(  sJ y)zTest that the `replace_undefined_by` param returns the right value for the
    positive_likelihood_ratio as defined by the user.r5  r   rz  NrH   r   r   isnanr   r   )r{  expectedr`   r_   r|  r~   s         ra   0test_likelihood_ratios_replace_undefined_by_0_fpr  *  sp     XXi FXXi F#:-A$ q 
xxxx1222(FMM(,CCCCrc   r   c                    t        j                  g d      }t        j                  g d      }t        |||       \  }}t        j                  |      rt        j                  |      sJ y|t	        j
                  |      k(  sJ y)zTest that the `replace_undefined_by` param returns the right value for the
    negative_likelihood_ratio as defined by the user.r   r   rz  Nr  )r{  r  r`   r_   r~   r}  s         ra   0test_likelihood_ratios_replace_undefined_by_0_tnr  F  sp     XXi FXXi F#:-A$ A  
xxxx1222(FMM(,CCCCrc   c                     t        j                  dgdz  dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z   dgdz  z         }t        | |      }t        |dd	
       |t        ||       k(  sJ t        j                  | dgdz        } t        j                  |dgdz        }t        | |ddg      |k(  sJ t        t        | |       d       t        j                  dgdz  dgdz  z   dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        t        | |      dd
       t        j                  dgdz  dgdz  z   dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        t        | |      dd
       t        t        | |d      dd
       t        t        | |d      dd
       y )Nr   (   rC   <   ro   rm  2   gʡE?r   r  r<   r   r   r   .   ,   4          g??g+?r?   weightsg_vO?	quadraticg#?)rH   r   r   r3   append)r   r   kappas      ra   test_cohen_kappar  b  s    
1#(aS2X%	&B	1#(aS2X%b0A38;	<Bb"%Eua0%b"---- 
2sQw	B	2sQw	BRQF3u<<<)"b137 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"b161E 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"b161E)"b(CVUVW"b+6rc   c                      ddg} t        j                  dgdz  dgdz  z         }t        j                  dgdz        }t        j                  t        d      5  t        ||| 	       d
d
d
       y
# 1 sw Y   y
xY w)zLTest that correct error is raised when users pass labels that are not in y1.rC   r<   r   r   r   rm  z6At least one label in `labels` must be present in `y1`r   r   N)rH   r   r   r   r   r   )rv   r   r   s      ra   (test_cohen_kappa_score_error_wrong_labelr    ss    VF	3%!)seai'	(B	3%"*	B	R
 1 	"b01 1 1s   A11A:zy_true, y_predr   r   c                     t        j                         5  t        j                  d        | |||      }ddd       t        j                  |      rt        j                        sJ y|k(  sJ y# 1 sw Y   >xY w)zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r   r   N)r   r   r   rH   r  )r   r`   r_   r   results        ra   !test_zero_division_nan_no_warningr    ss     
	 	 	" Eg&mDE 
xxxx&&&E Es   !A33A<c                     t        j                  t              5   | ||d      }ddd       dk(  sJ y# 1 sw Y   xY w)ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    r   r  Nr   )r   r  r   )r   r`   r_   r  s       ra   test_zero_division_nan_warningr    s@     
,	- >f=>S==> >s   6?c                     t         j                  j                  |       }|j                  ddd      }|j                  ddd      }t	        t        ||      t        j                  ||      d   d       y )Nr   r<   ro   sizer   rC   rm  )rH   rL   rM   randintr3   r   corrcoef)global_random_seedr[   r`   r_   s       ra   -test_matthews_corrcoef_against_numpy_corrcoefr    se    
))

 2
3C[[AB['F[[AB['F&&)2;;vv+Ft+Lbrc   c                    t         j                  j                  |       }|j                  ddd      }|j                  ddd      }|j	                  d      }t        |||      }t        |      }t        t        |      D 	cg c]A  }t        |      D ]1  }t        |      D ]!  }	|||f   |||	f   z  ||	|f   |||f   z  z
  # 3 C c}	}}      }
t        t        |      D cg c]c  }|d d |f   j                         t        j                  t        |      D cg c]  }t        |      D ]  }||k7  s	|||f    ! c}}      z  e c}}}      }t        j                  t        |      D cg c]c  }||d d f   j                         t        j                  t        |      D cg c]  }t        |      D ]  }||k7  s	|||f    ! c}}      z  e c}}}      }|
t        j                  ||z        z  }t        |||      }t        ||d       y c c}	}}w c c}}w c c}}}w c c}}w c c}}}w )Nr   r<   ro   r  rE  rm  )rH   rL   rM   r  randr   ry   rY  ranger  r   r3   )r  r[   r`   r_   r8  CNkmlcov_ytypr   gcov_ytytcov_ypyp
mcc_jurmanmcc_ourss                    ra   %test_matthews_corrcoef_against_jurmanr    s<    ))

 2
3C[[AB['F[[AB['FHHRLM}EAAA 1X	
 	
1X	
 1X		
  adGa1g!Q$!AqD' 11	
1	
1	
H  1X	
 	
  adGKKMffuQxL!qLAQ!Va1gLgLMN	
H vv 1X	
 	
  adGKKMffuQxL!qLAQ!Va1gLgLMN	
H BGGHx$788J }MH*b11	
 M	
 M	
sC   AH )8H-!H':H'H-88H:0H4	H4H:'H-4H:c           	         t         j                  j                  |       }|j                  ddd      D cg c]  }|dk(  rdnd }}t	        t        ||      d       |D cg c]  }|dk(  rdnd }}t	        t        ||      d       t        |ddg	      }t        j                  |dd      }t	        t        ||      d       t	        t        g d
g d
      d       t	        t        |dgt        |      z        d       g d}g d}t	        t        ||      d       dgdz  dgdz  z   }t        j                  t              5  t	        t        |||      d       d d d        y c c}w c c}w # 1 sw Y   y xY w)Nr   r<   ro   r  r   r   r   r   r   r_  r   )rC   r   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rC   rC   rC   rC   r   rC   rC   rC   )rC   rC   rC   r   r   rC   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rm  rE  )rH   rL   rM   r  r3   r   r*   wherery   r   r   AssertionError)	r  r[   r   r`   
y_true_invy_true_inv2y_1y_2masks	            ra   test_matthews_corrcoefr    st   
))

 2
3C.1kk!QRk.HIQ!Vc$IFI )&&93? 5;;qc#s*;J;)&*=rB #s<K((;S1K)&+>C ),EsK )&3%#f+2EFL GC
FC)#s3S9 38qcBhD 
~	& R-c3dKSQR R; J <.R Rs   E)"E.E33E<c                    t         j                  j                  |       }t        d      }d}|j	                  d|d      D cg c]  }t        ||z          }}t        t        ||      d       g d}g d}t        t        ||      d	       g d}g d
}t        t        ||      dt        j                  d      z         g d}g d}t        t        ||      d       g d}g d}t        t        ||      d       g d}	g d}
t        t        |	|
      d       g d}g d}g d}t        t        |||      d       g d}g d}g d}t        t        |||      d       y c c}w )Nr   r   r   ro   r  r   )r   r   rC   rC   r<   r<   )r<   r<   r   r   rC   rC   g      )rC   rC   r   r   r   r   ii  r   )r   r   r   r   	r   rC   r<   r   rC   r<   r   rC   r<   )	rC   rC   rC   r<   r<   r<   r   r   r   )r   r   rC   rC   r<   ri  rC   rC   rC   rC   r   rE  r   r   rC   rC   r   r   )	rH   rL   rM   ordr  chrr3   r   r  )r  r[   ord_a	n_classesr   r`   
y_pred_bad
y_pred_minr_   r  r  r8  s               ra   !test_matthews_corrcoef_multiclassr    s\   
))

 2
3CHEI&)kk!YRk&HIc%!)nIFI )&&93?  F#J)&*=tD  F#J)&*=sRWWWEU?UV FF)&&93? FF)&&93? &C
%C)#s3S9 FF#M&&F FF M&&F_ Js   E%n_pointsd   i'  c                    t         j                  j                  |      d }fd}t        j                  ddg|       }t	        t        ||      d       t        j                  g d|       }t	        t        ||      d        ||       \  }}t	        t        ||      d       t	        t        ||       |||             y )Nc                     t        | |      }|d   }|d   }|d   }t        |       }||z   |z  }||z   |z  }||z  ||z  z
  }	||z  d|z
  z  d|z
  z  }
|	t        j                  |
      z  S )NrC   rC   )rC   r   r  rC   )r   ry   rH   r  )r`   r_   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              ra   mcc_safez1test_matthews_corrcoef_overflow.<locals>.mcc_safeI  s    &vv6t$%	%	v;y(H4y(H4 8+h.AA"X-X>!h,Orww777rc   c                 v    j                  |       }|dj                  |       dz
  z  z   }|dkD  }|dkD  }||fS )Nr   r   )random_sample)r  x_truex_predr`   r_   r[   s        ra   	random_ysz2test_matthews_corrcoef_overflow.<locals>.random_ysU  sN    ""8,#!2!28!<s!BCC##v~rc   r   r   )r   r   rX  )rH   rL   rM   repeatr3   r   )r  r  r  r  arrr`   r_   r[   s          @ra   test_matthews_corrcoef_overflowr  D  s     ))

 2
3C
8 ))S#J
)C)#s3S9
))OX
.C)#s3S9x(NFF)&&93?)&&98FF;STrc   c                  ,   t        d      \  } }}t        | |d       \  }}}}t        |g dd       t        |g dd       t        |g dd       t        |g d       t	        | |d	d
      }t        |dd       t        | |d
      }t        |dd       t        | |d
      }	t        |	dd       t	        | |d      }t        |dd       t        | |d      }t        |dd       t        | |d      }	t        |	dd       t	        | |d      }t        |dd       t        | |d      }t        |dd       t        | |d      }	t        |	dd       t        j                  t              5  t	        | |d       d d d        t        j                  t              5  t        | |d       d d d        t        j                  t              5  t        | |d       d d d        t        j                  t              5  t        | |dd       d d d        t        | |g dd       \  }}}}t        |g dd       t        |g dd       t        |g dd       t        |g d       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)NFr   r   )(\?Q?gzG?r<   )HzG?g
ףp=
?rm   )Q?333333?r  )rf   rl   ro   rC   r   r  g(\?r   r   gRQ?r   gGz?r   r   r   r   r-  r   )r  g=
ףp=?r  )r  rm   r   )r  r  r  )rf   ro   rl   )rb   r!   r4   r5   r"   r#   r   r   r   r   r   )
r`   r_   r~   rZ   r   r   r   r   r   r   s
             ra   )test_precision_recall_f1_score_multiclassr  f  sT   'u5FFA 1NJAq!Qa!3Q7a!3Q7a!3Q7q,' 
1g	FBb$*	ffg	6Bb$*	&&'	2Bb$*		9Bb$*	ffg	6Bb$*	&&'	2Bb$*		<Bb$*	ffj	9Bb$*	&&*	5Bb$*	z	" ;	:;	z	" 8VVY78	z	" 434	z	" AFFIC@A 1y$JAq!Q a!3Q7a!3Q7a!3Q7q,'!; ;8 84 4A As0   ,I&I2I><J
&I/2I;>J
Jr   )r   r   r   r   Nc                     t        j                  g dg      }t        j                  g dg      }t        ||g dg |       \  }}}}t        |d       t        |d       t        |d       | t        |g d       y y )Nr  r   r   rC   rC   )r   r   rC   r<   )rv   warn_forr   r   r   rC   rC   r   )rH   r   r!   r5   )r   r`   r_   rZ   r   r   r   s          ra   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    su     XX|n%FXX|n%F0|b'JAq!Q q!q!q!1l+ rc   c                  @   t        j                  g d      } t        j                  g d      }t        | |d       \  }}}}t        | |d      \  }}}}|t        j                  |      k(  sJ |t        j                  |      k(  sJ |t        j                  |      k(  sJ t        | |d      \  }}}}t        j                  |       }	|t        j
                  ||	      k(  sJ |t        j
                  ||	      k(  sJ |t        j
                  ||	      k(  sJ y )N)r   rC   r   r   rC   rC   r   rC   r   r   rC   r   rC   r   rC   )rC   rC   r   rC   r   rC   rC   rC   rC   r   rC   r   rC   r   rC   r   r   r   r  )rH   r   r!   r   bincountr   )
r`   r_   r   r   r   r~   rZ   r   r   rj   s
             ra   .test_precision_recall_f1_score_binary_averagedr    s    XXCDFXXCDF 4FFDQMBB0QJAq!Q0TJAq!Qkk&!G

2w////

2w////

2w////rc   c                     t        j                  d      } 	 t        j                  g d      }t        j                  g d      }t        t	        ||d      dd       t        t        ||d      dd       t        t        ||d      dd       t        j                  d	i |  y # t        j                  d	i |  w xY w)
Nraise)rL  )r   rC   r<   r   rC   r<   )r<   r   rC   rC   r<   r   r   r   r   r<   r   )rH   seterrr   r3   r"   r#   r   )old_error_settingsr`   r_   s      ra   test_zero_precision_recallr    s     w/	(,-,-OFFGLcSTULI3PQRHVVWEsAN 			'&'		'&'s   A9B' 'B>c                      t        d      \  } }}t        | |ddg      }t        |ddgddgg       t        | |d	dg      }t        |d
d	gddgg       t        j                  |       dz   }t        | |d	|g      }t        |d
dgddgg       y )NFr   r   rC   r   r%  r   r   r<   r)  rf   )rb   r   r5   rH   max)r`   r_   r~   r  extra_labels        ra   .test_confusion_matrix_multiclass_subset_labelsr    s    'u5FFA 
&&!Q	8BrRGaV,- 
&&!Q	8BrRGb!W-. &&.1$K	&&![1A	BBrRGaV,-rc   zlabels, err_msgz+'labels' should contain at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     t        d      \  }}}t        j                  t        |      5  t	        |||        d d d        y # 1 sw Y   y xY w)NFr   r   r   )rb   r   r   r   r   )rv   r   r`   r_   r~   s        ra   test_confusion_matrix_errorr    sD     (u5FFA	z	1 878 8 8s   AAc            	         g d} t        j                  t        |             }t        | |       }|j                  t         j
                  k(  sJ t         j                  t         j                  t         j                  fD ]@  }t        | | |j                  |d            }|j                  t         j
                  k(  r@J  t         j                  t         j                  d t        fD ]@  }t        | | |j                  |d            }|j                  t         j                  k(  r@J  t        j                  t        |       dt         j                        }t        | | |      }|d   dk(  sJ |d   d	k(  sJ t        j                  t        |       d
t         j
                        }t        | | |      }|d   d
k(  sJ |d   dk(  sJ y )Nr   F)copyrE  l    rQ  r   r   r  l    l    )rH   onesry   r   rQ  int64bool_int32uint64astypefloat32float64objectfulluint32)rW   weightr  rQ  s       ra   test_confusion_matrix_dtyper    s|   AWWSV_F	!Q	B88rxx((BHHbii0 $a&--E-2RSxx288###$ **bjj$7 &a&--E-2RSxx2::%%%&
 WWSVZryy9F	!Qf	5Bd8z!!!d8z!!! WWSV0AF	!Qf	5Bd8****d8r>>rc   rQ  )Int64Float64booleanc                     t        j                  d      }t        j                  g d      }|j	                  ||       }|j	                  g dd      }t        ||      }t        ||      }t        ||       y)zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	rC   r   r   rC   r   rC   rC   r   rC   r   )	r   r   rC   rC   r   rC   rC   rC   rC   r  N)r   importorskiprH   r   Seriesr   r5   )rQ  pd	y_ndarrayr`   y_predictedoutputexpected_outputs          ra   %test_confusion_matrix_pandas_nullabler    sj     
		X	&B45IYYyY.F))7w)GKfk2F&y+>Ov/rc   c            	          t        j                         } t        | d      \  }}}d}t        ||t	        j
                  t        | j                              | j                        }||k(  sJ y )NFre   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rv   rw   r   rD   rb   r   rH   rI   ry   rw   r}   r`   r_   r~   r   r   s         ra   %test_classification_report_multiclassr!    sl    D'UCFFA
O #yyT../0&&	F _$$$rc   c                  >    g dg d}} d}t        | |      }||k(  sJ y )N)	r   r   r   rC   rC   rC   r<   r<   r<   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )r`   r_   r   r   s       ra   .test_classification_report_multiclass_balancedr#  9  s/    02MFF
O #662F_$$$rc   c                  x    t        j                         } t        | d      \  }}}d}t        ||      }||k(  sJ y )NFre   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r   rD   rb   r   r   s         ra   :test_classification_report_multiclass_with_label_detectionr%  K  sF    D'UCFFA
O #662F_$$$rc   c            	          t        j                         } t        | d      \  }}}d}t        ||t	        j
                  t        | j                              | j                  d      }||k(  sJ y )NFre   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rv   rw   digitsr  r   s         ra   1test_classification_report_multiclass_with_digitsr(  _  so    D'UCFFA
O #yyT../0&&F _$$$rc   c                      t        d      \  } }}t        j                  g d      |    } t        j                  g d      |   }d}t        | |      }||k(  sJ d}t        | |g d      }||k(  sJ y )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rw   rb   rH   r   r   )r`   r_   r~   r   r   s        ra   7test_classification_report_multiclass_with_string_labelr/  z  s    'u5FFAXX./7FXX./7F
O #662F_$$$
O #66PF_$$$rc   c                      t        d      \  } }}t        j                  g d      }||    } ||   }d}t        | |      }||k(  sJ y )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r.  r`   r_   r~   rv   r   r   s         ra   8test_classification_report_multiclass_with_unicode_labelr2    sW    'u5FFAXX:;FF^FF^F
O #662F_$$$rc   c                      t        d      \  } }}t        j                  g d      }||    } ||   }d}t        | |      }||k(  sJ y )NFr   )r*  greengreengreengreengreenr,  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r.  r1  s         ra   <test_classification_report_multiclass_with_long_string_labelr5    sW    'u5FFAXX23FF^FF^F
O #662F_$$$rc   c                      g d} g d}g d}d}t        j                  t        |      5  t        | |ddg|       d d d        y # 1 sw Y   y xY w)	Nr   r   r<   r   r   r   r<   r<   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r<   r  )r   r  r  r   )r`   r_   rw   r   s       ra   =test_classification_report_labels_target_names_unequal_lengthr:    sO    FF4L
BC	k	- XffaV,WX X Xs   AAc                      g d} g d}g d}d}t        j                  t        |      5  t        | ||       d d d        y # 1 sw Y   y xY w)Nr7  r8  r9  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r-  )r   r   r   r   )r`   r_   rw   r   s       ra   @test_classification_report_no_labels_target_names_unequal_lengthr<    sP    FF4L	. 
 
z	1 Iff<HI I Is   AAc                  ~    d} d}t        d|| d      \  }}t        d|| d      \  }}d}t        ||      }||k(  sJ y )Nr   r  rC   r   )rY   rX   r  rB   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rX   r~   r`   r_   r   r   s          ra   %test_multilabel_classification_reportr>    s_    II.	YQIAv /	YQIAvO #662F_$$$rc   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |t        j                  |            dk(  sJ t        | t        j                  |             dk(  sJ t        | t        j                  | j
                              dk(  sJ t        |t        j                  | j
                              dk(  sJ y )Nr   r   r   r   r   rC   )rH   r   r$   r   r   rG   r   s     ra   $test_multilabel_zero_one_loss_subsetr@    s    	9i(	)B	9i(	)BR C'''R A%%%R A%%%R^^B/0A555R^^B/0A555RXXbhh/0A555RXXbhh/0A555rc   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        j                  ddg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |d|z
        dk(  sJ t        | d| z
        dk(  sJ t        | t        j                  | j                              dk(  sJ t        |t        j                  | j                              d	k(  sJ t        | ||
      dk(  sJ t        | d|z
  |
      dk(  sJ t        | t        j
                  |       |
      dk(  sJ t        | d   |d         t        | d   |d         k(  sJ y )Nr   r   r   rC   r   UUUUUU?r   r   r   rE  gUUUUUU?gUUUUUU?)rH   r   r   r   rG   
zeros_like
sp_hamming)r   r   ws      ra   test_multilabel_hamming_lossrF    sm   	9i(	)B	9i(	)B
!QAB5(((B1$$$B1$$$AF#q(((AF#q(((BHHRXX./5888BHHRXX./3666Ba0H<<<AF!4	AAABMM"-Q?7JJJ1r!u%BqE2a5)AAAArc   c                     t        j                  g d      } t        j                  g d      }d}t        j                  t        |      5  t        | |dd       d d d        t        j                  g dg dg      } t        j                  g d	g d
g      }d}t        j                  t        |      5  t        | |dd       d d d        t        j                  g d      } t        j                  g d      }d}t        j                  t        |      5  t        | |d       d d d        d}t        j                  t        |      5  t        | |d       d d d        d}t        j                  t        |      5  t        | |dd       d d d        y # 1 sw Y   AxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   dxY w# 1 sw Y   y xY w)N)r   rC   r   rC   rC   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rU   r<   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   rC   rC   r   r<   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )rH   r   r   r   r   r   r  r  )r`   r_   r   msg1msg2msg3r   s          ra   test_jaccard_score_validationrN  #  s   XXo&FXXo&FOG	z	1 Effh!DE XXy),-FXXy),-F	6 	
 
z	. Fffh"EF XXo&FXXo&F	 	
 
z	. 8ffh78WD	z	. 9ffi89	  
k	- DffgCD DAE EF F8 89 9D Ds<   F7F(F4G GF%(F14F= G	Gc           	         t        j                  g dg dg      }t        j                  g dg dg      }t        ||d      dk(  sJ t        ||d      dk(  sJ t        ||d      dk(  sJ t        |t        j                  |      d      dk(  sJ t        |t        j                  |      d      dk(  sJ t        |t        j                  |j
                        d      dk(  sJ t        |t        j                  |j
                        d      dk(  sJ t        j                  g dg d	g      }t        j                  g d
g dg      }t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        t        ||dddg      d       t        t        ||dddg      d       t        t        ||d       t        j                  g d             t        j                  g dg dg      }t        j                  g d
g dg      }t        t        ||d      d       t        t        ||d      d       d}t        j                  t        |      5  t        ||dgd       d d d        d}t        j                  t        |      5  t        ||dgd       d d d        d}t        j                  t        |      5  t        t        j                  ddgg      t        j                  ddgg      d      dk(  sJ 	 d d d        d}t        j                  t        |      5  t        t        j                  ddgddgg      t        j                  ddgddgg      d      dk(  sJ 	 d d d        t        |       rJ y # 1 sw Y    xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   <xY w)Nr   r   r   r   r   r   rC   r   r   r   r   r   r   r   g?r<   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)rH   r   r   r   r   rG   r3   r5   r   r   r   r  r   rP  )recwarnr   r   r`   r_   rL  rM  r   s           ra   test_multilabel_jaccard_scorerQ  K  sw   	9i(	)B	9i(	)B
 R3t;;;R3q888R3q888R^^B/CqHHHR^^B/CqHHHRXXbhh/CqHHHRXXbhh/CqHHHXXy),-FXXy),-FffgFPffgFPffiH(SffiAG ffiAG ffd3RXX>U5V XXy),-FXXy),-FffgFPffjI7SD	z	. CffaS'BCD	z	. DffbT7CD	- 
 
,C	8 
"((QF8,bhhAx.@'R	

	, 
 
,C	8 
1a&1a&)*1a&1a&)*!
 	

 G}}AC CD D
 

 
s2    N/6N<,A OAO/N9<OOOc           	         g d}g d}g d}t               }|j                  |       |j                  |      }|j                  |      }t        t        ||      }t        t        ||      }ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]2  }t        |	|
      D ]!  \  }}t         |||       |||             # 4 t        j                  ddgddgddgg      }t        j                  ddgddgddgg      }t               5  t	        ||d      dk(  sJ 	 d d d        t        |       rJ y # 1 sw Y   xY w)N)antrS  catrT  rS  rT  birdrU  )rT  rS  rT  rT  rS  rU  rU  rT  )rS  rU  rT  rS  rU  rT  r   rC   r<   )r   r   r   Nr   r   r   )r)   rQ   	transformr   r   zipr3   rH   r   r6   rP  )rP  r`   r_   rv   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 ra   test_multiclass_jaccard_scorer_    s   GFGF#F		BFF6Nf%Jf%J!-@z:F						 1v1v1vsQC!dCO 8  #$5 G 	GW#GGD!''B	 XX1v1v1v./FXX1v1v1v./F		 FVVZ@AEEEF G}}F Fs   D88Ec                    t        dgdgd      dk(  sJ d}t        j                  t        |      5  t        ddgddgd      dk(  sJ 	 d d d        t        dgdgdd      d	k(  sJ t	        j
                  g d
      }t	        j
                  g d      }t        t        ||d      d       t        t        ||dd      d       t        |       rJ y # 1 sw Y   xY w)NrC   r   rU   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r  r   )rC   r   rC   rC   r   )rC   r   rC   rC   rC   r   rH  r   )r   r   r  r   rH   r   r3   rP  )rP  r   r`   r_   s       ra   !test_average_binary_jaccard_scorera    s    !qc84;;;	'  
,C	8 FaVaVX>#EEEF !qcQASHHHXXo&FXXo&FffhGQffh!Dg G}}F Fs   CC c                  (   t        j                  g dg dg      } t        j                  g dg dg      }d}t        j                  t        |      5  t        | |dd      }|t        j                  d      k(  sJ 	 d d d        y # 1 sw Y   y xY w)	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )rH   r   r   r  r   r   r   )r`   r_   r   scores       ra   (test_jaccard_score_zero_division_warningrf    s     XXy),-FXXy),-F	C 
 
,C	8 +ffivVc****+ + +s   *BBzzero_division, expected_scorer  )rC   r   c                 H   t        j                  g dg dg      }t        j                  g dg dg      }t        j                         5  t        j                  dt
               t        ||d|       }d d d        t        j                  |      k(  sJ y # 1 sw Y   $xY w)Nr   rc  r   r   rd  )	rH   r   r   r   r   r   r   r   r   )r   expected_scorer`   r_   re  s        ra   *test_jaccard_score_zero_division_set_valueri    s     XXy),-FXXy),-F		 	 	" 
g'=>FI]


 FMM.1111
 
s   *BB!c                  h   t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        | |d       \  }}}}t        |g dd       t        |g dd       t        |g d	d       t        |g d
d       t	        | |dd       }|}t        |g dd       t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  |             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d|z  |z  d|z  |z   z         t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  ||             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d       y )NrC   r   r   r   r   rC   r   r   r  rC   r   rC   r   r   )r   r   r   r   r<   )r   r   r   r   )r   r   rC   r   )rC   rC   rC   rC   r   r   )r   r  rC   r   r   g      ?r   g?r   r   r   r   r  r   rH   r   r!   r4   r   r3   r   r   r`   r_   rZ   r   r   r   f2rj   s           ra   +test_precision_recall_f1_score_multilabel_1rr    s$   
 XX|\<@AFXX|\<@AF0NJAq!Q a!5q9a!5q9a!7;aq1	VV!T	:BGb/15 1QJAq!Q7#3+,99FFG<bggbk
 1QJAq!Q33399FFG<	!a1q519% 1TJAq!Q7#3+,99FFJ?


2w' 1SJAq!Q33399FFINPSTrc   c                  j   t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        | |d       \  }}}}t        |g dd       t        |g d	d       t        |g d
d       t        |g dd       t	        | |dd       }|}t        |g dd       t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d|z  |z  d|z  |z   z         t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  |             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  ||             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      dd       y )Nrk  rl  r  r   r   r   rC   r  r   )r   r   r   r   r<   )r   r   r   r   )r   gQ?r   r   rC   r<   rC   r   rn  )r   皙?r   r   r         ?r   r   r   g      ?rB  r   r   rk   r  r   g&S?ro  rp  s           ra   +test_precision_recall_f1_score_multilabel_2rx  3  s'    XX|\<@AFXX|\<@AF 1NJAq!Qa!5q9a!5q9a!6:aq1	VV!T	:BGb/150QJAq!Q4 4 0199FFG<	!a1q519%
 1QJAq!Q4 5!6"99FFG<bggbk 1TJAq!Q5!5!=)99FFJ?


2w'
 1SJAq!Q 5!5!=)99FFI>rc   z%zero_division, zero_division_expected)r   r   r  c           	         t        j                  g dg dg dg      }t        j                  g dg dg dg      }t        ||d |       \  }}}}t        ||dddgd	       t        |dd
d|gd	       d}t        ||dd|gd	       t        |g dd	       t	        ||d	d |       }	|}
t        |	|dd|gd	       t        ||d|       \  }}}}t        j
                  |      rdn|}dt        j
                  |       z   }t        |d	|z   |z         t        |d|z   |z         d}t        ||       |J t        t	        ||d	d|       t        |	d              t        ||d|       \  }}}}t        |d       t        |d
       t        |d       |J t        t	        ||d	d|       d|z  |z  d|z  |z   z         t        ||d|       \  }}}}t        ||dk(  rdnd       t        |d
       d}t        |d|z         |J t        t	        ||d	d|       t        |	|
             t        ||d      \  }}}}t        |d       t        |d       t        |d       |J d }t        t	        ||d	d|       |d	       y )!Nrl  rk  r  r_  rt  rd  r   r   r<   r   r   r   rC   ru  r   r   r   rv  r   r         ?g?r  r   rn   r   r   r   r   g@r   r   rk   gZd;O?)rH   r   r!   r4   r   r  r3   r7   )r   zero_division_expectedr`   r_   rZ   r   r   r   
expected_frq  rj   value_to_sumvalues_to_averageexpected_results                 ra   7test_precision_recall_f1_score_with_an_empty_predictionr  t  s    XX|\<@AFXX|\<@AF 1MJAq!Q a"8#sC!H!La#sC1G!H!LJa*gq*!EqIaq1	VV!T	WBGb:tQ
"CQG0}JAq!Q !781>TL*@!AABA,0AABC,.2CCD J:&99'	
 	B%	 1}JAq!Q 5!30199FG=	
 
!a1q519%	 1
-JAq!Q $:a$?5SI3M->>?99FJm	
 	B(	 1SJAq!Q 5!5!5!99OFI]	
 		rc   r   )r   r   r   r   c                    t        j                  d      }t        j                  |      }t        j                         5  t        j
                  d       t        |||| |      \  }}}}t        ||| ||      }	d d d        J t        j                  |      r#	fD ]  }
t        j                  |
      rJ  y t        |      }t        |       t        |       t        |       t        	t        |             y # 1 sw Y   xY w)Nro   r   r   r   r   r   rz  )rH   r   rC  r   r   r   r!   r   r  r|   r3   )r   r   r   r`   r_   rZ   r   r   r   fbetar   s              ra   "test_precision_recall_f1_no_labelsr    s	    XXgF]]6"F		 	 	" 
g&4'

1a '

" 99 
xx!Q& 	$F88F###	$-(M=)=)=)u]34=
 
s   ;DDc                    t        j                  d      }t        j                  |      }t        }t	        j
                  t              5   |||| d      \  }}}}d d d        t        d       t        d       t        d       J t	        j
                  t              5  t        ||| d      }d d d        t        d       y # 1 sw Y   oxY w# 1 sw Y   "xY w)Nr  r   r  r   )	rH   r   rC  r!   r   r  r   r3   r   )	r   r`   r_   funcrZ   r   r   r   r  s	            ra   1test_precision_recall_f1_no_labels_check_warningsr    s    XXgF]]6"F*D	,	- E&&'D
1aE 11199	,	- GFFG#FG q!E EG Gs   
C
%C
CCc                    t        j                  d      }t        j                  |      }t        j                         5  t        j
                  d       t        ||d d|       \  }}}}t        ||dd |       }d d d        t        j                  |       } t        | | | gd       t        | | | gd       t        | | | gd       t        g dd       t        | | | gd       y # 1 sw Y   nxY w)Nr  r   r   r  rz  r<   rc  )
rH   r   rC  r   r   r   r!   r   r
  r4   )r   r`   r_   rZ   r   r   r   r  s           ra   /test_precision_recall_f1_no_labels_average_noner    s    XXgF]]6"F 
	 	 	" 
g&4'

1a Fd-

 JJ}-Ma-!NPQRa-!NPQRa-!NPQRaA.em]M%RTUV)
 
s   ;C''C0c                     t        j                  d      } t        j                  |       }t        j                  t
              5  t        | |d d      \  }}}}d d d        t        g dd       t        g dd       t        g dd       t        g dd       t        j                  t
              5  t        | |dd       }d d d        t        g dd       y # 1 sw Y   xY w# 1 sw Y   %xY w)Nr  rC   r  rc  r<   rn  )	rH   r   rC  r   r  r   r!   r4   r   )r`   r_   rZ   r   r   r   r  s          ra   4test_precision_recall_f1_no_labels_average_none_warnr  7  s    XXgF]]6"F 
,	- 
4FDq

1a

 aA.aA.aA.aA.	,	- BFFDAB eY2
 
B Bs   C6C*C'*C3c            	         t         t        }} dD ]d  }d}t        j                  ||      5   | g dg d|       d d d        d}t        j                  ||      5   | g dg d|       d d d        f d}t        j                  ||      5   | t	        j
                  d	d
gd	d
gg      t	        j
                  d	d
gd
d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d	d
gd
d
gg      t	        j
                  d	d
gd	d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d	d	gd	d	gg      t	        j
                  d
d
gd
d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d
d
gd
d
gg      t	        j
                  d	d	gd	d	gg      d       d d d        d}t        j                  ||      5   | d	d	gddgd       d d d        d}t        j                  ||      5   | ddgd	d	gd       d d d        t        j                  d      5 }t        j                  d       t        d
d
gd
d
gd       d}t        |j                         j                        |k(  sJ d}t        |j                         j                        |k(  sJ d}t        |j                         j                        |k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   "xY w# 1 sw Y   ixY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   sxY w# 1 sw Y   OxY w# 1 sw Y   +xY w# 1 sw Y   y xY w)NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   rC   rC   r<   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.rC   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rU   Tr   r   F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r!   r   r   r  rH   r   r   r   r   r   popr   )r   rE  r   r   r   s        ra   test_prf_warningsr  S  s   *,BqA. 5 	 \\!3' 	5iG4	5 	 \\!3' 	5iG4	5 	5!5*	  
as	# U	"((QFQF#
$bhhAA/?&@)TU	  
as	# U	"((QFQF#
$bhhAA/?&@)TU
	  
as	# S	"((QFQF#
$bhhAA/?&@'RS	  
as	# S	"((QFQF#
$bhhAA/?&@'RS
	  
as	# .	1a&2r(H-.	  
as	# .	2r(QFH-. 
	 	 	- 0h''AAI 	
 6::<''(C/// 	 6::<''(C/// 	 6::<''(C///-0 0K	5 	5	5 	5U UU US SS S. .. .0 0sl   LL'>L4.>M>M,>MM(<M5*B&NL$	'L1	4L>MMM%(M25M?Nc           	         t        j                         5  t        j                  d       dD ](  }t        g dg d||        t        g dg d||        * t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d	|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d	|        t        ddgd
d
gd|        t        d
d
gddgd|        d d d        t        j                  d      5 }t        j                  d       t        ddgddgd|        t        |      dk(  sJ 	 d d d        y # 1 sw Y   bxY w# 1 sw Y   y xY w)Nr   r  r   r  rd  rC   r   r   r   r   rU   Tr   r   )r   r   r   r!   rH   r   ry   )r   r   r   s      ra   )test_prf_no_warnings_if_zero_division_setr    s   		 	 	" 2
g& 3 	G+9g] ,9g]	 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(FRHhm	
 	(Hq!fhm	
a2
h 
	 	 	-  h''FQFHM	
 6{a   i2
 2
h   s   E-G# 9G/#G,/G8c           	         t        j                         5  t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        d d d        t        j                  d      5 }t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        | d	k(  r(t        |j                         j                        d
k(  sJ t        |      dk(  sJ t        ddgddg       | d	k(  r(t        |j                         j                        d
k(  sJ d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rC   r   r   rd  Tr   r   r   r  )
r   r   r   r#   rH   r   r   r  r   ry   r   r   s     ra   test_recall_warningsr    ss   		 	 	" 
g&HHq!fq!f%&HHq!fq!f%&'		

 
	 	 	- h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###aVaV$F"FJJL(() ." ""+ 
 
 s   AE$
CE0$E-0E9c           	         t        j                  d      5 }t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        | dk(  r(t        |j                         j                        d	k(  sJ t        |      dk(  sJ t        ddgddg       | dk(  r(t        |j                         j                        d	k(  sJ d d d        t        j                         5  t        j                  d
       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        d d d        y # 1 sw Y   }xY w# 1 sw Y   y xY w)NTr   r   rC   r   r   rd  r   r  r   )
r   r   r   r"   rH   r   r   r  r   ry   r  s     ra   test_precision_warningsr  !	  ss   		 	 	- h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###AA'F"FJJL(() ." ""+6 
	 	 	" 
g&HHq!fq!f%&HHq!fq!f%&'		

 
7 6
 
s   CE$AE0$E-0E9c           
         t        j                  d      5 }t        j                  d       t        t	        t
        d      fD ]  } |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       t        |      dk(  sJ  |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       t        |      dk(  sJ  |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       | d
k(  r*t        |j                         j                        dk(  rJ t        |      dk(  rJ  	 d d d        y # 1 sw Y   y xY w)NTr   r   r<   r   rC   r   r   rd  r   r  )r   r   r   r   r   r   rH   r   ry   r   r  r   )r   r   re  s      ra   test_fscore_warningsr  I	  s~   		 	 	- "(h'! <= 	(E1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 &

,,- 2- -- 6{a'''?	("( "( "(s   D6E.E.!E..E7c                  x   g d} g d}d}t        j                  g dg dg dg      }t        j                  g dg dg dg      }d}| ||f|||ffD ]Y  \  }}}t        t        t        t        t        d	
      fD ]/  }	t        j                  t        |      5   |	||       d d d        1 [ y # 1 sw Y   >xY w)N)rC   r<   r   r   rC   r<   r   rC   rJ  r   r   r   r4  rI  r<   r   r   )
rH   r   r"   r#   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indr`   r_   r   r   s
             ra   'test_prf_average_binary_data_non_binaryr  p	  s    II	1 
 9i;<J9i;<J	<  
Iv&	Z)  '
 Ka(	
 	'F z5 'vv&' '	'	'' 's   
B00B9c                     d} d}d}d}d}d}| t        j                  g dg dg d	g      f| t        j                  d
dgdd
gddgg      f|g df|g df|g df|t        j                  dgdgdgg      f|t        j                  d
gdgdgg      f|t        j                  dgdgdgg      f|t        j                  d
dgddgddgg      f|t        j                  ddgddgddgg      fg
}i | | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t        |d      D ]  \  \  }}	\  }
}	 |||
f   }|t	        j
                  t              5  t        |	|       d d d        ||
k7  rCdj                  ||
      }t	        j
                  t        |      5  t        |	|       d d d        |||| fvsdj                  |      }t	        j
                  t        |      5  t        |	|       d d d        t        |	|      \  }}}}||k(  sJ |j                  d      r"|j                  dk(  sJ |j                  dk(  s@J t        |t        j                  |	             t        |t        j                  |             t	        j
                  t              5  t        |	d d |       d d d         ddg}	d d!g}d"}t	        j
                  t        |      5  t        |	|       d d d        y # t        $ r ||
|f   }Y w xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   
xY w# 1 sw Y   xY w# 1 sw Y   y xY w)#Nmultilabel-indicator
multiclassrU   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   rC   )r<   r   rC   )r   r{  r   r<   r   r   r{  r   r   rX  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )rC   r<   )r   r<   r   )r<   )r   r<   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)rH   r   r   KeyErrorr   r   r   r%   format
startswithr5   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr~   r   s                      ra   test__check_targetsr  	  s    !C	B
C
C
"C
"C 
bhh	9i89:	bhhAAA/01	Y	i	o	RXXsQC!o&'	bhhaS1#'(	bhhuse,-.	bhhAAA/01	bhhc
S#Jc
;<=H	c
C	R" 
c
C 
S	4	
 
c
D 
b	2 
c
D 
c
D 
c
D 
c
D 
S	4 
c
D 
c
D 
c
D  
c
D!" 
S	4#$ 
c
D%& 
c
D	c
D	S	4	c
D-H2 %,HQ$? !, [eR	.u-H z* 'r2&' ~--3VE5-A  ]]:W= +"2r*+ + b#.4;;EBGzA /&r2./ / ,:"b+A(Kq(***%%l3||u,,,||u,,,"5"**R.9"5"**R.9z* ,r#2w+, ,A!,H )	B
B	3  
z	- r2 S  	.u-H	.' '+ +/ /, , sN   2L3M
M%M$M1M>3MM
M	M!	$M.	1M;	>Nc                      d} t        j                  t        t        j                  |             5  t        t        j                  g       t        j                  g              d d d        y # 1 sw Y   y xY w)NzIFound empty input array (e.g., `y_true` or `y_pred`) while a minimum of 1r   )r   r   r   reescaper%   rH   r   r  s    ra   *test__check_targets_raises_on_empty_inputsr  	  sL    
UC	z3	8 3rxx|RXXb\23 3 3s   3A--A6c                  <    ddg} ddg}t        | |      d   dk(  sJ y )Nr   rC   r   r  )r%   ra  s     ra   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  	  s.    VFWF&&)!,<<<rc   zy, target_typerU   r<   r  r   r4  r5  r  c                     |dv r1t        j                  t        d      5  t        | |        ddd       yt        | |       \  }}}}|dk(  sJ |j                  dk(  sJ |j                  dk(  sJ y# 1 sw Y   yxY w)z?Check correct behaviour when different target types are sparse.)rU   r  z+Sparse input is only supported when targetsr   Nr  r  )r   r   	TypeErrorr%   r  )rW   target_typey_type
y_true_out
y_pred_outr~   s         ra   !test__check_targets_sparse_inputsr  	  s     ..]]J
 	! 1a 	! 	! -;1a,@)
J////  E)))  E)))	! 	!s   A11A:c                      t        j                  g d      } t        j                  g d      }t        | |      dk(  sJ t        j                  g d      } t        j                  g d      }t        | |      dk(  sJ y )N)r   rC   rC   r   )g      !r   r{  g333333ӿr   )r   r<   r<   r   )rH   r   r   r`   pred_decisions     ra   test_hinge_loss_binaryr  
  sf    XXn%FHH34Mfm,777XXl#FHH34Mfm,777rc   c            
         t        j                  g dg dg dg dg dg dg      } t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d   z
  | d   d	   z   d| d	   d	   z
  | d	   d
   z   d| d
   d   z
  | d
   d	   z   d| d   d
   z
  | d   d	   z   d| d   d	   z
  | d   d
   z   g      }t        j                  |dd |       t        j                  |      }t	        ||       |k(  sJ y )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   rC   r<   rC   r   r<   rC   r   r<   r   r   r   outrH   r   clipr   r   )r  r`   dummy_lossesdummy_hinge_losss       ra   test_hinge_loss_multiclassr   
  sJ   HH((((((	
	M XX()F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,fm,0@@@@rc   c                      t        j                  g d      } t        j                  g dg dg dg dg      }d}t        j                  t        |      5  t        | |       d d d        y # 1 sw Y   y xY w)N)r   rC   r<   r<   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )rH   r   r   r   r   r   )r`   r  error_messages      ra   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  ;
  si    XXl#FHH((((		
M 	O  
z	7 *6=)* * *   A,,A5c            
         t        j                  g d      } t        j                  g d      }d}t        j                  t        t        j                  |            5  t        | |       d d d        t        j                  ddgddgddgddgddgddgddgg      }g d	}d
}t        j                  t        t        j                  |            5  t        | ||       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N)r<   rC   r   rC   r   rC   rC   )r   rC   r<   rC   r   r<   rC   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   rC   r<   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))r`   r  rv   )rH   r   r   r   r   r  r  r   )r`   r  r  rv   s       ra   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  L
  s     XX+,FHH23M	  
z=)A	B ?&>? HHq!fq!fq!fq!fq!fq!fqRSfUVMF	  
z=)A	B N&fMN N? ?N Ns   C+C7+C47D c            	         t        j                  g dg dg dg dg dg      } t        j                  g d      }t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d   z
  | d   d   z   d| d   d   z
  | d   d	   z   d| d	   d   z
  | d	   d   z   d| d
   d   z
  | d
   d	   z   g      }t        j                  |dd |       t        j                  |      }t	        || |      |k(  sJ y )Nr  皙r  r  r  r  )r   rC   r<   rC   r<   r   rC   r<   r   rC   r   r<   r   r   r  r   r  r  r`   rv   r  r  s        ra   .test_hinge_loss_multiclass_with_missing_labelsr  h
  s5   HH(((((	
M XXo&FXXl#F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,fmF;?OOOOrc   c            	         t        j                  g dg dg dg dg dg      } t        j                  g d      }t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d	   z
  | d   d   z   d| d	   d	   z
  | d	   d   z   d| d
   d   z
  | d
   d	   z   d| d   d	   z
  | d   d   z   g      }t        j                  |dd |       t        j                  |      }t	        t        || |      |       y )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r<   r<   r   r<   r   rC   r   r<   r   r   r  r   )rH   r   r  r   r3   r   r  s        ra   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer  
  s6   
 HH!!!!!	
M XXo&FXXi F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,6=8:Jrc   c            
         g d} g dg dg dg dg dg dg}t        j                  d|d   d   z
  |d   d   z   d|d   d   z
  |d   d   z   d|d   d   z
  |d   d	   z   d|d	   d   z
  |d	   d   z   d|d
   d	   z
  |d
   d   z   d|d   d   z
  |d   d	   z   g      }t        j                  |dd |       t        j                  |      }t	        | |      |k(  sJ y )N)r*  r+  r,  r+  whiter,  r  r  r  r  rC   r   r<   r   r   r   r  r  )r`   r  r  r  s       ra   +test_hinge_loss_multiclass_invariance_listsr  
  s6    ?F$$$$$$M 88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,fm,0@@@@rc   c            	      &   g d} t        j                  ddgddgddgddgddgd	d
gg      }t        | |      }t        j                  t	        j
                  t        j                  |       dk(  |d d df                }t        ||       g d} g dg dg dg}t        | |d      }t        |d       | dz  } |dz  }t        | |d      }t        |d       g d} ddgddgddgg}t        j                  t              5  t        | |       d d d        g d} g dg dg dg}g d}d }t        j                  t        t        j                  |      !      5  t        | ||"       d d d        g d#} ddgddgddgddgg}t        | |      }t        |d$       ddg} d%d&gddgg}t        j                  ddgddgg      }d'}t        j                  t        t        j                  |      !      5  t        | |       d d d        d%d&gddgddgg}d(}t        j                  t        t        j                  |      !      5  t        | |       d d d        t        j                  t        j                  |d d df                }t        | |ddg"      }t        ||       g d)} g d*g dg d+g}	t        | |	g d,"      }t        |t        j                  d              y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)-Nnor  r  yesr  r  r   r   rm   {Gz?Gz?r   rw  gMbP?g+?r  rC   rG  r   r   r   r   r   r   r   r   r   TrN  g躕ʀ?r<   Fg.L`@r   r   r   r   r   )rm   r   r   )r   rm   r   r   r   r   )r   r   r   zPy_true contains values {'b'} not belonging to the passed labels ['a', 'c', 'd'].r   r   hamspamr   r  CT?r   r   zy_true contains only one label (2). Please provide the list of all expected class labels explicitly through the labels argument.zBFound input variables with inconsistent numbers of samples: [3, 2]r   )r   r   r   r   r   r   rF  )rH   r   r   r   r
   logpmfr2   r   r   r   r  r  log)
r`   r_   loss	loss_truerv   	error_strr   true_log_losscalculated_log_lossy_score2s
             ra   test_log_lossr  
  s
   4FXX
sc3Z$SzD$<%QVXF FF#D))"((6*:e*CVAqD\RSSID)$ F@FFFd3DD)$ aKF
aKFFFe4DD-( FCj3*sCj1F	z	" ! ! F@FF	"  
z9)=	> 0/0 ,FCj3*sCj3*=FFF#DD)$ VFCj3*%Fhhc
S#J/0G	H  
z9)=	> ! ! Cj3*sCj1FTI	z9)=	> ! !
 WWRVVGAqDM233M"67Aq6B'7 F/BHFHY7DD266#;,'_! !0 0$! !
! !s0    K!K.0K; L!K+.K8;LLc                     t        j                  ddg|       }t        j                  ddg|       }t        ||      }t        j                  |      sJ y)zCheck the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   rC   r   N)rH   r   r   isfinite)rQ  r`   r_   r  s       ra   test_log_loss_epsr  
  sJ     XXq!fE*FXXq!fE*FFF#D;;trc   c                     t        j                  g d      }t        j                  ddgddgddgddgg|       }t        j                  t        d	
      5  t        ||       ddd       y# 1 sw Y   yxY w)zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r  r   r   r   r   r   r   r   z$The y_prob values do not sum to one.r   N)rH   r   r   r  r  r   )rQ  r`   r_   s      ra   'test_log_loss_not_probabilities_warningr    si     XXl#FXXSzC:SzC:FeTF	k)O	P ! ! ! !r  r   r   c                 L    t        | |      t        j                  d      k(  sJ y)z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   ra  s     ra   !test_log_loss_perfect_predictionsr  "  s"     FF#v}}Q'7777rc   c                  H   t        j                  g d      } t        j                  ddgddgddgddgg      }t        t        fg}	 ddlm}m} |j                  ||f       |D ]-  \  }} ||        ||      }}t        ||      }	t        |	d       / y # t        $ r Y >w xY w)	Nr  r   r   r   r   r   )	DataFramer  r  )
rH   r   r0   r  r  r  r  ImportErrorr   r2   )
y_try_prtypesr  r  TrueInputTypePredInputTyper`   r_   r  s
             ra   test_log_loss_pandas_inputr  0  s    8823D88c3Z#sc3Z#sDED]+,E,fi() ). )$}&t,mD.A'i(	)  s   B 	B! B!c                      t        j                  d      } t        j                  t        |       5  t        g dg dg dg dgg d       d d d        y # 1 sw Y   y xY w	NzLabels passed were ['spam', 'eggs', 'ham']. But this function assumes labels are ordered lexicographically. Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that the columns of y_prob correspond to this ordering.r   eggsr   r  r   r4  r   )r   r  r  r   )r  r  r   r  r  r   expected_messages    ra   test_log_loss_warningsr"  B  sT    yy	= 
k)9	: 
#	9-*	

 
 
   AAc                     t        j                  g d      } t        j                  g d      }t        j                  | |z
        dz  t	        |       z  }t        t        | |       d       t        t        | |      |       t        t        d| z   |      |       t        t        d| z  dz
  |      |       t        j                  d|z
  |f      }t        j                  d| z
  | f      }t        t        | |      |       t        t        ||      |       t        t        | |d      |       t        t        | |d	      |       t        t        | |d
      d|z         t        t        dgdg      d       t        t        dgdg      d       t        t        dgdg      d       t        t        dgdgd      d       t        t        dgdgd      d       y )Nr   rC   rC   r   rC   rC   r   r   rm   r   r   gffffff?r<   r   r   rC   auto)scale_by_halfTFr   r   g|Gz?r   r  foobarr   )rH   r   r   normry   r3   r   column_stack)r`   y_prob
true_scorey_prob_reshapedy_true_reshapeds        ra   test_brier_score_loss_binaryr1  Q  s   XX()FXX56FVf_-2S[@J(8#>(8*E(vv>
K(Va@*M ooq6z6&:;Oooq6z6&:;O(A:N(/JJW v>
 t<j u=q:~
 ("u5v>(!se4f=(!se4nE(%3%5I6R%3%59rc   c            	         t        t        g dg dg dg dgg d      d       t        t        g dg dg d	g d
g      d       t        t        g dg dg dg dg      d       t        t        g dg dg dg dg      d       y )Nr  rk  rl  )r  r  r   yamsr   r   rG  r  r  r  gt?r   )r   r   r   )r   r   r   )r   r   r   r   r<   )r3   r   r   rc   ra    test_brier_score_loss_multiclassr4  x  s    #<62	

 	 /J	
 		 /J	
 	
	 /J	
 	
	rc   c                  &   t        j                  g d      } t        j                  g d      }t        j                  t              5  t        | |dd         d d d        t        j                  t              5  t        | |dz          d d d        t        j                  t              5  t        | |dz
         d d d        t        j                  g d      } t        j                  g dg dg dg      }t        j                  t              5  t        | |dd         d d d        t        j                  t              5  t        | |dz          d d d        t        j                  t              5  t        | |dz
         d d d        t        j                  g d	      } t        j                  g d
      }t        j                  d      }t        j                  t        |      5  t        | |       d d d        g d} ddgddgddgg}d}t        j                  t        t        j                  |            5  t        | |       d d d        g d} g dg dg dg}g d}d}t        j                  t        t        j                  |            5  t        | ||       d d d        dg} ddgg}d}t        j                  t        t        j                  |            5  t        | |       d d d        t        t        | |ddg      d       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   uxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   nxY w# 1 sw Y   'xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nr%  r&  rC   r   rG  r  r  r  )r   rC   r<   r   r   r   r   r   zpThe type of the target inferred from y_true is multiclass but should be binary according to the shape of y_prob.r   r   r   zy_true and y_prob contain different number of classes: 3 vs 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2]r  r   r4  )r  r   r  r3  zwThe number of classes in labels is different from that in y_prob. Classes found in labels: ['eggs' 'ham' 'spam' 'yams']r   r  rm   r   zy_true contains only one label (eggs). Please provide the list of all expected class labels explicitly through the labels argument.r  r  )	rH   r   r   r   r   r   r  r  r3   )r`   r-  r  rv   s       ra   $test_brier_score_loss_invalid_inputsr7    s   XX()FXX56F	z	" -,- 
z	" /#./ 
z	" /#./
 XXi FXXIJF	z	" -,- 
z	" /#./ 
z	" /#./
 XXl#FXX*+FII	AM 
z	7 )() F!fq!fq!f%F	  
z=)A	B )() %FI.F,F	/ 
 
z=)A	B 878 XFCj\F	 
 
z=)A	B )() (PRVWK- -/ // /- -/ // /) )) )8 8) )sx   L9L *L-L:M2M(M!<M.M;#NL L*-L7:MMM!M+.M8;NNc                      t        j                  d      } t        j                  t        |       5  t        g dg dg dg dgg d       d d d        y # 1 sw Y   y xY wr  )r  r  r   r  r  r   r   s    ra   test_brier_score_loss_warningsr9    sZ    yy	= 
k)9	: 	
#
 +	
	
 	
 	
r#  c                      d} t        j                  t        |       5  t        g dg d       d d d        y # 1 sw Y   y xY w)Nz%y_pred contains classes not in y_truer   rc  r   )r   r  r  r   r  s    ra   #test_balanced_accuracy_score_unseenr;    s4    
1C	k	- 6	956 6 6s	   8Azy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                 T   t        | |dt        j                  |             }t               5  t	        | |      }d d d        t        j                  |      k(  sJ t	        | |d      }t	        | t        j                  | | d               }|||z
  d|z
  z  k(  sJ y # 1 sw Y   exY w)Nr   r   T)adjustedr   rC   )r#   rH   uniquer6   r   r   r   	full_like)r`   r_   macro_recallbalancedr=  chances         ra   test_balanced_accuracy_scorerC     s      		&0AL 
	 ;*66:; v}}\2222&vvEH$VR\\&&)-LMF6)a&j9999; ;s   BB'r   ))FTr  )r   r   )zeroonec                 8   t         j                  j                  d      }d|d   }}|j                  ||d      }| t        u r|j                  |      }n|j                         } | |||      }t        j                  t        j                  |            rJ y)	zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rm  r   T)r  replacer  r   N)	rH   rL   rM   choicer   uniformr  anyr  )r   r   r[   rX   r   r`   r_   r  s           ra   *test_classification_metric_pos_label_typesrL    s    * ))


#Cwr{yIZZiZ>F!!),FFi8Fvvbhhv&''''rc   zy_true, y_pred, expected_scorec                 P    t        | |d      t        j                  |      k(  sJ y)zCheck the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r   r  N)r   r   r   )r`   r_   rh  s      ra   2test_f1_for_small_binary_inputs_with_zero_divisionrN  6  s$     FF#6&--:WWWWrc   scoringr  )r   r   c                     t        j                  d      \  }}t        dd      j                  ||      }t	        |||| dd       y)	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )rB   r   )	max_depthrB   r<   r  )rO  n_jobserror_scoreN)r   make_classificationr+   rQ   r(   )rO  rV   rW   
classifiers       ra   9test_classification_metric_division_by_zero_nan_validatonrV  J  sC    ( ''Q7DAq'!!DHHANJJ1gaWUrc   c            	         g d} g d}t        j                  ddgddgddgddgdd	gd
dgg      }t        j                  ddgddgddgddgddgddgg      }t        | |      }t        | |d      }t        | |d      }d||z  z
  }|t	        j
                  |      k(  sJ t        j                  g d      }|d d j                         |j                         z  |d d df<   |dd  j                         |j                         z  |d d df<   t        | ||      }t        | ||d      }t        | ||d      }d||z  z
  }|t	        j
                  |      k(  sJ t        j                  ddgddgddgddgddgddgg      }t        | |      }d|cxk  rdk  sJ  J t        ||      }	|	t	        j
                  |      k(  sJ t        j                  ddgddgddgddgddgddgg      }t        | |      }|dk  sJ t        ||      }	|	t	        j
                  |      k(  sJ g d} t        j                  ddgddgddgddgddgddgg      }t        | |      }|dk(  sJ t        ||      }	|	dk(  sJ g d} g d}t        j                  ddgddgddgddgg      }t        | |      }|dk(  sJ t        ||      }	|	dk(  sJ g d}t        | ||      }
|
dk(  sJ g d} g d}t        j                  g d g d g d!g d"g      }t        | |      }d|cxk  rdk  sJ  J t        | ||      }d|cxk  rdk  sJ  J t        j                  g d#g d$g d"g d%g      }t        | |      }|dk  sJ t        | ||      }|dk  sJ y )&Nrb  r  r   rm   r   r   r   gffffff?g?r  r  ra  F)r`   r_   rO  rC   )r<   rC   r   r   r   rC   r   r   )r`   r_   r8  )r`   r_   r8  rO  r   r   r   r   rw  )r   rC   rC   rC   )r  r  r  r  )r<   r<   r<   r<   rE  )highrX  lowneutral)ffffff?r   r   r   )r   r   r   r   r  )r   r   r   r  r  )rH   r   r'   r   r   r   rY  )r`   y_true_stringr_   y_pred_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truer8  d2_score_stringd2_score_with_sample_weights              ra   test_d2_log_loss_scorerd  c  s   F;MXX#J#J#J#J4L4L	
	F ((#J#J#J#J#J#J	
	K !v>HVFeLN"&PUV)<<<Mv}}]3333 HH/0M%bq)--/-2C2C2EEK1%ab)--/-2C2C2EEK1 fMH #	N ##	 )<<<Mv}}]3333 XX#J#J#J#J#J#J	
	F !0HC'v>OfmmH5555 XX#J#J#J#J4L#J	
	F !0Ha<<'v>OfmmH5555  FXX#J#J#J#J#J#J	
	F !0Hq=='v>Oa F/MXXd|dD\D$<$NOF 0Hq=='v>Oa M"3m# '!+++ 0F(MXX		
F !0HC }MHCXX		
F !0Ha<< }MHa<<rc   c                      g d} g d}g d}t        j                  g dd      }t        | |||      }t        j                  g dd      }t        | |||      }d||z  z
  }t        | |||      }t	        ||       y	)
zCheck that d2_log_loss_score works when not all labels are present in y_true

    non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
    r<   r   r<   r   r   )r[  r   r   r   r   r   rC   )r8  rv   )r   r   r   rC   N)rH   tiler   r'   r2   )	r`   rv   r8  r_   log_loss_obsr]  log_loss_nullexpected_d2_scorer^  s	            ra   %test_d2_log_loss_score_missing_labelsrl    s    
 FF(MWWY'FFF-PVWL ''-0K=M L=88 mFH H/0rc   c                      g d} t        j                  g dd      }t        | |g d      }t        | |g d      }t        ||       y)zGCheck that d2_log_loss_score doesn't depend on the order of the labels.rf  r   rg  r   r   r-  N)rH   rh  r'   r2   )r`   r_   r^  d2_score_others       ra   "test_d2_log_loss_score_label_orderro    s=    FWWY'F 	BH&vviHNHn-rc   c                     g d} ddgddgddgg}d}t        j                  t        |      5  t        | |       d	d	d	       g d} ddgddgddgg}g d}d
}t        j                  t        |      5  t        | ||       d	d	d	       g d} g dg dg}d}t        j                  t        |      5  t        | |       d	d	d	       dg} ddgg}d}t        j                  t
        |      5  t        | |       d	d	d	       g d} ddgddgddgg}d}t        j                  t        |      5  t        | |       d	d	d	       g d} dg}ddgddgddgg}d}t        j                  t        |      5  t        | ||       d	d	d	       y	# 1 sw Y   QxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y	xY w)zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nz(number of classes in labels is differentr   )r   r   r   )r   r   r   rD  rC   zscore is not well-definedr   y_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r'   r  r   )r`   r_   errrv   s       ra   test_d2_log_loss_score_raisesrs  !  s    FCj3*sCj1F
/C	z	- *&&)*
 FCj3*sCj1FF
4C	z	- 9&&89 F/F
+C	z	- *&&)* SFCj\F
%C	,C	8 *&&)* FCj3*sCj1F
*C	z	- *&&)*
 FSFCj3*sCj1F
:C	z	- 9&&89 9O* *9 9* ** ** *9 9sG   F	2F2F#,F/-F;1G	FF #F,/F8;GGc                     g d} g d}g d}g d}g d}t        ||      }t        ||      }t        ||      }d||z  z
  }t        j                  |      |k(  sJ g d}t        ||      }|dk(  sJ t        ||d	
      }|dk(  sJ g d}t        |||       }|dk(  sJ t        ||| d	      }|dk(  sJ g d} g d}g d}g dg dg dg dg dg dg dg dg dg dg
}t        |||       }|dk(  sJ t        |||       }|dk(  sJ g dg dg dg dg dg dg dg dg dg dg
}t        |||       }|dkD  sJ t        |||       }|dkD  sJ y)zeTest that d2_brier_score gives expected outcomes in both the binary and
    multiclass settings.
    )r<   r<   r   rC   rC   rC   )r   rC   rC   r   r   rC   )r  r  r  r  r  r  )r   r   r   r   rm   r   )r   r   r   r   r   r   )r`   y_probarC   r   r  )r`   ru  r   )r   r   r   r   r   r   )r`   ru  r8  )r`   ru  r8  r   )
r<   rC   r   rC   rC   r<   rC   r   rC   r   )
r   r   r<   r<   r<   rC   rC   rC   rC   r   )
ddrv  ccrw  rw  bbrx  rx  rx  aa)r   r   rw  r  )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   r   r   N)r&   r   r   r   )	r8  r`   r\  ru  y_proba_refr^  brier_score_modelbrier_score_refd2_score_expecteds	            ra   test_d2_brier_scorer  R  s   
 'MF;M -G0KVW=H(H&fkJO-??=="&7777 -GVW=Hq==]GuUHq==
 -GwmH q==#	H q== 3M+FPM 	G wmH q==#H
 q==
 	G wmH c>>#H
 c>>rc   c                     g d} g d}g dg dg dg dg}t        | ||      }|dk(  sJ g d}t        | ||      }|t        j                  |      k(  sJ g dg dg dg dg}t        | ||      }t        j                  |      d	k(  sJ y
)zGTest that d2_brier_score gives expected outcomes when labels are passed)r   r<   r   r<   r   )r   r   r   )r`   ru  rv   r   r   r   r   N)r&   r   r   )r`   rv   ru  r^  new_d2_scoreneg_d2_scores         ra   test_d2_brier_score_with_labelsr    s    
 FF	G VWVLHq== F!PL6==2222 		G "PL==&",,,rc   z!y_true, y_pred, labels, error_msg)rC   r<   rC   r   r6  z7inferred from y_true is multiclass but should be binaryr  r  r  r  zpos_label is not specified)r   rC   r   r   rC   rC   r   z.variables with inconsistent numbers of samples)r   rC   r   rC   )g?r   r   r   z%y_prob contains values greater than 1)gr   r   r   z"y_prob contains values less than 0r   rq  rm  )r<   r   r   r<   )r   r   r   r   )r   r   r   r   z(Multioutput target data is not supportedr   )r   r   r   z"not belonging to the passed labelsrc  z*labels array needs to contain at least twoc                     t        j                  |       } t        j                  |      }t        j                  t        |      5  t        | ||       ddd       y# 1 sw Y   yxY w)zMTest that d2_brier_score raises the appropriate errors
    on invalid inputs.r   r   N)rH   asarrayr   r   r   r&   )r`   r_   rv   	error_msgs       ra   test_d2_brier_score_raisesr    sP    | ZZFZZF	z	3 6vvf56 6 6s   AA'c                      t        j                  dg      } t        j                  dg      }d}t        j                  t        |      5  t        | |       ddd       y# 1 sw Y   yxY w)zQTest that d2_brier_score emits a warning when there are less than
    two samplesrC   r   z+not well-defined with less than two samplesr   N)rH   r   r   r  r   r&   )r`   r_   warning_messages      ra   4test_d2_brier_score_warning_on_less_than_two_samplesr    sU     XXqc]FXXse_FCO	,O	D 'vv&' ' 's   
A  A)zarray_namespace, device, _c                 j   t        | |      }|j                  g d|      }|j                  g d|      }|j                  g d|      }t        d      5  t        |||      }t	        |      d   t	        |      d   k(  sJ t        |      t        |      k(  sJ 	 ddd       y# 1 sw Y   yxY w)	zTest that `confusion_matrix` works for all array types when `labels` are passed
    such that the inner boolean `need_index_conversion` evaluates to `True`.rF  r,   )r   r   r   Tarray_api_dispatchr   r   N)r1   r  r   r   r.   array_api_device)array_namespacer-   r~   xpr`   r_   rv   r  s           ra   test_confusion_matrix_array_apir  #  s     
ov	6BZZ	&Z1FZZ	&Z1FZZ	&Z1F	4	0 D!&&@V$Q'=+@+CCCC'+;F+CCCCD D Ds   AB))B2prob_metric
str_y_trueuse_sample_weightz$array_namespace, device_, dtype_namec                    t        ||      }|rt        j                  g d      nd}i }|r@t        j                  g d      }	t        j                  |	      }
d| j                  v r0d|d<   n*t        j                  g d      }	|j                  |	|      }
t        j                  g d	|
      }|j                  ||      } | |	|fd|i|}t        d      5   | |
|fd|i|}ddd       t        j                  |      k(  sJ |r-t        j                  g d      }	t        j                  |	      }
n*t        j                  g d      }	|j                  |	|      }
t        j                  g dg dg dg dg|
      }|j                  ||      } | |	|      }t        d      5   | |
|      }ddd       |t        j                  |      k(  sJ y# 1 sw Y   xY w# 1 sw Y   0xY w)zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
    and :func:`d2_log_loss_score` work correctly with the array API for binary
    and mutli-class inputs.
    r  Nr  brierr  r   rm  r,   )r   r   r   r   r   r8  Tr  )r   r   r   r   r  rz  )r   r   r   r   )r   r   r   r   )r   r   r   r   )r1   rH   r   r  __name__r   r   r   )r  r  r  r  device_
dtype_namer  r8  extra_kwargs	y_true_npy_true_xp_or_np	y_prob_np	y_prob_xpmetric_score_npmetric_score_xps                  ra   $test_probabilistic_metrics_array_apir  5  s    
ow	7B.?BHH\*TM LHH78	**Y/k*** ).L%HH\*	**Yw*?-Z@I

9W
5I!9,9=IO 
4	0 
%Y
6C
GS


 fmmO<<<< HH12	**Y/HH\*	**Yw*?    		
 I 

9W
5I!)Y7O	4	0 B%oyAB fmmO<<<<;
 
4B Bs   G-
G&G#&G/c                    t        ||      }|rt        j                  g d      nd}t        j                  g dg dg dg dg|      }|j                  ||      }t        j                  g d	g d
g dg dg|      }	|j                  |	|      }
 | ||	|      }t	        d      5   | ||
|      }ddd       t        j                  |      k(  sJ y# 1 sw Y   $xY w)zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
    and :func:`d2_log_loss_score` work correctly with the array API for
    multi-label inputs.
    r  Nr  rm  rl  )rC   rC   r   rC   r   r,   )r  gHzG?gq=
ףp?gQ?)r  gRQ?Q?gq=
ףp?)r  gQ?gQ?g)\(?)gQ?gףp=
?gp=
ף?g(\?rE  Tr  )r1   rH   r   r  r   r   r   )r  r  r  r  r  r  r8  r  	y_true_xpr  r  r  r  s                ra   /test_probabilistic_metrics_multilabel_array_apir  y  s     
ow	7B.?BHH\*TM		
 I 

9W
5I$$$$		
 I 

9W
5I!)YmTO	4	0 Y%i-XY fmmO<<<<Y Ys   +CC#)NF)r  r   	functoolsr   	itertoolsr   r   r   numpyrH   r   scipyr   r   scipy.spatial.distancer	   rD  scipy.statsr
   sklearnr   r   sklearn.baser   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   sklearn.metrics._classificationr%   r&   r'   sklearn.model_selectionr(   sklearn.preprocessingr)   r*   sklearn.treer+   sklearn.utils._array_apir-   r  r.   r/   sklearn.utils._mockingr0   sklearn.utils._testingr1   r2   r3   r4   r5   r6   sklearn.utils.extmathr7   sklearn.utils.fixesr8   r9   sklearn.utils.validationr:   rb   r   markparametrizerp  r   r   r   r   r   r   r   r   r   r   r   r   r  r  r!  r0  rB  rH  rV  r]  r`  rg  rk  rs  rw  r~  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r#  r%  r(  r/  r2  r5  r:  r<  r>  r@  rF  rN  rQ  r_  ra  rf  ri  rr  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
csr_matrixr  r  r  r  r  r  r  r  r  r
  r	  float16r  r  r  r  r"  r1  r4  r7  r9  r;  rC  rL  rN  thread_unsaferV  rd  rl  ro  rs  r  r  r  r  r  r  r  r   rc   ra   <module>r     s   	   2 2     8 ! ! ' ; 5     . 
 4 @ / 1  . > 7)(`>DB 61a*@A B& T{aVUOi=OP))(7D PQ R$ PQ*J R*JZ PQM RM,>6  BHH####	
	
 .=	
(9)(9  BHH#####	
 	
*;+*;
 =*	= OF .9.9%S : :%SPT< +%%;&) 
 #"((#56""((#56 N	
 #"((#56""((#56
B		
 #"((#56""((#56 N	
 #"((#56""((#56 H	
;$'P*Q'P*  #"((?3""((?3
<		
**"4 4-8M 9M;0 		S!E"C C 





 $S	!3'	$bff-S	!3'	'0		D	D$ $S	!3'	$c*	$c*	'0		D	D$>1 1a.9)aS1#J<8!$	' 9 :' )aS1#J<8!$	 9$2N RF5p c5\2U 3UB7(t $ST, U,0"(".& 	:;
QAB 
'(  886 "AB0 C0"%4%$%(%6 %F%,%.XI PQ% R%<6B(%DPJZ"J.+ 868:LM	2 N	2 PQ?U R?UD PQ= R=@ PQ+&&266266"23[	 R
[| !%$MN1a.9"5 : O &"5J $MN" O"& 1a.9 W : WF38e0P 1a.9:  :: z 61a*@A$ B$N 61a*@A$
 B$
N 61a*@A#( B#(L'>_D3= 			QC!qcA3/	0(;			QC!qcA3/	0,?			Iy)<	=|L** 8A6*"N8P4@A8I(X 2::rzz2::"FG
 H
 2::rzz2::"FG! H! 	I	aVaVaV,-	Y	95688)$
$N FIXX
&6 	34	34	34
:
: #&' C(($ $	1a&	8288QF+S1	1a&	8288QF+S1	1a&	8288QF+S1	1a&	8288QF+S1	XX HBFF3Karvv>O266:L7	V VSl1:..9b`F-@ '  E		
 ' (		
 " <		
  3		
 !0		
 3Z#sc3Z0,		
 <(!#786		
 o?F0		
 o?C8		
c7:v6w:v6'  "K"MDD $h@QR t}5,udm<*,U,W9= = 69=x $h@QR ,udm<*,U,W!= =!=rc   