B
    (b                 @   s*  d Z ddlmZ ddlmZ ddlmZ ddlmZm	Z	 ddl
Z
ddlZddlmZ ddlZddlZddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddlm"Z"m#Z# ddlm$Z$ ddlm%Z% ddlm&Z& ddlm'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddl/m1Z1 ddl2m3Z3 ddl4m5Z5 ddl4m6Z6 ddl4m7Z7 ddl8m9Z9 ddl8m:Z: dd l8m;Z; dd!l8m<Z< dd"l8m=Z= dd#l8m>Z> dd$l8m?Z? dd%l8m@Z@ dd&l8mAZA dd'l8mBZB dd(l8mCZC dd)l8mDZD dd*lEmFZF dd+lGmHZHmIZI dd,lJmKZK dd-lJmLZL dd.lMmNZN dd/lOmPZP dd0lQmRZR dd1lQmSZS dd2lQmTZT dd3lQmUZU dd4lQmVZV dd5lWmXZX dd6lYmZZZ dd7l[m\Z\m]Z] dd8l^m_Z_ G d9d: d:e`ZaG d;d< d<eHZbecd=d=gd>d=gd?d?gd@d?ggZdecd?d?d@d@gZedAdB ZfdCdD ZgdEdF ZhdGdH ZidIdJ ZjdKdL ZkdMdN ZldOdP Zme'dQdR ZndSdT ZodUdV ZpdWdX ZqdYdZ Zrd[d\ Zsd]d^ Ztd_d` Zudadb Zvdcdd Zwdedf Zxdgdh Zydidj Zzdkdl Z{dmdn Z|G dodp dpe0Z}e'dqdr Z~dsdt Zdudv Zdwdx Ze'dydz Zd{d| Zd}d~ Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Ze' dd Zdd Zdd Zdd Zdd Zdd Zdd ZG dd de0Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )zTest the search module    )	cStringIO)xrange)PY3_OR_LATER)chainproductN)GeneratorType)
sp_version)	_Iterable_Sized)assert_equal)assert_not_equal)assert_raises)assert_warns)assert_warns_message)assert_no_warnings)assert_raise_message)assert_falseassert_true)assert_array_equal)assert_array_almost_equal)assert_almost_equal)ignore_warnings)CheckingClassifierMockDataFrame)	bernoulliexponuniform)BaseEstimator)clone)NotFittedError)make_classification)
make_blobs)make_multilabel_classification)fit_grid_point)KFold)StratifiedKFold)StratifiedShuffleSplit)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)GridSearchCV)RandomizedSearchCV)ParameterGrid)ParameterSampler)FitFailedWarning)	LinearSVCSVC)DecisionTreeRegressor)DecisionTreeClassifier)KMeans)KernelDensity)f1_score)recall_score)accuracy_score)make_scorer)roc_auc_score)Imputer)Pipeline)RidgeSGDClassifier)OneTimeSplitterc               @   sb   e Zd ZdZdddZdd Zdd Zd	d
 Zdd ZeZ	eZ
eZdddZdddZdd ZdS )MockClassifierz8Dummy classifier to test the parameter search algorithmsr   c             C   s
   || _ d S )N)	foo_param)selfrA    rC   X/home/dcms/DCMS/lib/python3.7/site-packages/sklearn/model_selection/tests/test_search.py__init__L   s    zMockClassifier.__init__c             C   s$   t t|t|k t|| _| S )N)r   lennpuniqueclasses_)rB   XYrC   rC   rD   fitO   s    zMockClassifier.fitc             C   s
   |j d S )Nr   )shape)rB   TrC   rC   rD   predictT   s    zMockClassifier.predictc             C   s
   || j  S )N)rA   )rB   rJ   rC   rC   rD   	transformW   s    zMockClassifier.transformc             C   s
   || j  S )N)rA   )rB   rJ   rC   rC   rD   inverse_transformZ   s    z MockClassifier.inverse_transformNc             C   s   | j dkrd}nd}|S )N   g      ?g        )rA   )rB   rJ   rK   scorerC   rC   rD   rS   a   s    
zMockClassifier.scoreFc             C   s
   d| j iS )NrA   )rA   )rB   deeprC   rC   rD   
get_paramsh   s    zMockClassifier.get_paramsc             K   s   |d | _ | S )NrA   )rA   )rB   paramsrC   rC   rD   
set_paramsk   s    
zMockClassifier.set_params)r   )NN)F)__name__
__module____qualname____doc__rE   rL   rO   rP   rQ   predict_probapredict_log_probadecision_functionrS   rU   rW   rC   rC   rC   rD   r@   J   s   


r@   c               @   s   e Zd ZdZedd ZdS )LinearSVCNoScorez1An LinearSVC classifier that has no score method.c             C   s   t d S )N)AttributeError)rB   rC   rC   rD   rS   r   s    zLinearSVCNoScore.scoreN)rX   rY   rZ   r[   propertyrS   rC   rC   rC   rD   r_   p   s   r_   rR      c                s(   t t  fddtt D  d S )Nc                s   g | ]} | qS rC   rC   ).0i)gridrC   rD   
<listcomp>{   s    z3assert_grid_iter_equals_getitem.<locals>.<listcomp>)r   listrangerF   )rg   rC   )rg   rD   assert_grid_iter_equals_getitemz   s    rk   c           	      s\  ddddgi} t | }tt|t tt|t tt|d t| ddgdddgd	}t |}tt|d
 xHtdD ]<}t	dd |D }t|t	dd t
|d |d D  q|W t| t i  tt d tt i g t  tt fdd t dddgii ddgig}tt|d tt|ddiddii ddig t| d S )NfoorR   rd         Zhamspameggs)rl   bar   c             s   s"   | ]}t tt|  V  qd S )N)tupler   sorteditems)re   prC   rC   rD   	<genexpr>   s    z&test_parameter_grid.<locals>.<genexpr>c             s   s   | ]\}}d |d|fV  qdS )rq   rl   NrC   )re   xyrC   rC   rD   rw      s   rq   c                  s    d S )NrR   rC   rC   )emptyrC   rD   <lambda>       z%test_parameter_grid.<locals>.<lambda>C
   g      ?)r-   r   
isinstanceIterableSizedr   rF   rk   r   setr   ri   r   
IndexError)Zparams1Zgrid1Zparams2Zgrid2rf   ZpointsZ	has_emptyrC   )rz   rD   test_parameter_grid~   s2    "r   c              C   s   t  } t| ddddgidd}tj}t t_|tt |t_t|j	j
d t|jd jdddg |tt |t |t |t d|_tt|jtt d S )NrA   rR   rd   rm   )verboseZparam_foo_paramZsklearn)r@   r+   sysstdoutStringIOrL   rJ   ry   r   best_estimator_rA   r   cv_results_datarS   r\   r^   rP   scoringr   
ValueError)clfgrid_searchZ
old_stdoutrC   rC   rD   test_grid_search   s    


r   c          
   K   s   t ddd}t dgd dgd  }tddgd}| |d	dd
dgifdd
i|}ttd|j||t dd ttd|j||t dt 	dd |j||t dt 	dd d S )Nd   r~   r      rR   ro   rp   )expected_fit_paramsrA   rd   rm   cvz,Expected fit parameter(s) ['eggs'] not seen.)ro   z,Fit parameter spam has length 1; expected 4.)ro   rp   )
rG   arangereshapearrayr   r   AssertionErrorrL   oneszeros)klassZklass_kwargsrJ   ry   r   ZsearcherrC   rC   rD   -check_hyperparameter_searcher_with_fit_params   s     r   c               C   s   t t d S )N)r   r+   rC   rC   rC   rD    test_grid_search_with_fit_params   s    r   c               C   s   t tdd d S )NrR   )n_iter)r   r,   rC   rC   rC   rD   "test_random_search_with_fit_params   s    r   c              C   sn   t ddd} t dgd dgd  }tdgd}t|ddd	d
gidt did}tt|j	| | d S )Nr   r~   r   r   rR   ro   )r   rA   rd   rm   )
fit_params)
rG   r   r   r   r   r+   r   r   DeprecationWarningrL   )rJ   ry   r   r   rC   rC   rD   'test_grid_search_fit_params_deprecation   s    r   c           	   C   s   t ddd} t dgd dgd  }tdgd}t|ddd	d
gidt did}d}tt||j	| |t dd t|ddd	d
gidt did}t
td|j	| |t dd d S )Nr   r~   r   r   rR   ro   )r   rA   rd   rm   )r   zgIgnoring fit_params passed as a constructor argument in favor of keyword arguments to the "fit" method.)ro   zFit parameter spam has length 1)rG   r   r   r   r   r+   r   r   RuntimeWarningrL   r   r   )rJ   ry   r   r   Zexpected_warningrC   rC   rD   &test_grid_search_fit_params_two_places   s    r   c              C   s   t dd} tddd\}}dddg}tdd}t| d|id	d
}||| t|d|id	d
}||| t|j|j t|||||| t|d|i}tt	d|jdgg d S )Nr   )random_staterd   )r   centersg?rR   r~   r}   accuracy)r   z
no scoring)
r0   r!   r_   r+   rL   r   best_params_rS   r   	TypeError)r   rJ   ry   CsZclf_no_scorer   Zgrid_search_no_scorerC   rC   rD   test_grid_search_no_score   s    




r   c              C   s   t ddddd\} }tdd}ddgi}t||d d	| |}t||d
d	| |}tt |dd	| |}t||dd	| |}|| |}|| |}	|| |}
|| |}t|dk  t|	dk  t||	 t|	| t||
 d S )Nr   rd   g?r   )	n_samples	n_classesZflip_yr   )r   r}   g?)r   r   roc_aucg      ?)	r    r0   r+   rL   r_   rS   r   r   r   )rJ   ry   r   rg   Zsearch_no_scoringZsearch_accuracyZsearch_no_score_method_aucZ
search_aucZscore_no_scoringZscore_accuracyZscore_no_score_aucZ	score_aucrC   rC   rD   test_grid_search_score_method  s$    



r   c        
      C   s   t jd} tdddd\}}| ddd}tdd}ddgi}t tdt t	 g}x<|D ]4}t
|||d	}ttd
|j|| |j|||d q\W t t g}	x&|	D ]}t
|||d	}||| qW d S )Nr      rd   )r   r   r   rm   )r   r}   rR   )r   z*The 'groups' parameter should not be None.)groups)rG   randomZRandomStater    randintr0   r'   r(   r)   r*   r+   r   r   rL   r%   r&   )
rngrJ   ry   r   r   rg   Z	group_cvsr   gsZnon_group_cvsrC   rC   rD   test_grid_search_groups5  s"    



r   c              C   s0  t ddd} t dgd dgd  }dddgi}ttdd|ttdd|dd	g}i }x:|D ]2}x,d
D ]$}|j|d t|j	| |j
||< qpW qfW dddddg}xR|D ]J}d|}	tt|	|d j|}
t |
|d | st||d kstqW x,|d D ] }||krt|d j| qW d S )Nr   r~   r   r   rR   r}   rd   )r   )r   )TFwarn)return_train_scoresplit0_train_scoresplit1_train_scoresplit2_train_scoremean_train_scorestd_train_scorezYou are accessing a training score ({!r}), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=Truer   TF)rG   r   r   r   r+   r0   r,   rW   r   rL   r   formatr   FutureWarninggetallcloser   )rJ   ry   rg   
estimatorsresult	estimatorvalZ
train_keyskeymsgZtrain_scorerC   rC   rD   test_return_train_score_warnP  s.    





r   c              C   s   t ddd} t dgd dgd  }dddg}ttddd|i}|| | t|jj	|j	 tt
 d	d
dgi}|| | tt|d ttddd|i}tt|d ttddd|idd}|| | tt|d d S )Nr   r~   r   r   rR   g?)r   r}   alphag      ?g       @rI   F)refit)rG   r   r   r   r+   r0   rL   r   r   rI   r=   r   hasattr)rJ   ry   r   r   rC   rC   rD   test_classes__propertys  s     

r   c              C   sb   t  } t| ddgi}|tt tt|d t| ddgidd}|tt tt|d d S )NrA   rR   r   r   )r   )r@   r+   rL   rJ   ry   r   r   r,   )r   r   random_searchrC   rC   rD   test_trivial_cv_results_attr  s    r   c              C   s   t  } xd ddggD ]r}t| ddddgidd}|tt tt|d	 o\t|d
o\t|d x$dD ]}ttd| t	||t qfW qW x0dD ](}tt
dt| i |ddddjtt qW d S )Nr   	precisionrA   rR   rd   rm   F)r   r   best_index_r   )rO   r\   r]   rP   rQ   zHrefit=False. %s is available only after refitting on the best parameters) r   Trecallr   zIFor multi-metric scoring, the parameter refit must be set to a scorer key)accprec)r   r   )r@   r+   rL   rJ   ry   r   r   r   r   getattrr   )r   r   r   fn_namer   rC   rC   rD   test_no_refit  s"    


r   c              C   sF   t dddd\} }t }t|dddgi}tt|j| d d | d S )	N   r   r   )r   
n_featuresr   r}   g?g      ?   )r    r0   r+   r   r   rL   )X_y_r   r   rC   rC   rD   test_grid_search_error  s    r   c              C   sn   t dddd\} }dgdgdgd}t }t||}|| | tdddd}|| | t|j|jj d S )	Nr   r   r   )r   r   r   g      ?rbfg?)r}   kernelgamma)r    r1   r+   rL   r   Z
dual_coef_r   )r   r   
param_dictr   r   rC   rC   rD   test_grid_search_one_grid_point  s    
r   c              C   sV   t  } d }tr$t| dtddi}nt| dtddi}|tt t|j	j
d d S )NrA   rR   rn   rd   )r@   r   r+   rj   r   rL   rJ   ry   r   r   rA   )r   r   rC   rC   rD   /test_grid_search_when_param_grid_includes_range  s    r   c              C   s   ddi} t  }ttdt||  dg i} t  }ttdt||  ddi} t  }ttdt||  dtdddi} t  }ttt||  d S )	Nr}   g      ?zYParameter values for parameter (C) need to be a sequence(but not a string) or np.ndarray.zCParameter values for parameter (C) need to be a non-empty sequence.z1,2,3rr   rm   rd   )r1   r   r   r+   rG   r   r   r   )r   r   rC   rC   rD   test_grid_search_bad_param_grid  s*    


r   c              C   s   t dddd\} }t }t|dddgi}|| d d |d d  || dd  }|jj}t| } t }t|dddgi}|| d d 	 |d d  || dd  }|jj}t
t||kd	k t|| d S )
Nr   r   r   )r   r   r   r}   g?g      ?r   g?)r    r0   r+   rL   rO   r   r}   sp
csr_matrixZtocoor   rG   meanr   )r   r   r   r   y_predr}   y_pred2C2rC   rC   rD   test_grid_search_sparse  s    
 r   c              C   sL  t dddd\} }t }t|dddgidd	}|| d d
 |d d
  || d
d  }|jj}t| } t }t|dddgidd	}|| d d
 |d d
  || d
d  }|jj}t	|| t
|| dd }t|dd}	t|dddgi|	d	}|| d d
 |d d
  || d
d  }
|jj}t
|| t	||
 d S )Nr   r   r   )r   r   r   r}   g?g      ?f1)r   r   c             S   s   t | | S )N)r6   )Zy_true_Zy_pred_rC   rC   rD   f1_loss$  s    z0test_grid_search_sparse_scoring.<locals>.f1_lossF)Zgreater_is_better)r    r0   r+   rL   rO   r   r}   r   r   r   r   r9   )r   r   r   r   r   r}   r   r   r   ZF1LossZy_pred3ZC3rC   rC   rD   test_grid_search_sparse_scoring  s,    



r   c        	      C   s   t dddd\} }t| d d | d d j}|d d }tdd}t|dd	d
gi}||| t|jdk t| dd  | d d j}|dd  }|	|}tt
||kdk tt|j| | d S )Nr   r   r   )r   r   r   r   precomputed)r   r}   g?g      ?)r    rG   dotrN   r1   r+   rL   r   best_score_rO   r   r   r   tolist)	r   r   K_trainy_trainr   r   ZK_testZy_testr   rC   rC   rD   #test_grid_search_precomputed_kernel0  s    

r   c              C   sD   t d} t d}tdd}t|dddgi}tt|j| | d S )N)r~      )r~   r   )r   r}   g?g      ?)rG   r   r   r1   r+   r   r   rL   )r   r   r   r   rC   rC   rD   3test_grid_search_precomputed_kernel_error_nonsquareL  s
    


r   c               @   s*   e Zd ZdZd	ddZdd Zdd ZdS )
BrokenClassifierz*Broken classifier that cannot be fit twiceNc             C   s
   || _ d S )N)	parameter)rB   r   rC   rC   rD   rE   Y  s    zBrokenClassifier.__init__c             C   s   t t| d  d| _d S )Nhas_been_fit_T)r   r   r   )rB   rJ   ry   rC   rC   rD   rL   \  s    zBrokenClassifier.fitc             C   s   t |jd S )Nr   )rG   r   rM   )rB   rJ   rC   rC   rD   rO   `  s    zBrokenClassifier.predict)N)rX   rY   rZ   r[   rE   rL   rO   rC   rC   rC   rD   r   V  s   
r   c              C   sX   t ddd} t dgd dgd  }tt dddgigddd	}|| | d S )
Nr   r~   r   r   rR   r   r   T)r   r   )rG   r   r   r   r+   r   rL   )rJ   ry   r   rC   rC   rD   
test_refitd  s
    
r   c              C   s   t ddddd} t dddd}d	d
 }dd
 }t||d}t|ddddgi}|| |tt t	t
|d d S )Ni,  r~   r   rm   rd   i        c             S   s   | j dd  dkS )NrR   )r   rm   rd   )rM   )rx   rC   rC   rD   r{   u  r|   z$test_gridsearch_nd.<locals>.<lambda>c             S   s   | j dd  dkS )NrR   )r   r   )rM   )rx   rC   rC   rD   r{   v  r|   )check_Xcheck_yrA   rR   r   )rG   r   r   r   r+   rL   rS   rJ   ry   r   r   )ZX_4dZy_3dr   r   r   r   rC   rC   rD   test_gridsearch_ndq  s    r   c              C   s   t ddd} t dgd dgd  }tdd d}td	d
}t|dddd	gi|d}||  |	| | t
t|d d S )Nr   r~   r   r   rR   c             S   s
   t | tS )N)r   ri   )rx   rC   rC   rD   r{     r|   z test_X_as_list.<locals>.<lambda>)r   rm   )n_splitsrA   rd   )r   r   )rG   r   r   r   r   r$   r+   rL   r   rS   r   r   )rJ   ry   r   r   r   rC   rC   rD   test_X_as_list}  s    
r  c              C   s   t ddd} t dgd dgd  }tdd d}td	d
}t|dddd	gi|d}|| | 	| | t
t|d d S )Nr   r~   r   r   rR   c             S   s
   t | tS )N)r   ri   )rx   rC   rC   rD   r{     r|   z test_y_as_list.<locals>.<lambda>)r   rm   )r  rA   rd   )r   r   )rG   r   r   r   r   r$   r+   rL   r   rS   r   r   )rJ   ry   r   r   r   rC   rC   rD   test_y_as_list  s    
r  c                 s   t t fg} y"ddlm}m} | ||f W n tk
r@   Y nX tddd}t	dgd dgd  }x| D ]~\  || }} fdd}fd	d
}t
||d}	t|	ddddgi}
|
|||| |
| tt|
d qtW d S )Nr   )Series	DataFramer   r~   r   rR   c                s
   t |  S )N)r   )rx   )InputFeatureTyperC   rD   check_df  s    z#test_pandas_input.<locals>.check_dfc                s
   t |  S )N)r   )rx   )
TargetTyperC   rD   check_series  s    z'test_pandas_input.<locals>.check_series)r   r   rA   rd   rm   r   )r   Zpandasr  r  appendImportErrorrG   r   r   r   r   r+   rL   rS   rO   r   r   )typesr  r  rJ   ry   ZX_dfZy_serr  r	  r   r   rC   )r  r  rD   test_pandas_input  s"    

r  c              C   s   t dd\} }tdd}ddg}xDdD ]<}t|tdddgd	||d
}|| | t|jd d q&W t|tdddgd	dd}|| | t|jd d t|tdddgd	d}||  t|jd d d S )Nr   )r   adjusted_rand_scorefowlkes_mallows_score)r  r  rd   rm   rn   )
n_clusters)
param_gridr   r   r  )r  r   )r  )r!   r4   r+   dictrL   r   r   )rJ   ry   kmr   r   r   rC   rC   rD   test_unsupervised_grid_search  s    



r  c              C   sr   dd } t ddddgddgddggd\}}tt tdddgd| d	}|| t|jd
 d t|jd d S )Nc             S   s   | j dkrdS dS )Ng?*   r   )	bandwidth)r   rJ   rC   rC   rD   custom_scoring  s    z2test_gridsearch_no_predict.<locals>.custom_scoringg?rR   r   )cluster_stdr   r   g{Gz?)r  )r  r   r  r  )r!   r+   r5   r  rL   r   r   r   )r  rJ   _searchrC   rC   rD   test_gridsearch_no_predict  s    
r  c              C   s  ddgt ddd} t| ddd}dd	 |D }tt|d x<|D ]4}t|d
 dk td|d   kopdkn   qDW ddddddddddddgi} t| ddd}tdd	 |D dd	 |D  tdkr
dt ddi} t| ddd}tdd	 |D dd	 |D  d S )Nr   linearr   rR   )r   r}   r~   )param_distributionsr   r   c             S   s   g | ]}|qS rC   rC   )re   rx   rC   rC   rD   rh     s    z&test_param_sampler.<locals>.<listcomp>r   )r   r  r}   rd   rm   rn   r   rr   r      	   c             S   s   g | ]}|qS rC   rC   )re   rx   rC   rC   rD   rh     s    )r      c             S   s   g | ]}|qS rC   rC   )re   rx   rC   rC   rD   rh     s    )r   r.   r   rF   r   r   )r  samplersamplessamplerC   rC   rD   test_param_sampler  s$    

$


r$  c                s   | j  tt fdd|D  tt fdd|D  tt fdd|D  tt fdd|D  | jr~| j ndg}x$|D ]}t d|  jt	j
k qW d S )Nc             3   s    | ]}t  | tjjV  qd S )N)r   rG   maMaskedArray)re   param)
cv_resultsrC   rD   rw     s   z/check_cv_results_array_types.<locals>.<genexpr>c             3   s   | ]} | j tkV  qd S )N)dtypeobject)re   r   )r(  rC   rD   rw     s    c             3   s    | ]}t  | tjjV  qd S )N)r   rG   r%  r&  )re   r   )r(  rC   rD   rw     s   c             3   s(   | ] }| d s | jtjkV  qdS )ZrankN)
startswithr)  rG   float64)re   r   )r(  rC   rD   rw     s   rS   zrank_test_%s)r   r   allr   anymultimetric_scorer_keysr)  rG   int32)r  
param_keys
score_keysZscorer_keysr   rC   )r(  rD   check_cv_results_array_types  s    
r5  c                sB   t t  t|| d  tt fdd|| D  d S )N)rV   c             3   s   | ]} | j fkV  qd S )N)rM   )re   r   )r(  n_candrC   rD   rw     s   z(check_cv_results_keys.<locals>.<genexpr>)r   rt   r1  r   r-  )r(  r3  r4  r6  rC   )r(  r6  rD   check_cv_results_keys  s    r7  c                s   | j rttdt| d n| j tt fddt| j	D j
} d } d }t|}ttt| d}tt|| xTt|D ]H}t|| j||  t|| j||d d f  t|| j||  qW d S )Nznot available for multi-metricZgrid_scores_c                s   g | ]} d |  qS )zsplit%d_test_scorerC   )re   rf   )r(  rC   rD   rh     s   z<check_cv_results_grid_scores_consistency.<locals>.<listcomp>mean_test_scorerV   )r/  r   r`   r   r   rG   Zvstackri   rj   	n_splits_rN   rF   r   r   r   
parametersr   Zcv_validation_scoresZmean_validation_score)r  Z
res_scoresZ	res_means
res_paramsr6  Zgrid_scoresrf   rC   )r(  rD   (check_cv_results_grid_scores_consistency  s$    

r<  c        
         sD  t dddd\} }d}d}tdgdd	gd
dgdtdgddgdg}d}d}|}xdD ]}tt |||d}	|	| | t||	j |	j tt	 d dk t fdd|D  t fdd|D  t
|	|| t ||| |	j t|	jd }tt	 fddt|D  tt	 fddt|D  t|	 qVW d S )N2   rn   r  )r   r   r   rm   rr   r   rR   r~   g?)r   r}   r   polyrd   )r   degree)param_Cparam_degreeparam_gammaparam_kernel)r8  r   rank_test_scoresplit0_test_scoresplit1_test_scoresplit2_test_scorer   r   r   std_test_scorer   mean_fit_timestd_fit_timemean_score_timestd_score_time)FT)r   iidr  rD  c             3   s&   | ]}|d k	rt  | dkV  qdS )rD  r   N)r-  )re   k)r(  rC   rD   rw   C  s    z.test_grid_search_cv_results.<locals>.<genexpr>c             3   s.   | ]&}d |kr|dk	rt  | dkV  qdS )timerD  rR   N)r-  )re   rN  )r(  rC   rD   rw   E  s    rV   c             3   sJ   | ]B} d  | dkr d j | o@ d j | o@ d j |  V  qdS )rC  r  r@  rB  rA  N)mask)re   rf   )r(  rC   rD   rw   N  s   c             3   sL   | ]D} d  | dkr d j |  oB d j |  oB d j | V  qdS )rC  r   r@  rB  rA  N)rP  )re   rf   )r(  rC   rD   rw   S  s   )r    r  r+   r1   rL   r   rM  r   r   r-  r5  r7  rF   rj   r<  )
rJ   ry   r  Zn_grid_pointsrV   r3  r4  n_candidatesrM  r  rC   )r(  rD   test_grid_search_cv_results'  s4    	
rR  c              C   s   t dddd\} }d}d}ttddtd	dd
}d}d}|}xdD ]x}tt ||||d}	|	| | t||	j |	j}
t	|	|| t
|
||| tt|
d jpt|
d j t|	 qDW d S )Nr=  rn   r  )r   r   r   rm      r~   )Zscaleg?)r}   r   )r@  rB  )r8  r   rD  rE  rF  rG  r   r   r   rH  r   rI  rJ  rK  rL  )FT)r   r   rM  r  r@  rB  )r    r  r   r,   r1   rL   r   rM  r   r5  r7  r   r.  rP  r<  )rJ   ry   r  n_search_iterrV   r3  r4  r6  rM  r  r(  rC   rC   rD   test_random_search_cv_results[  s$    	


rU  c                 sL  t ddgddgddgddggddddd\} }tj| jd tjd}d|t|dkd d d d < d|t|dkd d d d < || g| |gg}tt d	dd
gi|d}tt dd	dd
gi|d}xH||fD ]:  	| | t
 j tt fddt jD }tt fddt jD } jd d } jd d }	tt fddt jD } jd d }
 jd d }t jd d d t|ddg t|ddg d}td|d d  d|d d   }t|| t|	| t|
d t|d qW tt d	dd
gi|dd}tt dd	dd
gi|dd}x||fD ]  	| | t j tt fddt jD } jd d } jd d }	tt fddt jD } jd d }
 jd d }t jd d d t|ddg t|t| t|	t| t|
d t|d qVW d S )Nr   rR   g?FP   )r   r   r  shuffler   )r)  rd   r}   r~   )r  r   )r   r  r   c             3   s    | ]} j d |  d V  qdS )zsplit%d_test_scorer   N)r   )re   s_i)r  rC   rD   rw     s   z(test_search_iid_param.<locals>.<genexpr>c             3   s    | ]} j d |  d V  qdS )zsplit%d_train_scorer   N)r   )re   rX  )r  rC   rD   rw     s   r8  rH  c             3   s    | ]} j d |  d V  qdS )zsplit%d_train_scorer   N)r   )re   rX  )r  rC   rD   rw     s   r   r   r@  gUUUUUU?g      ?g      ?g      ?)r  r   rM  )r   r  r   rM  c             3   s    | ]} j d |  d V  qdS )zsplit%d_test_scorer   N)r   )re   s)r  rC   rD   rw     s   c             3   s    | ]} j d |  d V  qdS )zsplit%d_train_scorer   N)r   )re   rY  )r  rC   rD   rw     s   )r!   rG   r   rM   boolwherer+   r1   r,   rL   r   rM  r   ri   rj   r9  r   r   r   sqrtr   r   r   Zstd)rJ   ry   rP  r   r   r   Ztest_cv_scoresZtrain_cv_scoresZ	test_meanZtest_stdZ
train_meanZ	train_stdZexpected_test_meanZexpected_test_stdrC   )r  rD   test_search_iid_param}  sr      










r]  c           
   C   s   t dddd\} }d}tdgddgd	dgd
tdgddgdg}xxdD ]p}g }xXttttdddfD ]<}tt ||||dd}|| | t|j	| |
| qhW t|d|i qFW d S )Nr=  rn   r  )r   r   r   rm   r   rR   r~   g?)r   r}   r   r>  rd   )r   r?  )FT)r   r   r   r   F)r   rM  r  r   r   rM  )r    r  r9   r8   r7   r+   r1   rL   r   rM  r
  *compare_cv_results_multimetric_with_single)rJ   ry   r  rV   rM  Zgrid_searchesr   r   rC   rC   rD   'test_grid_search_cv_results_multimetric  s     


r_  c              C   s   t dddd\} }d}d}d}ttdd	tjd
dddd}xdD ]}xdD ]}g }x\dD ]T}|rvt|trrdn|}tddd}	t|	||||||dd}
|
| | |	|
 q\W t
|d|i |rNt|d |d	 | qNW qDW d S )Nr=  rn   r  )r   r   r   rm   rS  )r   r   irR   r   g?)base)r}   r   )TF))r   r   r   r   r   T)probabilityr   )r   r   rM  r  r   r   r   rM  )r    r  rG   Zlogspacer   rs   r1   r,   rL   r
  r^  )compare_refit_methods_when_refit_with_acc)rJ   ry   r  rT  r   rV   rM  r   Zrandom_searchesr   r   rC   rC   rD   )test_random_search_cv_results_multimetric  s0     



rd  c                s   t | j| t| j tt| jd | j dd |j D }|	dd |j D  tt
 fdddD  tj dd   D d	d | D  d
S )zCompare multi-metric cv_results with the ensemble of multiple
    single metric cv_results from single metric grid/random search)r   r   c             S   s    i | ]\}}|t d d|qS )z_score$Z	_accuracy)resub)re   rN  vrC   rC   rD   
<dictcomp>  s   z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>c             S   s    i | ]\}}|t d d|qS )z_score$Z_recall)re  rf  )re   rN  rg  rC   rC   rD   rh    s   c             3   s    | ]}t  | d kV  qdS )rR   N)rG   r-  )re   rN  )cv_results_multirC   rD   rw   #  s    z=compare_cv_results_multimetric_with_single.<locals>.<genexpr>)rK  rL  rI  rJ  c             S   s    i | ]\}}| d s||qS )_time)endswith)re   rN  rg  rC   rC   rD   rh  *  s    c             S   s    i | ]\}}| d s||qS )rj  )rk  )re   rN  rg  rC   rC   rD   rh  ,  s    N)r   rM  r   r/  r   rt   r0  r   ru   updater-  rG   testing)search_multi
search_accZ
search_recrM  Zcv_results_acc_recrC   )ri  rD   r^    s    


r^  c             C   s   |rt | jd n
t| j t |j| tdddd\}}x*dD ]"}tt| ||t||| q@W t| ||||| x"dD ]}t t| |t|| qW dS )	zDCompare refit multi-metric search methods with single metric methodsr   r   rn   r  )r   r   r   )rO   r\   r]   )r   r   r   N)r   r   r   r!   r   r   rS   )rn  ro  r   rJ   ry   methodr   rC   rC   rD   rc  0  s    


rc  c              C   s   t ddd\} }ddddgi}tt |d}tt d	|d
}x||fD ]}|| | |j}t|d d |d d  t|d d |d d  tt	|d d |d d  tt	|d d |d d  t|jd ddd	g qFW d S )Nr=  r  )r   r   r}   rR   gjt?gMbP?)r  rm   )r   r  r8  r   r   rd   rD  )
r!   r+   r1   r,   rL   r   r   r   rG   r   )rJ   ry   r  r   r   r  r(  rC   rC   rD   (test_search_cv_results_rank_tie_breakingA  s"    rq  c              C   s   dgdgdgdgdggdddddg } }t  t f}ddd gi}tdd}x6|D ].}t|||d	| |}t|jd
 dd g qPW d S )NrR   rd   rm   rn   r   r   r   )r   )r   Zparam_random_state)r2   r3   r$   r+   rL   r   r   )rJ   ry   r   est_parametersr   estr   rC   rC   rD   !test_search_cv_results_none_param^  s    (


rt  c              C   s  t dd} dgdgdgdgg}ddddg}t| dddgiddd}t| dddgidddd	}x||fD ]}||| x<d
D ]4}tt|j| dk tt|j| dk  q|W xPdD ]H}t|j| d dk t|j| d dk tt|j| dk  qW qfW d S )Nr   )r   rR   rd   rm   rn   r}   )r   error_score)r   ru  r   )rI  rJ  )rK  rL  g        )r0   r+   r,   rL   r   rG   r-  r   )svcrJ   ry   r   rsr  r   rC   rC   rD   test_search_cv_timingj  s    


rx  c           	      s|  d} t dd}tddd\}}dddg}xLd	D ]B}t|d
|i|| d||j}t| }dtdd t| D  }t	t
t|| t| d}	j} xt|D ]\ }
|j|
d tt fddt| D }xt|	||D ]x\}\}}||| ||  |dkr8t|| ||| }n&|dkr^||| }t|| |}t|||  qW qW q0W d S )Nrm   r   )r   rd   )r   r   g?rR   r~   )r   r   r}   )r   r   )r8  rD  c             s   s   | ]}d | V  qdS )zsplit%d_test_scoreNrC   )re   Zcv_irC   rC   rD   rw     s   z9test_grid_search_correct_score_results.<locals>.<genexpr>)r  )r}   c             3   s    | ]}j d |    V  qdS )zsplit%d_test_scoreN)r   )re   rY  )candidate_ir   rC   rD   rw     s   r   r   )r0   r!   r+   rL   r   ri   r1  rs   rj   r   r-  rG   Zin1dr%   r9  	enumeraterW   r   splitr6   rO   r^   r:   r   )r  r   rJ   ry   r   rS   r(  Zresult_keysZexpected_keysr   r}   Z	cv_scoresrf   traintestcorrect_scoredecrC   )ry  r   rD   &test_grid_search_correct_score_results  s6    




r  c              C   s  t dd\} }tdd}tdd}tt}xddiddiddifD ]}x|| |D ]\}}t| |t|||||dd\}}	}
t|jf |}|	| | ||  ||| | || }t
|| t||	 t|
|j qTW qBW ttd	t| |||||d
|idd d S )Nr   )r   r}   g?g{Gz?gMbP?F)r   z:scoring value should either be a callable, string or None.rS   T)r    r%   r0   r9   r8   r{  r#   r   rW   rL   r   r   sizer   r   )rJ   ry   r   rv  ZscorerrV   r|  r}  Zthis_scoresZthis_paramsZn_test_samplesrs  Zexpected_scorerC   rC   rD   test_fit_grid_point  s&    



r  c              C   s   t  } t| ddddgidd}|tt tt|}t|	t|	t t
| ddddgiddd}|tt tt|}t|	t|	t d S )NrA   rR   rd   rm   T)r   )r   r   )r@   r+   rL   rJ   ry   pickleloadsdumpsr   rO   r,   )r   r   Zgrid_search_pickledr   Zrandom_search_pickledrC   rC   rD   test_pickle  s    


r  c              C   s  t ddd\} }dddddgi}tdd	}tdd	tdd	g}x|D ]}t|||d
}|| | |jd }xtt|D ]v}|j	f ||  x`t
|| |D ]L\}	\}
}|| |
 ||
  || | || }t||jd|	  |  qW qzW qDW x|D ]}t|||dd}|| | |jd }xtt|D ]z}|j	f ||  xbt
|| |D ]N\}	\}
}|| |
 ||
  || | || }t||jd|	  |  q\W q6W qW d S )NTr   )Zreturn_indicatorr   Z	max_depthrR   rd   rm   rn   )r   )r   rV   zsplit%d_test_score)r   r   )r"   r$   r2   r3   r+   rL   r   rj   rF   rW   rz  r{  rS   r   r,   )rJ   ry   rr  r   r   rs  r   r;  cand_irf   r|  r}  r~  r   rC   rC   rD   &test_grid_search_with_multioutput_data  s@    


 


r  c              C   sR   t ddd} dddddg}tdd}t|i dd	| |}tt|d
 d S )Nr   r   rb   r   rR   F)rb  rd   )r   r\   )rG   r   r   r1   r+   rL   r   r   )rJ   ry   r   r   rC   rC   rD   test_predict_proba_disabled  s
    
r  c              C   sz   t jdt jddd} t j| dd d f< dddddg}tdtd	d
dfdt fg}t|ddddgidd	| | d S )Nr   )r)  r   rb   rd   r   rR   Zimputerr   NaN)ZstrategyZmissing_values
classifierZclassifier__foo_paramrm   )r   )
rG   r   r,  r   nanr<   r;   r@   r+   rL   )rJ   ry   rv   rC   rC   rD   test_grid_search_allows_nans  s    r  c               @   s0   e Zd ZdZdZd
ddZdddZdd	 ZdS )FailingClassifierz,Classifier that raises a ValueError on fit()rd   Nc             C   s
   || _ d S )N)r   )rB   r   rC   rC   rD   rE     s    zFailingClassifier.__init__c             C   s   | j tjkrtdd S )Nz%Failing classifier failed as required)r   r  FAILING_PARAMETERr   )rB   rJ   ry   rC   rC   rD   rL     s    zFailingClassifier.fitc             C   s   t |jd S )Nr   )rG   r   rM   )rB   rJ   rC   rC   rD   rO      s    zFailingClassifier.predict)N)N)rX   rY   rZ   r[   r  rE   rL   rO   rC   rC   rC   rD   r    s
   

r  c                 s   t dddd\} }t }t|ddddgigdd	d
dttj| | tjd }fdd t fddt	|D st
t|ddddgigdd	tddttj| | tjd }t fddt	|D st
d S )Nr   r~   r   )r   r   r   r   rR   rd   r   Fg        )r   r   ru  rV   c                s$   t t fddtjD S )Nc             3   s    | ]} j d |   V  qdS )zsplit%d_test_scoreN)r   )re   rY  )r   rf   rC   rD   rw   9  s   zOtest_grid_search_failing_classifier.<locals>.get_cand_scores.<locals>.<genexpr>)rG   r   ri   rj   r9  )rf   )r   )rf   rD   get_cand_scores8  s    z<test_grid_search_failing_classifier.<locals>.get_cand_scoresc             3   s4   | ],}j d  | tjkrt |dkV  qdS )param_parameterg        N)r   r  r  rG   r-  )re   r  )r  r   rC   rD   rw   <  s   z6test_grid_search_failing_classifier.<locals>.<genexpr>r  c             3   s6   | ].}j d  | tjkrtt |V  qdS )r  N)r   r  r  rG   r-  isnan)re   r  )r  r   rC   rD   rw   E  s   )r    r  r+   r   r/   rL   rF   r   r-  rj   r   float)rJ   ry   r   rQ  rC   )r  r   rD   #test_grid_search_failing_classifier$  s    
r  c              C   sJ   t dddd\} }t }t|ddddgigdd	d
d}tt|j| | d S )Nr   r~   r   )r   r   r   r   rR   rd   r   Fraise)r   r   ru  )r    r  r+   r   r   rL   )rJ   ry   r   r   rC   rC   rD   )test_grid_search_failing_classifier_raiseK  s
    
r  c              C   s  ddgdddgd} t | dd}ttt| t | d	d}t|}tt|d	 xt| D ]}t||k qXW td
td
td
d} t | ddd}t|}tt|d dd |D }ttt	|d t
ddddgd}t |dd}t|}tt|d d S )Nr   rR   abc)firstsecondr   )r   rr   r~   )r  r  r  c   r  )r   r   c             S   s&   g | ]}d |d |d |d f qS )z	a%db%dc%dr  r  r  rC   )re   rv   rC   rC   rD   rh   k  s   z7test_parameters_sampler_replacement.<locals>.<listcomp>g      ?)r.   r   r   ri   r   rF   r-   r   rj   r   r   )rV   r!  r"  valuesZhashable_samplesZparams_distributionrC   rC   rD   #test_parameters_sampler_replacementZ  s&    r  c              C   s   ddgi} t ddd}ddddddg}ttdd	d
| d}tt|d ||| || |	| dd	gi} ttdd	d
| d}tt|d ||| tt|d d S )Nlosslog   rr   rb   r   rR   gMbP?Zhinge)Ztolr  )r   r  r\   )
rG   r   r   r+   r>   r   r   rL   r\   r]   )r  rJ   ry   r   rC   rC   rD   #test_stochastic_gradient_loss_paramv  s    



r  c              C   sR   t ddd} ddddddg}tdd}t|dddgid	d
}|| | d S )Nrr   rb   r   rR   )r   r}   g?g?F)r  r   )rG   r   r   r0   r+   rL   )rJ   ry   r   r   rC   rC   rD   %test_search_train_scores_set_to_false  s    
r  c        	         s   d} d}t | dd\}}ttdddddd	git|| d
d  || ttdddddd	git|dd}||| ttt|ddd||t	 ttdddddd	git|ddd||d}||| ttdddddd	git|dddd}||| dd }t
j||j||j t
jdd  j D dd |j D  ttdddddddgit|ddd  || xldD ]di }x4tdD ](t fddtdD |< qW t|d |d  t|d |d  qW d S )Nr   r   r   )r   r   )r   r}   g?g?g333333?)r  r   )r  r   )r  T)r  rW  r   c             S   s   xdD ]}|  | qW | S )N)rI  rJ  rK  rL  )pop)r(  r   rC   rC   rD   _pop_time_keys  s    
z>test_grid_search_cv_splits_consistency.<locals>._pop_time_keysc             S   s    i | ]\}}| d s||qS )rj  )rk  )re   rN  rg  rC   rC   rD   rh    s    z:test_grid_search_cv_splits_consistency.<locals>.<dictcomp>c             S   s    i | ]\}}| d s||qS )rj  )rk  )re   rN  rg  rC   rC   rD   rh    s    )r  rW  )r|  r}  rn   c             3   s$   | ]} j d |f   V  qdS )zsplit%d_%s_scoreN)r   )re   rY  )r   param_i
score_typerC   rD   rw     s   z9test_grid_search_cv_splits_consistency.<locals>.<genexpr>rR   rd   rm   )r    r+   r0   r?   rL   r$   r   r   r{  r   rG   rm  r   r   ru   rj   ri   r   )	r   r  rJ   ry   Zgs2Zgs3Zgs4r  Zper_param_scoresrC   )r   r  r  rD   &test_grid_search_cv_splits_consistency  sX    






r  c              C   sH   t  } t| ddddgidd}|tt ||t}tt| d S )NrA   rR   rd   rm   )r   )r@   r+   rL   rJ   ry   rQ   rP   r   )r   r   ZX_round_triprC   rC   rD   +test_transform_inverse_transform_round_trip  s
    r  )r[   Zsklearn.externals.six.movesr   r   r   Z sklearn.externals.joblib._compatr   	itertoolsr   r   r  r   r  r   re  numpyrG   Zscipy.sparsesparser   Zsklearn.utils.fixesr   r	   r   r
   r   Zsklearn.utils.testingr   r   r   r   r   r   r   r   r   r   r   r   r   Zsklearn.utils.mockingr   r   Zscipy.statsr   r   r   Zsklearn.baser   r   Zsklearn.exceptionsr   Zsklearn.datasetsr    r!   r"   Zsklearn.model_selectionr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   Z#sklearn.model_selection._validationr/   Zsklearn.svmr0   r1   Zsklearn.treer2   r3   Zsklearn.clusterr4   Zsklearn.neighborsr5   Zsklearn.metricsr6   r7   r8   r9   r:   Zsklearn.preprocessingr;   Zsklearn.pipeliner<   Zsklearn.linear_modelr=   r>   Z$sklearn.model_selection.tests.commonr?   r*  r@   r_   r   rJ   ry   rk   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r$  r5  r7  r<  rR  rU  r]  r_  rd  r^  rc  rq  rt  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rC   rC   rC   rD   <module>   s   &"$#	#
4"^ #-	'
N