B
    (bH                 @   s  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	Z
ddlmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl m"Z" ddl m#Z# ddl m$Z$ ddl m%Z% ddl m&Z& ddl m'Z' ddl m(Z( ddl m)Z) ddl m*Z* ddl m+Z+ dd l m,Z, dd!l m-Z- dd"l.m/Z/ dd#l0m1Z1 dd$l0m2Z2 dd%l0m3Z3 dd&l0m4Z4 dd'l5m6Z6 dd(l5m7Z7 dd)l5m8Z8 dd*l5m9Z9 dd+l5m:Z: dd,l5m;Z; dd-l5m<Z< dd.l=m>Z> dd/l?m@Z@mAZAmBZB dd0l?mCZCmDZD dd1lEmFZF dd2lGmHZH dd3lImJZJ dd4lKmLZL dd5lKmMZM dd6lNmOZO dd7lPmQZR dd8lSmTZT dd9lSmUZU dd:lVmWZW dd;lXmYZY dd<l0mZZZ dd=l0m[Z[ dd>l\m]Z] dd?l m^Z^ ye_ W n e`k
r>   dZ_Y nX G d@dA dAeTZaG dBdC dCeaZbG dDdE dEeTZcG dFdG dGecZdG dHdI dIeeZfe
gdJZheehZie
jdddKdKdLdLdMdMdNdNg
Zke
jdKdKdKdLdLdLdMdMdMdMg
Zlee
mdOZndPdQ ZodRdS ZpdTdU ZqdVdW ZrdXdY ZsdZd[ Ztd\d] Zud^d_ Zvd`da Zwdbdc Zxddde Zydfdg Zzdhdi Z{djdk Z|dldm Z}dndo Z~dpdq Zdrds Zdtdu Zdvdw Zdxdy Zdzd{ Zd|d} Zd~d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )zTest the validation module    )divisionN)sleep)
coo_matrix
csr_matrix)assert_true)assert_false)assert_equal)assert_almost_equal)assert_raises)assert_raise_message)assert_warns)assert_warns_message)assert_no_warnings)assert_raises_regex)assert_greater)assert_less)assert_array_almost_equal)assert_array_equal)CheckingClassifierMockDataFrame)cross_val_score)cross_val_predict)cross_validate)permutation_test_score)KFold)StratifiedKFold)LeaveOneOut)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)learning_curve)validation_curve)_check_is_permutation)make_regression)load_boston)	load_iris)load_digits)explained_variance_score)make_scorer)accuracy_score)confusion_matrix)precision_recall_fscore_support)precision_score)r2_score)check_scoring)RidgeLogisticRegressionSGDClassifier)PassiveAggressiveClassifierRidgeClassifier)KNeighborsClassifier)SVC)KMeans)Imputer)LabelEncoder)Pipeline)	cStringIO)BaseEstimator)clone)OneVsRestClassifier)shuffle)make_classification)make_multilabel_classification)OneTimeSplitter)GridSearchCVc               @   s<   e Zd ZdZdd ZdddZdd Zdd	d
Zdd ZdS )MockImprovingEstimatorz+Dummy classifier to test the learning curvec             C   s   || _ d| _d | _d S )Nr   )n_max_train_sizestrain_sizesX_subset)selfrE    rI   \/home/dcms/DCMS/lib/python3.7/site-packages/sklearn/model_selection/tests/test_validation.py__init__W   s    zMockImprovingEstimator.__init__Nc             C   s   || _ |jd | _| S )Nr   )rG   shaperF   )rH   rG   y_subsetrI   rI   rJ   fit\   s    zMockImprovingEstimator.fitc             C   s   t d S )N)NotImplementedError)rH   XrI   rI   rJ   predicta   s    zMockImprovingEstimator.predictc             C   s2   |  |rdt| j| j  S t| j| j S d S )Ng       @)_is_training_datafloatrF   rE   )rH   rP   YrI   rI   rJ   scored   s    
zMockImprovingEstimator.scorec             C   s
   || j kS )N)rG   )rH   rP   rI   rI   rJ   rR   k   s    z(MockImprovingEstimator._is_training_data)N)NN)	__name__
__module____qualname____doc__rK   rN   rQ   rU   rR   rI   rI   rI   rJ   rD   U   s   

rD   c                   s2   e Zd ZdZ fddZdd Zd	ddZ  ZS )
!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitc                s   t t| | d | _d S )N)superrZ   rK   x)rH   rE   )	__class__rI   rJ   rK   q   s    z*MockIncrementalImprovingEstimator.__init__c             C   s
   | j |kS )N)r\   )rH   rP   rI   rI   rJ   rR   v   s    z3MockIncrementalImprovingEstimator._is_training_dataNc             K   s"   |  j |jd 7  _ |d | _d S )Nr   )rF   rL   r\   )rH   rP   yparamsrI   rI   rJ   partial_fity   s    z-MockIncrementalImprovingEstimator.partial_fit)N)rV   rW   rX   rY   rK   rR   r`   __classcell__rI   rI   )r]   rJ   rZ   o   s   rZ   c               @   s<   e Zd ZdZdddZdd Zdd Zdd
dZdd Zd	S )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c             C   s   d | _ || _d S )N)rG   param)rH   rd   rI   rI   rJ   rK      s    z#MockEstimatorWithParameter.__init__c             C   s   || _ |jd | _| S )Nr   )rG   rL   rF   )rH   rG   rM   rI   rI   rJ   rN      s    zMockEstimatorWithParameter.fitc             C   s   t d S )N)rO   )rH   rP   rI   rI   rJ   rQ      s    z"MockEstimatorWithParameter.predictNc             C   s   |  |r| jS d| j S )N   )rR   rd   )rH   rP   r^   rI   rI   rJ   rU      s    z MockEstimatorWithParameter.scorec             C   s
   || j kS )N)rG   )rH   rP   rI   rI   rJ   rR      s    z,MockEstimatorWithParameter._is_training_data)rc   )NN)	rV   rW   rX   rY   rK   rN   rQ   rU   rR   rI   rI   rI   rJ   rb   ~   s   

rb   c                   s(   e Zd ZdZ fddZdd Z  ZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                s,   t t| dd d| _tt| | ||S )Nfit_called_zfit is called the second timeT)r   hasattrrg   r[   typerN   )rH   rG   rM   )r]   rI   rJ   rN      s
    z)MockEstimatorWithSingleFitCallAllowed.fitc             C   s   t d S )N)rO   )rH   rP   rI   rI   rJ   rQ      s    z-MockEstimatorWithSingleFitCallAllowed.predict)rV   rW   rX   rY   rN   rQ   ra   rI   rI   )r]   rJ   rf      s   rf   c            	   @   s@   e Zd ZdZdddZdddZd	d
 ZdddZdddZdS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc             C   s   || _ || _d S )N)aallow_nd)rH   rk   rl   rI   rI   rJ   rK      s    zMockClassifier.__init__Nc          	   C   sF  || _ || _|	| _|
dk	r"|
|  | jr8|t|d}|jdkrP| jsPtd|dk	rt|j	d |j	d kd
|j	d |j	d  |dk	rt|j	d tttkd
|j	d ttt |dk	rd}t|j	d |j	d k|
|j	d |j	d  |dk	rBd	}t|j	tj	k|
|j	d |j	d
 tj	d tj	d
  | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})re   )	dummy_int	dummy_str	dummy_objrl   reshapelenndim
ValueErrorr   rL   formatnpuniquer^   P_sparse)rH   rP   rT   sample_weightclass_priorsparse_sample_weightsparse_paramro   rp   rq   callbackfmtrI   rI   rJ   rN      s<    

zMockClassifier.fitc             C   s&   | j r|t|d}|d d df S )Nrm   r   )rl   rr   rs   )rH   TrI   rI   rJ   rQ      s    zMockClassifier.predictc             C   s   ddt | j  S )Ng      ?re   )rw   absrk   )rH   rP   rT   rI   rI   rJ   rU      s    zMockClassifier.scorec             C   s   | j | jdS )N)rk   rl   )rk   rl   )rH   deeprI   rI   rJ   
get_params   s    zMockClassifier.get_params)r   F)	NNNNNNNNN)NN)F)	rV   rW   rX   rY   rK   rN   rQ   rU   r   rI   rI   rI   rJ   rj      s   
  
*
rj   )
      re   r   rn         c              C   sP  t  } xtddD ]}|| _t| tt}t|| tt t	ttd d d g}t| t
|}t|| t
| t| t
t}t|| t
t t| t
|}t|| t
| qW dd }t|d} t| t t }t|d} t| tt }ttt| ttdd	 td d d d tjf }t d
d} t| |t}t dd} ttt| |t d S )Nir   rm   c             S   s
   t | tS )N)
isinstancelist)r\   rI   rI   rJ   <lambda>      z&test_cross_val_score.<locals>.<lambda>)check_X)check_yZsklearn)scoringT)rl   F)rj   rangerk   r   rP   y2r   rU   rw   column_stackX_sparser   tolistr
   ru   newaxis)clfrk   scoresmultioutput_y
list_checkX_3drI   rI   rJ   test_cross_val_score   s.    



r   c           
   C   sj  t dd\} }t }d}tt|t|| |ttttfd tt|t|| |ttfd tt|d t|| |dd tt|d t|| |dd tt|t|| |ttggd d	}ttd
t|| |t d tt|t|| |dd tt	}ttdt|| ||d ttdt|| |d|id tt
}ttdtt | ||d ttdtt | |d|id ttdtt | |dd d S )Nr   )random_statez.*must be unique strings.*)r   zEmpty list.*rI   zDuplicate.*)f1_micror   zG.*should either be.*string or callable.*for single.*.*dict.*for multi.*zAn empty dictr   zJClassification metrics can't handle a mix of binary and continuous targetsZfooz!scoring must return a number, gotz#'mse' is not a valid scoring value.Zmse)r@   rj   r   ru   r   r)   r-   r*   dictr,   r+   r6   )rP   r^   	estimatorZerror_message_regexpZmulticlass_scorerZmultivalued_scorerrI   rI   rJ   )test_cross_validate_invalid_scoring_param  sR    



r   c           	   c   s~  t dd} tddd\}}tdd}tddd\}}tddd}x6|||f|||ffD ]\}}}	t|	d	}
t|	d
}g }g }g }g }x| ||D ]\}}t||| || }	|	|
|	|| ||  |	||	|| ||  |	|
|	|| ||  |	||	|| ||  qW t
|}t
|}t
|}t
|}||||f}t|	|||fV  t|	|||fV  qXW d S )Nr   )n_splits   r   )	n_samplesr   )r   linear)kernelr   neg_mean_squared_errorr2)r   r$   r0   r@   r6   r/   splitr=   rN   appendrw   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)cvZX_regZy_regregZX_clfZy_clfr   rP   r^   estZ
mse_scorerZ	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scorestraintestr   rI   rI   rJ   test_cross_validateZ  s4    

$





r   c              C   s   t dd\} }t }i }x"dD ]}tt|| ||d||< qW dd}tt||d jd}t	||d d srt
d|d	 kst
d S )
Nr   )r   )FTwarn)return_train_scorezYou are accessing a training score ({!r}), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=Truetrain_scorer   TF)r@   rj   r   r   rv   r   FutureWarninggetrw   ZallcloseAssertionError)rP   r^   r   resultvalmsgr   rI   rI   rJ   +test_cross_validate_return_train_score_warn  s    

r   c          	   C   s   |\}}}}xdD ]\}}	|r@t | ||ddd}
t|
d | nt | ||dddd}
tt|
t tt|
|	 t|
d | |rt | ||dd	gd}t|d
 | nt | ||dd	gdd}tt|t tt||	 t|d | qW d S )N))Tr   )Frn   r   r   )r   r   r   F)r   r   r   Z
test_scorer   train_r2test_r2)r   r   r   r   r   r   rs   )r   rP   r^   r   r   r   r   r   r   Zdict_lenZmse_scores_dictZr2_scores_dictrI   rI   rJ   r     s.    

r   c          
   C   s  |\}}}}dt tddf}td}	|	td}
xhdD ]^}xV|D ]L}|rt| ||d|d}t|d | t|d	 | nt| ||d|d
d}tt|t t	t|
 |r|
n|	 t|d | t|d | t|d tjkstt|d tjks
tt|d tjks"tt|d tjks:tt|d dksRtt|d dk sjtt|d dkstt|d dk sJtqJW q<W d S )N)r   r   r   )r   test_neg_mean_squared_errorfit_time
score_time)r   train_neg_mean_squared_error)TFr   )r   r   r   r   F)r   r   r   r   r   r   r   r   r   )r)   r.   setunionr   r   r   r   r   r   keysri   rw   Zndarrayr   all)r   rP   r^   r   r   r   r   r   Zall_scoringZkeys_sans_trainZkeys_with_trainr   r   Z
cv_resultsrI   rI   rJ   r     sD    


r   c           
   C   sp   t dddd\} }tdd}t tdt t g}x8|D ]0}ttdt|| ||d ttdt	|| ||d q8W d S )	N   r   r   )r   	n_classesr   r   )r   z*The 'groups' parameter should not be None.)r   rP   r^   r   )
r@   r6   r   r   r   r    r   ru   r   r   )rP   r^   r   Z	group_cvsr   rI   rI   rJ   #test_cross_val_score_predict_groups  s    

r   c                 s   t t fg} y"ddlm}m} | ||f W n tk
r@   Y nX xR| D ]J\  tt }} fdd}fdd}t||d}t	||| qHW d S )Nr   )Series	DataFramec                s
   t |  S )N)r   )r\   )InputFeatureTyperI   rJ   r   	  r   z-test_cross_val_score_pandas.<locals>.<lambda>c                s
   t |  S )N)r   )r\   )
TargetTyperI   rJ   r   
  r   )r   r   )
r   pandasr   r   r   ImportErrorrP   r   r   r   )typesr   r   X_dfy_sercheck_dfcheck_seriesr   rI   )r   r   rJ   test_cross_val_score_pandas  s    
r   c              C   s   t dd} t }|j|j }}td}t| |||d}td}g }x^|||D ]N\}}tjt	|tj
d}	tjt	|tj
d}
d|	|< d|
|< |||f qPW t| |||d}t|| d S )Nr   )r   r   )r   )dtypere   )r6   r&   datatargetr   r   r   rw   zerosrs   boolr   r   )svmirisrP   r^   kfoldZscores_indicesZcv_masksr   r   Z
mask_trainZ	mask_testZscores_masksrI   rI   rJ   test_cross_val_score_mask  s    
r   c              C   s   t dd} t }|j|j }}t||j}t| ||}t dd} t| ||}t|| t dd d} t| ||}t|| t dd} t	t
t| || t	t
t| | | d S )NZprecomputed)r   r   c             S   s   t | |jS )N)rw   dotr   )r\   r^   rI   rI   rJ   r   .  r   z2test_cross_val_score_precomputed.<locals>.<lambda>)r6   r&   r   r   rw   r   r   r   r   r
   ru   r   )r   r   rP   r^   Zlinear_kernelZscore_precomputedZscore_linearZscore_callablerI   rI   rJ    test_cross_val_score_precomputed"  s    




r   c           	      s   t  } tjd }ttt}ttdgtdgtdgffdd}tt	d}d dt
  fdd	}t|t|| || |d
}t| tt|d d S )Nr   re   )r   re   )rL   r   *   Z42c                s(   t | j  t | j t | j d S )N)r   ro   rp   rq   )r   )	DUMMY_INT	DUMMY_OBJ	DUMMY_STRrI   rJ   assert_fit_paramsI  s    z:test_cross_val_score_fit_params.<locals>.assert_fit_params)rz   r{   r|   r}   ro   rp   rq   r~   )
fit_params)rj   rP   rL   rs   rw   rx   r^   r   r   eyeobjectonesr   )r   r   r   ZW_sparsery   r   r   rI   )r   r   r   rJ   test_cross_val_score_fit_params<  s&    
$r   c           	      sl   t  } g   fdd}tjdd  t|}t| tt|dd}W d Q R X t|dddg t dksht	d S )Nc                s     | |f dS )Ng      ?)r   )Zy_testZ	y_predict)_score_func_argsrI   rJ   
score_func`  s    z3test_cross_val_score_score_func.<locals>.score_funcT)recordrn   )r   r   g      ?)
rj   warningscatch_warningsr)   r   rP   r^   r   rs   r   )r   r   r   rU   rI   )r   rJ   test_cross_val_score_score_func\  s    r   c              C   s"   G dd d} t tt|  t d S )Nc               @   s   e Zd ZdS )z4test_cross_val_score_errors.<locals>.BrokenEstimatorN)rV   rW   rX   rI   rI   rI   rJ   BrokenEstimatorm  s   r   )r
   	TypeErrorr   rP   )r   rI   rI   rJ   test_cross_val_score_errorsl  s    r   c              C   s   t  } tdd}t|| j| jdd}t|dddddgd t|| j| jddd	}t|dddddgd t|| j| jd
dd	}t|dddddgd d S )Nr   )r   r   )r   g
ףp=
?g      ?r   accuracy)r   r   Zf1_weighted)r&   r6   r   r   r   r   )r   r   r   Z	zo_scoresZ	f1_scoresrI   rI   rJ   3test_cross_val_score_with_score_func_classifications  s    


r   c        	      C   s   t ddddd\} }t }t|| |dd}t|dddd	d
gd t|| |ddd}t|dddd	d
gd t|| |ddd}tdddddg}t||d tt}t|| |d|d}t|dddd	d
gd d S )Nr   r   r   r   )r   
n_featuresn_informativer   )r   gGz?g
ףp=
?gGz?gq=
ףp?r   r   )r   r   r   )r   r   g(\؇gzGIgGz&qg\(qg)\G)r$   r0   r   r   rw   r   r)   r(   )	rP   r^   r   r   Z	r2_scoresZneg_mse_scoresZexpected_neg_mser   Z	ev_scoresrI   rI   rJ   /test_cross_val_score_with_score_func_regression  s    
r   c           
   C   sv  t  } | j}t|}| j}tdd}td}t|||d|dd\}}}t|d t|dd	 t|||d|dt	
|jd
d\}	}
}t|	|k t||k tdd}td}t|||d|dt	
|jd
d\}	}
}t|	|k t||k dd }t|}t|||d||d
d\}}
}t|dd t|dd t	t	t|d}t|||d|dd\}}}t|d t|d d S )Nr   )r   r   r   r   )n_permutationsr   r   g?g        re   r   )r   r   r   groupsr   c             S   s"   | |k  | |k   | jd  S )Nr   )sumrL   )Zy_trueZy_predrI   rI   rJ   custom_score  s    z,test_permutation_score.<locals>.custom_scored   )r   r   r   r   g(\?g{Gz?rn   g      ?g?)r&   r   r   r   r6   r   r   r   r	   rw   r   sizer   r)   modarangers   r   )r   rP   r   r^   r   r   rU   r   ZpvalueZscore_group_Zpvalue_groupZ
svm_sparseZ	cv_sparser   ZscorerrI   rI   rJ   test_permutation_score  sB    




r  c              C   sv   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	d
dfdt	 fg}t
|| |dd d S )N   )r   r   rm   r   r   re   imputermeanNaN)strategymissing_values
classifierr   )r   )rw   r  float64rr   nanrepeatrL   r:   r8   rj   r   )rP   r^   prI   rI   rJ   &test_permutation_test_score_allow_nans  s    r  c              C   sv   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	d
dfdt	 fg}t
|| |dd d S )Nr  )r   r   rm   r   r   re   r  r  r	  )r
  r  r  r   )r   )rw   r  r  rr   r  r  rL   r:   r8   rj   r   )rP   r^   r  rI   rI   rJ   test_cross_val_score_allow_nans  s    r  c        	      C   s0  t ddgddgddgddgddgddgddgddgddgddgg
} t ddgddgddgddgddgddgddgddgddgddgg
}tdd	}ttd
d}ttdd}ttdd}t|| ||dd}t|| ||dd}t|| ||dd}t|dddddg t|dddddg t|dddddg d S )Nr   r   rn   r   re   rm   )Zn_neighborsmicro)ZaveragemacroZsamplesr   )r   r   g      ?g      ?gUUUUUU?g      ?)rw   r   r5   r)   r-   r   r	   )	rP   r^   r   Zscoring_microZscoring_macroZscoring_samplesZscore_microZscore_macroZscore_samplesrI   rI   rJ   test_cross_val_score_multilabel  s    "$"$
r  c           
   C   s  t  } | j| j }}t }t }t|}x>|||D ].\}}||| ||  |	|| ||< q8W t
||||d}t|| t
|||}tt|t| t }t
||||d}tt|t| | }	|	|	t|	k9 }	t|	}	t
||	|}tt|t| t
t |}tt|t| G dd d}
ttt
||||
 d tdd\}}d}tt|t
t ||dtdd	 d S )
N)r   c               @   s   e Zd ZdddZdS )z%test_cross_val_predict.<locals>.BadCVNc          	   s   s>   x8t dD ],}tddddgtddddd	gfV  q
W d S )
Nr   r   re   r   rn   r            )r   rw   r   )rH   rP   r^   r   irI   rI   rJ   r     s    z+test_cross_val_predict.<locals>.BadCV.split)NN)rV   rW   rX   r   rI   rI   rI   rJ   BadCV  s   r  T)
return_X_yzNumber of classes in training fold (2) does not match total number of classes (3). Results may not be appropriate for your use case.predict_probar   )methodr   )r%   r   r   r   r0   rw   Z
zeros_liker   rN   rQ   r   r   r   rs   r   copyZmedianr   r7   r
   ru   r&   r   RuntimeWarningr1   )ZbostonrP   r^   r   r   Zpreds2r   r   predsZXspr  Zwarning_messagerI   rI   rJ   test_cross_val_predict  s:    

r#  c           
   C   s  t dddd\} }tt | |dd}t|jd tdd	\} }tt | |dd}t|jd
 | d d } |d d }ttdtt | |dt	dd t
dd	\} }tddd}t|| |dd}t|jd t|}| | ||  } }ttdt|| |t	dddd d S )Nr   2   r   )r   r   r   decision_function)r  )r$  T)r  )   rn   r   zOnly 1 class/es in training fold, this is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified folds)r  r   r   Zovo)r   Zdecision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*rn   )r   )r   r  )r@   r   r1   r   rL   r&   r   ru   r4   r   r'   r6   rw   Zargsortr   )rP   r^   r"  r   indrI   rI   rJ   .test_cross_val_predict_decision_function_shape.  s4    



r)  c              C   s`   t dddd\} }tt | |dd}t|jd tdd	\} }tt | |dd}t|jd
 d S )Nr   r$  r   )r   r   r   r  )r  )r$  r   T)r  )r&  rn   )r@   r   r1   r   rL   r&   )rP   r^   r"  rI   rI   rJ   *test_cross_val_predict_predict_proba_shape\  s    

r*  c              C   s`   t dddd\} }tt | |dd}t|jd tdd	\} }tt | |dd}t|jd
 d S )Nr   r$  r   )r   r   r   predict_log_proba)r  )r$  r   T)r  )r&  rn   )r@   r   r1   r   rL   r&   )rP   r^   r"  rI   rI   rJ   .test_cross_val_predict_predict_log_proba_shapej  s    

r,  c        
      C   sV  t  } | j| j }}t|}t||d d d g}tddd}t|||}t|j	d t|||}t|j	d t|||}t
|j	d t|||}t
|j	d dd }t|d	}t|| | }t|d
}t||| }tt | | dd}tt || dd}|d d d d tjf }dd }	t|	d	}t|||}t
|j	d d S )Nrm   Fr   )Zfit_interceptr   )r&  )r&  r   c             S   s
   t | tS )N)r   r   )r\   rI   rI   rJ   r     r   z4test_cross_val_predict_input_types.<locals>.<lambda>)r   )r   r%  )r  c             S   s
   | j dkS )Nrn   )rt   )r\   rI   rI   rJ   r     r   )r&   r   r   r   rw   r   r0   r   r   rL   r   r   r   r1   r   )
r   rP   r^   r   r   r   predictionsr   r   Zcheck_3drI   rI   rJ   "test_cross_val_predict_input_typesx  s6    


r.  c                 s   t t fg} y"ddlm}m} | ||f W n tk
r@   Y nX xR| D ]J\  tt }} fdd}fdd}t||d}t	||| qHW d S )Nr   )r   r   c                s
   t |  S )N)r   )r\   )r   rI   rJ   r     r   z/test_cross_val_predict_pandas.<locals>.<lambda>c                s
   t |  S )N)r   )r\   )r   rI   rJ   r     r   )r   r   )
r   r   r   r   r   r   rP   r   r   r   )r   r   r   r   r   r   r   r   rI   )r   r   rJ   test_cross_val_predict_pandas  s    
r/  c              C   sV   t  } | j| j }}t }dtt|jd i}t||||d}t	|t
d d S )Nr|   r   )r   rn   )r&   r   r   rj   r   rw   r   rL   r   r   r   )r   rP   r^   r   r   rk   rI   rI   rJ   &test_cross_val_score_sparse_fit_params  s    r0  c              C   s  d} d}t | ddddddd\}}t| |d |  }xJdD ]@}tjdd	0}t|||t|d
tddd|d\}}}	W d Q R X t|dkrt	d|d j
 t|jd t|	jd t|tddd t|jddtddd t|	jddtddd tjdd	2}t|||t|| dtddd|d\}
}}W d Q R X t|dkrlt	d|d j
 t|| t||	 q>W d S )Nr   rn   re   r   r   )r   r   r   n_redundantr   n_clusters_per_classr   )FTT)r   )r   g?g      ?r   )r   rF   r?   zUnexpected warning: %r)r   rn   r   )axisgffffff?)r   r   )r@   rD   r   r   r!   r   rw   linspacers   RuntimeErrormessager   rL   r   r   r  rB   )r   r   rP   r^   r   shuffle_trainwrF   train_scorestest_scoresZtrain_sizes2Ztrain_scores2Ztest_scores2rI   rI   rJ   test_learning_curve  s>    

r;  c           
   C   s   t dddddddd\} }td}t|| d dtdd	d
d\}}}t|tddd
 t|jddtdd	d
 t|jddtdd	d
 d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   r   rn   g?g      ?r   )r^   r   rF   )r3  gffffff?)r@   rD   r!   rw   r4  r   r   r  )rP   r  r   rF   r9  r:  rI   rI   rJ    test_learning_curve_unsupervised  s     r<  c           	   C   sz   t dddddddd\} }td}tj}t t_zt|| |ddd\}}}W d tj }tj  |t_X d	|ksvtd S )
Nr   re   r   r   )r   r   r   r1  r   r2  r   r   rn   )r   verbosez[learning_curve])	r@   rD   sysstdoutStringIOr!   getvaluecloser   )rP   r^   r   Z
old_stdoutrF   r9  r:  outrI   rI   rJ   test_learning_curve_verbose  s    

rD  c           	   C   s:   t dddddddd\} }td}ttt|| |dd d S )Nr   re   r   )r   r   r   r1  r   r2  r   T)exploit_incremental_learning)r@   rD   r
   ru   r!   )rP   r^   r   rI   rI   rJ   5test_learning_curve_incremental_learning_not_possible  s    rF  c              C   s   t dddddddd\} }td}x~dD ]v}t|| |dd	td
dd|d\}}}t|tddd t|jddtddd t|jddtd
dd q(W d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   r   )FTrn   Tg?g      ?r   )r   rE  rF   r?   )r3  gffffff?)r@   rZ   r!   rw   r4  r   r   r  )rP   r^   r   r7  rF   r9  r:  rI   rI   rJ   (test_learning_curve_incremental_learning  s    

rG  c              C   s   t dddddddd\} }td}t|| d ddtd	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   r   rn   Tg?g      ?r   )r^   r   rE  rF   )r3  gffffff?)r@   rZ   r!   rw   r4  r   r   r  )rP   r  r   rF   r9  r:  rI   rI   rJ   5test_learning_curve_incremental_learning_unsupervised  s    
rH  c        
   	   C   s   t dddddddd\} }tddd}tdd d	d
}t|| ||ddd\}}}t|| |d|d	d\}}}	t|| t|jdd|jdd t|jdd|	jdd d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   g?g      ?r   F)max_itertolr?   rn   T)rF   r   rE  )r   rF   rE  )r3  )r@   rw   r4  r3   r!   r   r   r  )
rP   r^   rF   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchrI   rI   rJ   <test_learning_curve_batch_and_incremental_learning_are_equal-  s"    

rQ  c           	   C   s   t dddddddd\} }td}ttt|| |dddgd ttt|| |dd	d
gd ttt|| |dddgd ttt|| |dddgd ttt|| |dddgd d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   r   rn   )r   rF   g        g      ?g?g?   )r@   rD   r
   ru   r!   )rP   r^   r   rI   rI   rJ   0test_learning_curve_n_sample_range_out_of_boundsE  s    rS  c              C   sZ   t dddddddd\} }td}ttt|| |dtdddd\}}}t|ddg d S )	Nrn   re   r   r   )r   r   r   r1  r   r2  r   gQ?g      ?)r   rF   )r@   rD   r   r!  r!   rw   r4  r   )rP   r^   r   rF   r  rI   rI   rJ   1test_learning_curve_remove_duplicate_sample_sizesV  s    rT  c           
   C   s   t dddddddd\} }td}tdd}t|| ||td	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   re   r   r   )r   r   r   r1  r   r2  r   r   rn   )r   g?g      ?r   )r   rF   )r3  gffffff?)	r@   rD   r   r!   rw   r4  r   r   r  )rP   r^   r   r   rF   r9  r:  rI   rI   rJ   (test_learning_curve_with_boolean_indicesa  s    
 rU  c              C   s  t ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg} t dddddddddddddddg}t dddddddddddddddg}tdd dd}tdd}t|| ||dt ddd|ddd	\}}}t|jddt dddg t|jddt dddg tt	t|| ||dt ddd|d 	 t|| ||dt ddd|dddd!
\}}	}
t|	jdd|jdd t|
jdd|jdd d S )"Nre   r   rn   r   r   r  r  r                             r   	   r   F)rI  rJ  r?   )r   g333333?g      ?T)r   n_jobsrF   r   r?   r   )r3  g      ?gIq?g      ?)r   r`  rF   r   )r   r`  rF   r   r?   r   rE  )
rw   r   r3   r   r!   r4  r   r  r
   ru   )rP   r^   r   r   r   rN  rO  rP  rK  rL  rM  rI   rI   rJ    test_learning_curve_with_shufflep  s2    .$((
ra  c           	   C   s   t dddddddd\} }tddd}tjdd }tt | |d|dd	\}}W d Q R X t|dkrxtd
|d j	 t
|jdd| t
|jddd|  d S )Nr   re   r   )r   r   r   r1  r   r2  r   r   T)r   rd   )
param_nameparam_ranger   zUnexpected warning: %r)r3  )r@   rw   r4  r   r   r"   rb   rs   r5  r6  r   r  )rP   r^   rc  r8  r9  r:  rI   rI   rJ   test_validation_curve  s    
rd  c           	   C   sF   t dddddddd\} }tddd}tt | |d|dd\}}d S )Nr   re   r   )r   r   r   r1  r   r2  r   r   rd   )rb  rc  r   )r@   rw   r4  r"   rf   )rP   r^   rc  r  rI   rI   rJ   %test_validation_curve_clone_estimator  s    re  c           
   C   s   d} d}t ddd\}}ttddd||dddd	d	gt|| d
d}ttt|dd d f d  ttddd||dddd	d	gt|ddd}ttt|dd d f d  ttddd||dddd	d	gt|dd}tt	|t	| d S )Nr   r   r   )r   r   r   )r   r   Cg?g?)r   r   )r   )r   r   re   rn   r   T)r   r?   )r   )
r@   r"   r6   rB   r   rw   Zvsplithstackr   r   )r   r   rP   r^   Zscores1Zscores2Zscores3rI   rI   rJ   +test_validation_curve_cv_splits_consistency  s$    rh  c              C   sv   t jd} t d}| | tt|d ttt |dd d|d< tt|d ttt 	|dfd d S )Nr   r      )
rw   randomZRandomStater  r?   r   r#   r   deleterg  )rngr  rI   rI   rJ   test_check_is_permutation  s    

rm  c              C   sj   t dddddd\} }t| }t|}ttdd}t|| |dd	}t|||dd	}| }t|| d S )
Nr   re   FT)r   Zn_labelsZallow_unlabeledZreturn_indicatorr   r   )r   r   )r   )rA   r   r>   r6   r   Ztoarrayr   )rP   r^   r   Zy_sparseZclassifr"  Zpreds_sparserI   rI   rJ   (test_cross_val_predict_sparse_prediction  s    rn  c             C   sP  t  }|j|j }}t||dd\}}tt|}t }dddg}x|D ]}t| |||d}tt|t| t	
t||g}	t| |}
x<|||D ],\}}| || ||  |
|| |	|< qW t| ||||d}t|	| t| ||d ||d}t|| t| ||d	 ||d}t|| t| ||d
||d}t|| qJW d S )Nr   )r   r%  r  r+  )r  )r  r   re   r   str)r&   r   r   r?   rs   r   r   r   r   rw   r   getattrr   rN   r   r   Zastype)r   r   rP   r^   classesr   methodsr  r-  expected_predictionsfuncr   r   Zpredictions_y1Zpredictions_y2Zpredictions_ystrrI   rI   rJ   #check_cross_val_predict_with_method  s4    






ru  c               C   s   t t  d S )N)ru  r1   rI   rI   rI   rJ   "test_cross_val_predict_with_method  s    rv  c              C   s   t ddd} t|  d S )Nlogr   )Zlossr   )r2   ru  )r   rI   rI   rJ   &test_cross_val_predict_method_checking  s    rx  c              C   s(   t tdddddgidd} t|  d S )Nr   )r   rf  g?re   r   )r   )rC   r1   ru  )r   rI   rI   rJ   /test_gridsearchcv_cross_val_predict_with_method   s    

ry  c             C   s   t t||g}t||}x|| |D ]|\}}	|| | ||  || |	 }
|dkrnt t|	|f}nt t|	|ft |jj	}|
|d d |j
f< |||	< q*W |S )Nr  )rw   r   rs   rp  r   rN   fullZfinfor   minZclasses_)rP   r^   r   rq  r   r  rs  rt  r   r   Zexpected_predictions_Zexp_pred_testrI   rI   rJ   get_expected_predictions'  s    
r|  c              C   s  t ddd} t dd tdD }d}tdd}td	d}t }d
ddg}x|D ]}t }t|| |||d}	t	| |||||}
t
|
|	 t|| |||d}	t	| |||||}
t
|
|	 tt tdddd}t|| |||d}	||}t	| |||||}
t
|
|	 qXW d S )Nr  r   r   c             S   s   g | ]}|d  qS )r   rI   ).0r\   rI   rI   rJ   
<listcomp>>  s    z7test_cross_val_predict_class_subset.<locals>.<listcomp>r   rn   )r   r   r%  r  r+  )r  r   r   )r   )rw   r  rr   r   r   r   r9   r1   r   r|  r   r?   r  Zfit_transform)rP   r^   rq  Zkfold3Zkfold4lerr  r  r   r-  rs  rI   rI   rJ   #test_cross_val_predict_class_subset;  s4    












r  c                 s   t  } | j| j }}t }tjddd}|d |  tj	|j
tjdtj	|j
ddtjd z8t||| fd	d
d ttt|||fdd
d W d d\ x@tdD ]4}yt|j
 P W q tk
r   td Y qX qW X d S )NwbF)moderk  s   Hello world!!!!!)r   rI   r)rL   r  r   c                s    S )NrI   )r   rP   r^   )rU   rI   rJ   r   o  r   z#test_score_memmap.<locals>.<lambda>)r   c                s    S )NrI   )r   rP   r^   )r   rI   rJ   r   r  r   )NNrn   g      ?)r&   r   r   rj   tempfileNamedTemporaryFilewriterB  rw   Zmemmapnamer  r   r
   ru   r   osunlinkWindowsErrorr   )r   rP   r^   r   tfr  rI   )rU   r   rJ   test_score_memmapd  s&    
r  c                 s   t t fg} y"ddlm}m} | ||f W n tk
r@   Y nX xf| D ]^\ t }|j|j }} || }} fdd}fdd}	t	||	d}
t
|
|| qHW d S )Nr   )r   r   c                s
   t |  S )N)r   )r\   )r   rI   rJ   r     r   z4test_permutation_test_score_pandas.<locals>.<lambda>c                s
   t |  S )N)r   )r\   )r   rI   rJ   r     r   )r   r   )r   r   r   r   r   r   r&   r   r   r   r   )r   r   r   r   rP   r^   r   r   r   r   r   rI   )r   r   rJ   "test_permutation_test_score_pandas  s    
r  )rY   
__future__r   r>  r   r  r  timer   numpyrw   Zscipy.sparser   r   Zsklearn.utils.testingr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   Zsklearn.utils.mockingr   r   Zsklearn.model_selectionr   r   r   r   r   r   r   r   r   r   r    r!   r"   Z#sklearn.model_selection._validationr#   Zsklearn.datasetsr$   r%   r&   r'   Zsklearn.metricsr(   r)   r*   r+   r,   r-   r.   Zsklearn.metrics.scorerr/   Zsklearn.linear_modelr0   r1   r2   r3   r4   Zsklearn.neighborsr5   Zsklearn.svmr6   Zsklearn.clusterr7   Zsklearn.preprocessingr8   r9   Zsklearn.pipeliner:   Zsklearn.externals.six.movesr;   r@  Zsklearn.baser<   r=   Zsklearn.multiclassr>   Zsklearn.utilsr?   r@   rA   Z$sklearn.model_selection.tests.commonrB   rC   r  	NameErrorrD   rZ   rb   rf   r   rj   r   rP   r   r   r^   r   r   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r#  r)  r*  r,  r.  r/  r0  r;  r<  rD  rF  rG  rH  rQ  rS  rT  rU  ra  rd  re  rh  rm  rn  ru  rv  rx  ry  r|  r  r  r  rI   rI   rI   rJ   <module>   s  
A
(F'#0 21.-	$
""')