B
    `S$                 @   s&  d Z ddlmZ ddlmZ ddlmZ ddlZddlm  mZ	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" dgZ#eZ$dd Z%dd Z&dd Z'G dd dej(Z)G dd de)Z*G dd de)Z+dd Z,dS )z?Base class for variational layers for building neural networks.    )absolute_import)division)print_functionN)random)distribution)independent)kullback_leibler)mvn_diag)normal)layers)prefer_static)FULLY_REPARAMETERIZED)expectation)
SeedStreamVariationalLayerc          	      s6   t  fdd| jtdd tj jD ddS )zMonte Carlo KL Divergence.c                s     |  |  S )N)log_prob)w)qr m/home/dcms/DCMS/lib/python3.7/site-packages/tensorflow_probability/python/experimental/nn/variational_base.py<lambda>1       z+kl_divergence_monte_carlo.<locals>.<lambda>c             s   s   | ]}|t kV  qd S )N)r   ).0rtr   r   r   	<genexpr>5   s   z,kl_divergence_monte_carlo.<locals>.<genexpr>r   )Zsamplesr   Zuse_reparameterizationZaxis)r   r   alltfnestflattenZreparameterization_type)r   r   r   r   )r   r   r   kl_divergence_monte_carlo.   s    r    c             C   s   t | |S )zExact KL Divergence.)kl_libZkl_divergence)r   r   r   r   r   r   kl_divergence_exact:   s    r"   c             C   sj   t | tjr&| dd}| dd}n<t| dkr>| d }}n$t| dkrT| \}}ntd| ||fS )zReturns `kernel`, `bias` tuple.kernelNbias      zUnable to unpack weights: {}.)
isinstancecollectionsMappinggetlen
ValueErrorformat)weightsr#   r$   r   r   r   unpack_kernel_and_bias?   s    
r/   c                   s   e Zd ZdZddeejjdej	df fdd	Z
edd Zedd Zed	d
 Zedd Zedd Zedd Zedd ZdddZdd Z  ZS )r   z&Base class for all variational layers.Nc
       
         sf   t t| j|	d || _|| _|| _|| _|| _|| _t	|| j
d| _|| _tjj|j|jdd dS )a  Base class for variational layers.

    # mean ==> penalty_weight =          1 / train_size
    # sum  ==> penalty_weight = batch_size / train_size

    Args:
      posterior: ...
      prior: ...
      activation_fn: ...
      penalty_weight: ...
      posterior_penalty_fn: ...
      posterior_value_fn: ...
      seed: ...
      dtype: ...
      name: Python `str` prepeneded to ops created by this object.
        Default value: `None` (i.e., `type(self).__name__`).
    )name)ZsaltF)Zcheck_typesN)superr   __init__
_posterior_prior_activation_fn_penalty_weight_posterior_penalty_fn_posterior_value_fnr   r0   _seed_dtyper   r   Zassert_same_structuredtype)
self	posteriorprioractivation_fnpenalty_weightposterior_penalty_fnposterior_value_fnseedr;   r0   )	__class__r   r   r2   P   s    zVariationalLayer.__init__c             C   s   | j S )N)r:   )r<   r   r   r   r;   x   s    zVariationalLayer.dtypec             C   s   | j S )N)r3   )r<   r   r   r   r=   |   s    zVariationalLayer.posteriorc             C   s   | j S )N)r4   )r<   r   r   r   r>      s    zVariationalLayer.priorc             C   s   | j S )N)r5   )r<   r   r   r   r?      s    zVariationalLayer.activation_fnc             C   s   | j S )N)r6   )r<   r   r   r   r@      s    zVariationalLayer.penalty_weightc             C   s   | j S )N)r7   )r<   r   r   r   rA      s    z%VariationalLayer.posterior_penalty_fnc             C   s   | j S )N)r8   )r<   r   r   r   rB      s    z#VariationalLayer.posterior_value_fnTc             K   s   t j|| jdd}| j| j|  d}|rf| | j| j|}|d k	rj| jd k	rj|t j	| j|jd9 }nd }| j
||f|}| | | | |S )Ninputs)r;   r0   )rC   )r;   )r   Zconvert_to_tensorr;   rB   r=   r9   rA   r>   r@   cast_evalZ_set_extra_lossZ_set_extra_result)r<   rE   Zis_trainingkwargsr   Zpenaltyoutputsr   r   r   eval   s    

zVariationalLayer.evalc             C   s   t dd S )Nz%Subclass failed to implement `_eval`.)NotImplementedError)r<   rE   r.   r   r   r   rG      s    zVariationalLayer._eval)T)__name__
__module____qualname____doc__r    tfdDistributionsampler   float32r2   propertyr;   r=   r>   r?   r@   rA   rB   rJ   rG   __classcell__r   r   )rD   r   r   M   s"   
c                   sL   e Zd ZdZddeejjede	j
df fdd	Zedd Zdd Z  ZS )	,VariationalReparameterizationKernelBiasLayerz,Variational reparameterization linear layer.Nc                s2   t t| j|||||||	|
|d	 || _|| _d S )N)r?   r@   rA   rB   rC   r;   r0   )r1   rV   r2   _apply_kernel_fn_unpack_weights_fn)r<   r=   r>   apply_kernel_fnr?   r@   rA   rB   unpack_weights_fnrC   r;   r0   )rD   r   r   r2      s    
z5VariationalReparameterizationKernelBiasLayer.__init__c             C   s   | j S )N)rX   )r<   r   r   r   rZ      s    z>VariationalReparameterizationKernelBiasLayer.unpack_weights_fnc             C   sN   |  |\}}|}|d k	r&| ||}|d k	r6|| }| jd k	rJ| |}|S )N)rZ   rW   r?   )r<   xr.   r#   r$   yr   r   r   rG      s    

z2VariationalReparameterizationKernelBiasLayer._eval)rL   rM   rN   rO   r    rP   rQ   rR   r/   r   rS   r2   rT   rZ   rG   rU   r   r   )rD   r   rV      s   rV   c                   sL   e Zd ZdZddeejjede	j
df fdd	Zedd Zdd Z  ZS )	!VariationalFlipoutKernelBiasLayerz!Variational flipout linear layer.Nc                s2   t t| j|||||||	|
|d	 || _|| _d S )N)r?   r@   rA   rB   rC   r;   r0   )r1   r]   r2   rW   rX   )r<   r=   r>   rY   r?   r@   rA   rB   rZ   rC   r;   r0   )rD   r   r   r2      s    
z*VariationalFlipoutKernelBiasLayer.__init__c             C   s   | j S )N)rX   )r<   r   r   r   rZ      s    z3VariationalFlipoutKernelBiasLayer.unpack_weights_fnc             C   s   |  |\}}|}|d k	r|  | jj|dd \}}t|\}}	|tjt||j| 	 d9 }t
jd|	d}
| ||
j| 	 d}|tjt||j| 	 d9 }|| ||7 }|d k	r|| }| jd k	r| |}|S )N)valuer   )r;   rC   g        )locscale)rC   )rZ   r=   Zsample_distributionsget_spherical_normal_loc_scale
tfp_randomZ
rademacherr   shaper;   r9   
normal_libNormalrW   rR   r?   )r<   r[   r.   r#   r$   r\   Zkernel_dist_Z
kernel_locZkernel_scaleZkernel_perturbr   r   r   rG      s,    

z'VariationalFlipoutKernelBiasLayer._eval)rL   rM   rN   rO   r    rP   rQ   rR   r/   r   rS   r2   rT   rZ   rG   rU   r   r   )rD   r   r]      s   r]   c             C   sL   t | tjrt| jS t | tjtjfr4| j	| j
fS tdt| jd S )Nz>Expected kernel `posterior` to be spherical Normal; saw: "{}".)r'   independent_libZIndependentra   r   rd   re   mvn_diag_libZMultivariateNormalDiagr_   r`   	TypeErrorr-   typerL   )dr   r   r   ra     s    
ra   )-rO   
__future__r   r   r   r(   Ztensorflow.compat.v2compatZv2r   Ztensorflow_probability.pythonr   rb   Z+tensorflow_probability.python.distributionsr   Zdistribution_libr   rg   r   r!   r	   rh   r
   rd   Z-tensorflow_probability.python.experimental.nnr   Z
layers_libZ&tensorflow_probability.python.internalr   Z9tensorflow_probability.python.internal.reparameterizationr   Z)tensorflow_probability.python.monte_carlor   Z.tensorflow_probability.python.util.seed_streamr   __all__rP   r    r"   r/   ZLayerr   rV   r]   ra   r   r   r   r   <module>   s2   Y-C