B
    !`Sq                 @   sx  d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ e ZdZ dZ!dZ"dZ#dZ$dZ%dZ&dddddZ'dZ(G dd de)Z*G dd de)Z+d d! Z,d"d# Z-d$d% Z.d&d' Z/d(d) Z0d*d+ Z1d,d- Z2G d.d/ d/ej3Z4d0d1 Z5dS )2zThe Embedding Projector plugin.    )absolute_import)division)print_functionN)wrappers)json_format)text_format)Respond)tf)base_plugin)metadata)ProjectorConfig)
tb_logging   z/infoz/tensorz	/metadataz/runsz
/bookmarksz/sprite_imagez	image/bmpz	image/gifz
image/jpegz	image/png)ZbmpZgifZjpegZpngzapplication/octet-streamc               @   s(   e Zd ZdZdd Zdd Zdd ZdS )	LRUCachez;LRU cache.

    Used for storing the last used tensor.
    c             C   s$   |dk rt d|| _t | _d S )Nr   zThe cache size must be >=1)
ValueError_sizecollectionsOrderedDict_dict)selfsize r   ]/home/dcms/DCMS/lib/python3.7/site-packages/tensorboard/plugins/projector/projector_plugin.py__init__E   s    zLRUCache.__init__c             C   s4   y| j |}|| j |< |S  tk
r.   d S X d S )N)r   popKeyError)r   keyvaluer   r   r   getK   s    
zLRUCache.getc             C   sb   |d krt dy| j| W n2 tk
rR   t| j| jkrN| jjdd Y nX || j|< d S )Nzvalue must be != NoneF)last)r   r   r   r   lenr   popitem)r   r   r   r   r   r   setS   s    zLRUCache.setN)__name__
__module____qualname____doc__r   r   r"   r   r   r   r   r   ?   s   r   c               @   s    e Zd ZdZdd Zdd ZdS )EmbeddingMetadatazMetadata container for an embedding.

    The metadata holds different columns with values used for
    visualization (color by, label by) in the "Embeddings" tab in
    TensorBoard.
    c             C   s   || _ g | _i | _dS )zConstructs a metadata for an embedding of the specified size.

        Args:
          num_points: Number of points in the embedding.
        N)
num_pointscolumn_namesname_to_values)r   r(   r   r   r   r   f   s    zEmbeddingMetadata.__init__c             C   s   t |tr t |d tr tdt |tjrD|jdkrDtd|j t|| jkrhtd| jt|f || jkr~td| | j	
| || j|< dS )a  Adds a named column of metadata values.

        Args:
          column_name: Name of the column.
          column_values: 1D array/list/iterable holding the column values. Must be
              of length `num_points`. The i-th value corresponds to the i-th point.

        Raises:
          ValueError: If `column_values` is not 1D array, or of length `num_points`,
              or the `name` is already used.
        r   zS"column_values" must be a flat list, but we detected that its first entry is a listr   z6"column_values" should be of rank 1, but is of rank %dz;"column_values" should be of length %d, but is of length %dz$The column name "%s" is already usedN)
isinstancelistr   npZndarrayndimr    r(   r*   r)   append)r   Zcolumn_nameZcolumn_valuesr   r   r   
add_columnp   s"    


zEmbeddingMetadata.add_columnN)r#   r$   r%   r&   r   r0   r   r   r   r   r'   ^   s   
r'   c          
   C   sd   t jj| d@}g }x4|D ],}|d}|r|ttt|	d qW W d Q R X t
j|ddS )Nr
	float32)dtype)r	   iogfileGFilerstripr/   r,   mapfloatsplitr-   array)fpathftensorliner   r   r   _read_tensor_tsv_file   s    

(rB   c             C   s2   t |dkrtd|tj| dd}||S )N   zTensor must be 2D, got shape {}r4   )r5   )r    r   formatr-   ZfromfileZreshape)r>   shaper@   r   r   r   _read_tensor_binary_file   s    rF   c             C   sL   t jjtj t jj }|| krHt jt jj t j }t jt j| |S | S )N)ospathsepr   PLUGINS_DIRpardirabspathjoin)
assets_dirZsub_pathZtwo_parents_upr   r   r   _assets_dir_to_logdir   s
    rO   c       
   
   C   s   x|D ]\}}|| krlt  }tj|tj}tjj	|rttjj
|d}| }W dQ R X t|| n| | }t|}t|}	|	sq|j|	krdS qW dS )zEReturns true if the latest checkpoint has changed in any of the runs.r1   NTF)r   rG   rH   rM   r   PROJECTOR_FILENAMEr	   r6   r7   existsr8   readr   MergerO   _find_latest_checkpointmodel_checkpoint_path)
configsrun_path_pairsrun_namerN   configconfig_fpathr?   file_contentlogdir	ckpt_pathr   r   r   _latest_checkpoints_changed   s     
r^   c             C   sH   | j |}|sdS yt|}|dkr,t |S  tk
rB   dS X dS )zParses and asserts a positive (>0) integer query parameter.

    Args:
      request: The Werkzeug Request object
      param_name: Name of the parameter.

    Returns:
      Param, or None, or -1 if parameter is not a positive integer.
    Nr   )argsr   intr   )request
param_nameparamr   r   r   _parse_positive_int_param   s    
re   c             C   s2   t j| } t j| s.t jt j|| S | S )N)rG   rH   
expanduserisabsrM   dirname)r>   rZ   r   r   r   _rel_to_abs_asset_path   s    ri   c               C   s
   t jdkS )zCReturn true if we're not using the fake TF API stub implementation.Zstub)r	   __version__r   r   r   r   	_using_tf   s    rk   c               @   s  e Zd ZdZejZdd Zdd Zdd Z	dd	 Z
d
d Zedd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zejjd d! Zejjd"d# Zejjd$d% Zejjd&d' Zejjd(d) Zejjd*d+ Zejjd,d- Z d.S )/ProjectorPluginzEmbedding projector.c             C   sZ   |j | _ |j| _i | _d| _i | _d| _d| _tt| _	d| _
d| _| j rV| j  | _dS )zInstantiates ProjectorPlugin via TensorBoard core.

        Args:
          context: A base_plugin.TBContext instance.
        NF)multiplexerr\   readers	run_paths_configsold_num_run_pathsconfig_fpathsr   _TENSOR_CACHE_CAPACITYtensor_cache
_is_active!_thread_for_determining_is_activeZRunPaths)r   contextr   r   r   r      s    
zProjectorPlugin.__init__c             C   st   d}t | jt| jt| jt| jt| j	t
| jdt| jtj|ddt| jtj|ddt| jtj|di	S )NZtf_projector_pluginz	/index.jszindex.jsz/projector_binary.htmlzprojector_binary.htmlz/projector_binary.jszprojector_binary.js)
RUNS_ROUTE_serve_runsCONFIG_ROUTE_serve_configTENSOR_ROUTE_serve_tensorMETADATA_ROUTE_serve_metadataBOOKMARKS_ROUTE_serve_bookmarksSPRITE_IMAGE_ROUTE_serve_sprite_image	functoolspartial_serve_filerG   rH   rM   )r   Zasset_prefixr   r   r   get_plugin_apps
  s    zProjectorPlugin.get_plugin_appsc             C   sB   | j s
dS | jrdS | jr | jS tj| jdd}|| _|  dS )zDetermines whether this plugin is active.

        This plugin is only active if any run has an embedding.

        Returns:
          Whether any run has embedding data to show in the projector.
        FTZProjectorPluginIsActiveThread)targetname)rm   ru   rv   	threadingThread_determine_is_activestart)r   Z
new_threadr   r   r   	is_active   s    zProjectorPlugin.is_activec             C   s   t jdddS )Nz	/index.jsT)Zes_module_pathZdisable_reload)r
   ZFrontendMetadata)r   r   r   r   frontend_metadataA  s    z!ProjectorPlugin.frontend_metadatac             C   s   | j rd| _d| _dS )zDetermines whether the plugin is active.

        This method is run in a separate thread so that the plugin can
        offer an immediate response to whether it is active and
        determine whether it should be active in a separate thread.
        TN)rV   ru   rv   )r   r   r   r   r   F  s    z$ProjectorPlugin._determine_is_activec             C   st   t | j }| | tdd |D s:|d| jf |  sNt| j	|rni | _
| |\| _	| _|   | j	S )z7Returns a map of run paths to `ProjectorConfig` protos.c             s   s   | ]\}}|d kV  qdS ).Nr   ).0runrH   r   r   r   	<genexpr>Z  s    z*ProjectorPlugin.configs.<locals>.<genexpr>r   )r,   ro   items _append_plugin_asset_directoriesanyr/   r\   _run_paths_changedr^   rp   rn   _read_latest_config_filesrr   %_augment_configs_with_checkpoint_info)r   rW   r   r   r   rV   Q  s    


zProjectorPlugin.configsc             C   s*   t t| j }|| jkr&|| _dS dS )NTF)r    r,   ro   keysrq   )r   Znum_run_pathsr   r   r   r   f  s
    
z"ProjectorPlugin._run_paths_changedc          
   C   s  x| j  D ]\}}x|jD ]}|jdr@|jd d |_|jr t|j| j| }| j	||jf}|d kryt
|}W n  tk
r   t||j}Y nX | j||jf| |js |jt|t|d g q W | |}|sqd }|jr|jd js|jd }|j| | }x| D ]v\}	}
t|
dkrFq,d|	krTq,| |	|}|s|j }|	|_|r|j|_|j|_|js,|j|
 q,W qW g }x*| j  D ]\}}|js|| qW x|D ]}| j |= | j|= qW d S )Nz:0r   rC   z.OPTIMIZER_SLOT)rp   r   
embeddingstensor_nameendswithtensor_pathri   rr   rt   r   rB   UnicodeDecodeErrorrF   tensor_shaper"   extendr    _get_reader_for_runremoveZget_variable_to_shape_map_get_embeddingaddmetadata_pathbookmarks_pathr/   )r   r   rY   	embeddingr>   r@   readerZspecial_embeddingZvar_mapr   r   Zruns_to_remover   r   r   r   m  s`    




z5ProjectorPlugin._augment_configs_with_checkpoint_infoc          
   C   s  i }i }x|D ]\}}t  }tj|tj}tjj	|rltjj
|d}| }	W dQ R X t|	| d}
x0|jD ]&}|jrx|jstj|j|_d}
P qxW |jst|}t|}|s|
sq|r||_|jrt rtjj|jd std|j q|||< |||< qW ||fS )zLReads and returns the projector config files in every run
        directory.r1   NFT*zCheckpoint file "%s" not found)r   rG   rH   rM   r   rP   r	   r6   r7   rQ   r8   rR   r   rS   r   r   r   basenamerU   rO   rT   rk   globloggerwarning)r   rW   rV   rr   rX   rN   rY   rZ   r?   r[   Zhas_tensor_filesr   r\   r]   r   r   r   r     sD    
z)ProjectorPlugin._read_latest_config_filesc             C   sr   || j kr| j | S | j| }d }|jrdt rdytj|j}W n" tk
rb   t	d|j Y nX || j |< |S )NzFailed reading "%s")
rn   rp   rU   rk   r	   trainZload_checkpoint	Exceptionr   r   )r   r   rY   r   r   r   r   r     s    



z#ProjectorPlugin._get_reader_for_runc             C   s   |  ||}|r|jS d S )N)r   r   )r   r   rY   embedding_infor   r   r   _get_metadata_file_for_tensor  s    z-ProjectorPlugin._get_metadata_file_for_tensorc             C   s   |  ||}|r|jS d S )N)r   r   )r   r   rY   r   r   r   r   _get_bookmarks_file_for_tensor  s    z.ProjectorPlugin._get_bookmarks_file_for_tensorc             C   s   d|kr|d S |S d S )N:z:0r   )r   r   r   r   r   _canonical_tensor_name  s    z&ProjectorPlugin._canonical_tensor_namec             C   s8   |j s
d S x(|j D ]}| |j| |kr|S qW d S )N)r   r   r   )r   r   rY   infor   r   r   r     s    zProjectorPlugin._get_embeddingc             C   sd   x^| j tj D ]H\}}tj|kr(qtj| j	| tj
tj}|tj|f}|| qW d S )N)rm   ZPluginAssetsr   ZPLUGIN_ASSETS_NAMEr   rP   rG   rH   rM   ro   rJ   rL   r/   )r   rW   r   ZassetsrN   Zassets_path_pairr   r   r   r   	  s    
z0ProjectorPlugin._append_plugin_asset_directoriesc          	   C   sL   t jt jt|}t|d"}t|d }t||	 |dS Q R X dS )zReturns a resource file.rbr   )content_typeN)
rG   rH   rM   rh   __file__open	mimetypes
guess_typer   rR   )r   	file_pathrb   Zres_path	read_filemimetyper   r   r   r     s    zProjectorPlugin._serve_filec             C   s   t |t| j dS )z,Returns a list of runs that have embeddings.zapplication/json)r   r,   rV   r   )r   rb   r   r   r   ry     s    zProjectorPlugin._serve_runsc             C   sZ   |j d}|d kr"t|dddS || jkr>t|d| ddS | j| }t|t|dS )Nr   z!query parameter "run" is requiredz
text/plaini  zUnknown run: "%s"zapplication/json)r`   r   r   rV   r   ZMessageToJson)r   rb   r   rY   r   r   r   r{   $  s    

zProjectorPlugin._serve_configc          	   C   s~  |j d}|d kr"t|dddS |j d}|d krDt|dddS t|d}|dkrdt|d	ddS || jkrt|d
| ddS | j| }| ||}|st|d|| j| f ddS t|| j| }tj	j
|rtj	j
|rt|d| ddS d}tj	j
|d^}g }	xR|D ]J}
|	|
 t|	dkrBd|	d krBd}|rt|	|| krP qW W d Q R X t|d|	dS )Nr   z!query parameter "run" is requiredz
text/plaini  r   z"query parameter "name" is requirednum_rowsr_   z,query parameter num_rows must be integer > 0zUnknown run: "%s"z>No metadata file found for tensor "%s" in the config file "%s"z "%s" not found, or is not a filer   r1   r   r3    )r`   r   r   re   rV   r   rr   ri   r	   r6   r7   rQ   isdirr8   r/   r    rM   )r   rb   r   r   r   rY   r>   Znum_header_rowsr?   linesrA   r   r   r   r   5  sX    




zProjectorPlugin._serve_metadatac          
   C   s  |j d}|d kr"t|dddS |j d}|d krDt|dddS t|d}|dkrdt|d	ddS || jkrt|d
| ddS | j| }| j||f}|d kr| ||}|r"|jr"t|j| j	| }t
jj|st|d| ddS yt|}W n" tk
r   t||j}Y nX nz| |}	|	r>|	|sVt|d||jf ddS y|	|}W n6 t
jjk
r }
 zt|t|
ddS d }
~
X Y nX | j||f| |r|d | }|jdkr|jddd}| }t||dS )Nr   z!query parameter "run" is requiredz
text/plaini  r   z"query parameter "name" is requiredr   r_   z,query parameter num_rows must be integer > 0zUnknown run: "%s"zTensor file "%s" does not existz,Tensor "%s" not found in checkpoint dir "%s"r4   F)r5   copyzapplication/octet-stream)r`   r   r   re   rV   rt   r   r   ri   rr   r	   r6   r7   rQ   rB   r   rF   r   r   Z
has_tensorrU   Z
get_tensorerrorsZInvalidArgumentErrorstrr"   r5   Zastypetobytes)r   rb   r   r   r   rY   r@   r   r>   r   eZ
data_bytesr   r   r   r}   q  sn    





 zProjectorPlugin._serve_tensorc          	   C   s  |j d}|st|dddS |j d}|d kr@t|dddS || jkr\t|d| ddS | j| }| ||}|st|d|| j| f ddS t|| j| }tjj	
|rtjj	|rt|d	| ddS d }tjj	|d
}| }W d Q R X t||dS )Nr   z!query parameter "run" is requiredz
text/plaini  r   z"query parameter "name" is requiredzUnknown run: "%s"z?No bookmarks file found for tensor "%s" in the config file "%s"z "%s" not found, or is not a filer   zapplication/json)r`   r   r   rV   r   rr   ri   r	   r6   r7   rQ   r   r8   rR   )r   rb   r   r   rY   r>   Zbookmarks_jsonr?   r   r   r   r     s>    

z ProjectorPlugin._serve_bookmarksc             C   s,  |j d}|st|dddS |j d}|d kr@t|dddS || jkr\t|d| ddS | j| }| ||}|r~|jjst|d|| j| f ddS tj	
|jj}t|| j| }tjj|rtjj|rt|d	| ddS tjj|d
}| }|  td |}	t|	t}
t|||
S )Nr   z!query parameter "run" is requiredz
text/plaini  r   z"query parameter "name" is requiredzUnknown run: "%s"zBNo sprite image file found for tensor "%s" in the config file "%s"z#"%s" does not exist or is directoryr   )r`   r   r   rV   r   ZspriteZ
image_pathrr   rG   rH   rf   ri   r	   r6   r7   rQ   r   r8   rR   closeimghdrwhat_IMGHDR_TO_MIMETYPE_DEFAULT_IMAGE_MIMETYPE)r   rb   r   r   rY   r   r>   r?   Zencoded_image_stringZ
image_typeZ	mime_typer   r   r   r     sD    

z#ProjectorPlugin._serve_sprite_imageN)!r#   r$   r%   r&   r   ZPLUGIN_NAMEZplugin_namer   r   r   r   r   propertyrV   r   r   r   r   r   r   r   r   r   r   RequestZapplicationr   ry   r{   r   r}   r   r   r   r   r   r   rl      s0   !B.
<L+rl   c             C   sT   t  s
d S y,tj| }|s4tjtj| tj}|S  tjj	k
rN   d S X d S )N)
rk   r	   r   Zlatest_checkpointrG   rH   rM   rK   r   ZNotFoundError)Zdir_pathr]   r   r   r   rT     s    rT   )6r&   
__future__r   r   r   r   r   r   r   rG   r   numpyr-   Zwerkzeugr   Zgoogle.protobufr   r   Ztensorboard.backend.http_utilr   Ztensorboard.compatr	   Ztensorboard.pluginsr
   Ztensorboard.plugins.projectorr   Z2tensorboard.plugins.projector.projector_config_pb2r   Ztensorboard.utilr   Z
get_loggerr   rs   rz   r|   r~   rx   r   r   r   r   objectr   r'   rB   rF   rO   r^   re   ri   rk   ZTBPluginrl   rT   r   r   r   r   <module>   s^   :
    2