U
    {hDg                     @   s  d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z
mZmZmZ d dlmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z: ddl(m;Z; ddl<m=Z=m>Z> eede dZ?G dd dZ@G dd dZAG dd de8ZBG dd deBZCG d d! d!e9ZDG d"d# d#ZEG d$d% d%eEZFG d&d' d'e7ZGeGZHdS )(    N)OrderedDict)reduce)SearchQuery
SearchRankSearchVector)DEFAULT_DB_ALIASNotSupportedErrorconnectionstransaction)AvgCountFManagerQ	TextFieldValue)
LOOKUP_SEP)CastLength)InsertQuery)	force_str)cached_property   )AutocompleteFieldRelatedFieldsSearchFieldget_indexed_models)
IndexEntry)AndBoostMatchAllNotOrPhrase	PlainText)ADDMULORget_content_type_pk!get_descendants_content_types_pks   )BaseSearchBackendBaseSearchQueryCompilerBaseSearchResultsFilterFieldError   )Lexeme)get_sql_weights
get_weight Zoutput_fieldc                   @   sb   e Zd ZdZdd Zdd Zdd Zdd	d
Zedd Z	edd Z
edd Zedd ZdS )ObjectIndexerzW
    Responsible for extracting data from an object to be inserted into the index.
    c                 C   s$   || _ | | _|j| _|j| _d S N)objget_search_fieldssearch_fieldsconfigautocomplete_config)selfr7   backend r>   z/var/www/dating/data/www/fatepal.com/env/lib/python3.8/site-packages/wagtail/search/backends/database/postgres/postgres.py__init__)   s    
zObjectIndexer.__init__c                    s^   t |tr|S t |tr0d fdd|D S t |trVd fdd| D S t|S )N, c                 3   s   | ]}  |V  qd S r6   prepare_value.0itemr<   r>   r?   	<genexpr>4   s     z.ObjectIndexer.prepare_value.<locals>.<genexpr>c                 3   s   | ]}  |V  qd S r6   rB   rD   rG   r>   r?   rH   7   s     )
isinstancestrlistjoindictvaluesr   )r<   valuer>   rG   r?   rC   /   s    


zObjectIndexer.prepare_valuec                 c   s   t |tr*|t|j| ||fV  nt |trN|d| ||fV  npt |tr||}|d krnd S t |tr|	 }nt
|r| }|g}|D ]"}|jD ]}| ||E d H  qqd S )ND)rI   r   r2   boostrC   	get_valuer   r   r   allcallablefieldsprepare_field)r<   r7   fieldZsub_objZsub_objsZ	sub_fieldr>   r>   r?   rV   ;   s&    






zObjectIndexer.prepare_fieldFc                    sJ   dd |D }dd |D }|s$t S |r.| jn| j t fdd|D S )zW
        Converts an array of strings into a SearchVector that can be indexed.
        c                 S   s   g | ]\}}|  |fqS r>   )striprE   textweightr>   r>   r?   
<listcomp>]   s     z+ObjectIndexer.as_vector.<locals>.<listcomp>c                 S   s   g | ]\}}|r||fqS r>   r>   rY   r>   r>   r?   r\   ^   s      c                    s(   g | ] \}}t t|t d | dqS )r4   )r[   r:   )r   r   r   rY   Zsearch_configr>   r?   r\   f   s   )EMPTY_VECTORr;   r:   r%   )r<   textsfor_autocompleter>   r]   r?   	as_vectorY   s    
zObjectIndexer.as_vectorc                 C   s   t | jjS )zO
        Returns the value to use as the ID of the record in the index
        )r   r7   pkrG   r>   r>   r?   idp   s    zObjectIndexer.idc                 C   sV   g }| j D ]@}| | j|D ],\}}}t|tr|jdkr|||f qq
| |S )z
        Returns all values to index as "title". This is the value of all SearchFields that have the field_name 'title'
        titler9   rV   r7   rI   r   
field_nameappendra   r<   r_   rW   Zcurrent_fieldrQ   rO   r>   r>   r?   rd   w   s    
zObjectIndexer.titlec                 C   sV   g }| j D ]@}| | j|D ],\}}}t|tr|jdks|||f qq
| |S )zr
        Returns all values to index as "body". This is the value of all SearchFields excluding the title
        rd   re   rh   r>   r>   r?   body   s    
zObjectIndexer.bodyc                 C   sP   g }| j D ]6}| | j|D ]"\}}}t|tr|||f qq
| j|ddS )zl
        Returns all values to index as "autocomplete". This is the value of all AutocompleteFields
        T)r`   )r9   rV   r7   rI   r   rg   ra   rh   r>   r>   r?   autocomplete   s    

zObjectIndexer.autocompleteN)F)__name__
__module____qualname____doc__r@   rC   rV   ra   r   rc   rd   ri   rj   r>   r>   r>   r?   r5   $   s   



r5   c                   @   sp   e Zd ZdddZdd Zdd Zdd	d
Zdd Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )IndexNc                 C   sd   || _ | j j| _|d krtn|| _t| j | _| jjdkrBtd| jj	dk| _
tj| j| _d S )N
postgresqlz?You must select a PostgreSQL database to use PostgreSQL search.ia )r=   
index_namenamer   db_aliasr	   
connectionvendorr   Z
pg_version_enable_upsertr   _default_managerusingentries)r<   r=   rs   r>   r>   r?   r@      s    
zIndex.__init__c                 C   s   d S r6   r>   r<   modelr>   r>   r?   	add_model   s    zIndex.add_modelc                 C   s   d S r6   r>   rG   r>   r>   r?   refresh   s    zIndex.refreshFc                 C   sn   | j jtddjddtdd }|r4| j }n| j jdd}|jtddjddj|td d d	S )
a  
        Refreshes the value of the title_norm field.

        This needs to be set to 'lavg/ld' where:
         - lavg is the average length of titles in all documents (also in terms)
         - ld is the length of the title field in this document (in terms)
        rd   )title_lengthr   )Ztitle_length__gtr~   Ztitle_length__avg      ?)Z
title_normN)ry   annotater   filterZ	aggregater   updater   )r<   fullZlavgry   r>   r>   r?   _refresh_title_norms   s     

zIndex._refresh_title_normsc                 C   sN   |j | jjtdt dd}t|}| jj	|dj
|d}|  d S )Nrb   	object_idr   )Zcontent_type_id__inZobject_id__in)rw   rx   rs   r   r   r   rN   r)   ry   r   excludedelete)r<   r{   Zexisting_pksZcontent_types_pksZstale_entriesr>   r>   r?   delete_stale_model_entries   s    
z Index.delete_stale_model_entriesc                 C   s"   t  D ]}|jjs| | qd S r6   )r   _metaparentsr   rz   r>   r>   r?   delete_stale_entries   s    
zIndex.delete_stale_entriesc                 C   s   |  |jj|g d S r6   )	add_itemsr   r{   r<   r7   r>   r>   r?   add_item   s    zIndex.add_itemc              	   C   sF  t tj| jd}g }g }g }g }|D ]}|||jf |tjd|j	}	|	
|| j\}
}||
 || |tjd|j}	|	
|| j\}
}||
 || |tjd|j}	|	
|| j\}
}||
 || q&ddd t|||D }| j }|dtjj|f | W 5 Q R X |   d S )	N)rt   rd   rj   ri   rA   c              	   S   s*   g | ]"\}}}d | d| d| dqS )z	(%s, %s, rA   z, 1.0)r>   )rE   abcr>   r>   r?   r\     s   z*Index.add_items_upsert.<locals>.<listcomp>a  
                INSERT INTO %s (content_type_id, object_id, title, autocomplete, body, title_norm)
                (VALUES %s)
                ON CONFLICT (content_type_id, object_id)
                DO UPDATE SET title = EXCLUDED.title,
                              title_norm = 1.0,
                              autocomplete = EXCLUDED.autocomplete,
                              body = EXCLUDED.body
                )r   r   Zget_compilerrt   extendrc   rC   r   	get_fieldrd   Zas_sqlrg   rj   ri   rL   zipcursorexecuteZdb_tabler   )r<   content_type_pkindexerscompilerZ	title_sqlZautocomplete_sqlZbody_sqlZdata_paramsindexerrO   sqlparamsZdata_sqlr   r>   r>   r?   add_items_upsert   sR    
 


 


 

	

zIndex.add_items_upsertc              
   C   s   i }|D ]}|j |j|jf||j< q| jj|d}t|j| djddd}|D ]*}|| \}}	}
|j|dj	||	|
d qTg }| D ]2}||kr|| \}}	}
|
t||||	|
d q| j| |   d S )	N)content_type_idr   r   T)Zflatr   )rd   rj   ri   )r   r   rd   rj   ri   )rd   rj   ri   rc   ry   r   	frozensetkeysZvalues_listr   rg   r   Zbulk_creater   )r<   r   r   Zids_and_datar   Zindex_entries_for_ctZindexed_idsZ
indexed_idrd   rj   ri   Zto_be_createdr   r>   r>   r?   add_items_update_then_create*  sF       
z"Index.add_items_update_then_createc                    sN   |  }|sd S  fdd|D }|rJt|} jr: jn j}||| d S )Nc                    s   g | ]}t | jqS r>   )r5   r=   )rE   r7   rG   r>   r?   r\   V  s     z#Index.add_items.<locals>.<listcomp>)r8   r(   rv   r   r   )r<   r{   Zobjsr9   r   r   Zupdate_methodr>   rG   r?   r   Q  s    zIndex.add_itemsc                 C   s   |j  j| jd d S )Nrx   )Zindex_entriesrS   _raw_deleters   )r<   rF   r>   r>   r?   delete_itemc  s    zIndex.delete_itemc                 C   s   | j S r6   )rr   rG   r>   r>   r?   __str__f  s    zIndex.__str__)N)F)rk   rl   rm   r@   r|   r}   r   r   r   r   r   r   r   r   r   r>   r>   r>   r?   ro      s   

:'ro   c                       s   e Zd ZdZdZeZ fddZdd Zdd Z	d!d
dZ
d"ddZd#ddZd$ddZdd Zdd Zdd Zdd Zd%ddZdd Zdd  Z  ZS )&PostgresSearchQueryCompilerandFc                    sL   t  j||   t _jd kr0 _n fddjD _d S )Nc                    s   i | ]}|j | d qS ))rU   )get_search_field)rE   field_lookupZlocal_search_fieldsr<   r>   r?   
<dictcomp>~  s     z8PostgresSearchQueryCompiler.__init__.<locals>.<dictcomp>)superr@   get_search_fields_for_modelr1   sql_weightsrU   r9   )r<   argskwargs	__class__r   r?   r@   o  s    
z$PostgresSearchQueryCompiler.__init__c                 C   s   |j S r6   r:   r<   r=   r>   r>   r?   
get_config  s    z&PostgresSearchQueryCompiler.get_configc                 C   s   | j j S r6   )querysetr{   Zget_searchable_search_fieldsrG   r>   r>   r?   r     s    z7PostgresSearchQueryCompiler.get_search_fields_for_modelNc                 C   s~   |d kr| j }t|kr(|td\}}nd }|D ]H}t|| jrR|j|krR|  S t|tr0|j|kr0| ||j  S q0d S )Nr/   )	r9   r   splitrI   TARGET_SEARCH_FIELD_TYPErf   r   r   rU   )r<   r   rU   Zsub_field_namerW   r>   r>   r?   r     s    
z,PostgresSearchQueryCompiler.get_search_fieldc                    sL  t |trt|j }|sd S | }t|jd}|D ],}t|d}|jdkr\||M }q8||O }q8t|d dS t |t	rt|jd dS t |t
rd}	t|	t j|j dS t |tr؈j|j  dS t |ttfr8 fd	d
|jD }
t |t}r| }|r*tdd |
S tdd |
S td|jj d S )N)invertprefix)r   r   raw)Zsearch_typer:   phrasezBThe Boost query is not supported by the PostgreSQL search backend.r:   r   c                    s   g | ]}j | d qS )r   build_tsquery_contentrE   subqueryr:   r   r<   r>   r?   r\     s   zEPostgresSearchQueryCompiler.build_tsquery_content.<locals>.<listcomp>c                 S   s   | |@ S r6   r>   r   r   r>   r>   r?   <lambda>      zCPostgresSearchQueryCompiler.build_tsquery_content.<locals>.<lambda>c                 S   s   | |B S r6   r>   r   r>   r>   r?   r     r   7`%s` is not supported by the PostgreSQL search backend.)rI   r$   Zquery_stringr   popr0   LAST_TERM_IS_PREFIXoperatorr   r#   r   warningswarnRuntimeWarningr   r   r!   r   r"   
subqueriesr   NotImplementedErrorr   rk   )r<   queryr:   r   ZtermsZ	last_termZlexemestermZ
new_lexememsgZsubquery_lexemesZis_andr>   r   r?   r     sV    






  
  
z1PostgresSearchQueryCompiler.build_tsquery_contentc                 C   s   | j ||dS )Nr   r   )r<   r   r:   r>   r>   r?   build_tsquery  s    z)PostgresSearchQueryCompiler.build_tsqueryr   c                    s   t |tttfr>tj|djd} dkr:| 9 }|S t |trf |j9  j	|j
 dS t |trt fdd|jD d S t |trt fdd|jD t|jpd S td	|jj d S )
Nr   )weightsr   r:   rQ   c                 3   s$   | ]}d j | d V  qdS )r/   r   Nbuild_tsrankr   rQ   r:   r<   vectorr>   r?   rH     s   z;PostgresSearchQueryCompiler.build_tsrank.<locals>.<genexpr>r/   c                 3   s    | ]}j | d V  qdS )r   Nr   r   r   r>   r?   rH     s   r   )rI   r#   r$   r!   r   r   r   r   rQ   r   r   r   r&   r   r"   r%   lenr   r   rk   )r<   r   r   r:   rQ   rank_expressionr>   r   r?   r     s<    



z(PostgresSearchQueryCompiler.build_tsrankc                 C   s   t dt dft ddfgS )NZindex_entries__titleZindex_entries__title_normZindex_entries__bodyr   r   r<   search_queryr>   r>   r?   get_index_vectors  s    
z-PostgresSearchQueryCompiler.get_index_vectorsc                    s    fdd| j  D S )Nc                    s$   g | ]\}}t | jd |jfqS r   )r   r:   rQ   rE   r   Zsearch_fieldr   r>   r?   r\     s   zBPostgresSearchQueryCompiler.get_fields_vectors.<locals>.<listcomp>r9   itemsr   r>   r   r?   get_fields_vectors  s    
z.PostgresSearchQueryCompiler.get_fields_vectorsc                 C   s"   | j d kr| |S | |S d S r6   )rU   r   r   r   r>   r>   r?   get_search_vectors#  s    

z.PostgresSearchQueryCompiler.get_search_vectorsc                    s:    fdd|D }|d }|dd  D ]}||7 }q(|S )Nc                    s&   g | ]\}}j |j d | qS r   )r   r   )rE   r   rQ   r:   r<   r>   r?   r\   +  s   zFPostgresSearchQueryCompiler._build_rank_expression.<locals>.<listcomp>r   r/   r>   )r<   vectorsr:   Zrank_expressionsr   Zother_rank_expressionr>   r   r?   _build_rank_expression*  s    
z2PostgresSearchQueryCompiler._build_rank_expressionc                 C   s   t | jtr| j|| S t | jtr>t | jjtr>| j S | j| j|d}| |}| 	||}|d d }|dd  D ]\}	}
|
|	dd}q|| jj|dj|d}| jr|| d}n|jjs|d}td}|d k	r|jf ||i}||| S )	Nr   r   r/   z||F)Z_vector_z-pkrb   )rI   r   r    r   r!   r   noner   r   r   Z_combiner   r   Zorder_by_relevanceorder_bydescr   )r<   r:   startstopscore_fieldr   r   r   Zcombined_vectorr   rQ   r   r>   r>   r?   search6  s*    


z"PostgresSearchQueryCompiler.searchc                 C   s$   | | jjd | }tf ||iS )N__)Zget_attnamer   r{   r   )r<   rW   lookuprO   lhsr>   r>   r?   _process_lookupW  s    z+PostgresSearchQueryCompiler._process_lookupc                 C   s@   |dkrt | }n |dkr.tdd |D }nd S |r<| }|S )NANDr'   c                 S   s   g | ]}t |qS r>   )r   )rE   Zfilr>   r>   r?   r\   `  s     z@PostgresSearchQueryCompiler._connect_filters.<locals>.<listcomp>)r   r'   )r<   filtersZ	connectorZnegatedqr>   r>   r?   _connect_filters[  s    
z,PostgresSearchQueryCompiler._connect_filters)N)NF)N)Nr   )N)rk   rl   rm   ZDEFAULT_OPERATORr   r   r   r@   r   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__r>   r>   r   r?   r   j  s"   

H

%
!r   c                   @   s4   e Zd ZdZeZdd Zdd Zdd Zdd	 Z	d
S )!PostgresAutocompleteQueryCompilerTc                 C   s   |j S r6   )r;   r   r>   r>   r?   r   o  s    z,PostgresAutocompleteQueryCompiler.get_configc                 C   s   | j j S r6   )r   r{   Zget_autocomplete_search_fieldsrG   r>   r>   r?   r   r  s    z=PostgresAutocompleteQueryCompiler.get_search_fields_for_modelc                 C   s   t ddfgS )NZindex_entries__autocompleter   r   r   r>   r>   r?   r   u  s    z3PostgresAutocompleteQueryCompiler.get_index_vectorsc                    s    fdd| j  D S )Nc                    s$   g | ]\}}t | jd ddfqS )rP   )r:   r[   r   )r   r:   r   r   r>   r?   r\   y  s   	zHPostgresAutocompleteQueryCompiler.get_fields_vectors.<locals>.<listcomp>r   r   r>   r   r?   r   x  s    
	z4PostgresAutocompleteQueryCompiler.get_fields_vectorsN)
rk   rl   rm   r   r   r   r   r   r   r   r>   r>   r>   r?   r   k  s   r   c                   @   s2   e Zd ZdddZdd Zdd ZdZd	d
 ZdS )PostgresSearchResultsFc                 C   s:   |rd }d }n| j }| j}| jj| j| j||| jdS )N)r   )r   r   query_compilerr   r   r=   Z_score_field)r<   	for_countr   r   r>   r>   r?   get_queryset  s    z"PostgresSearchResults.get_querysetc                 C   s   t |  S r6   )rK   r   rG   r>   r>   r?   
_do_search  s    z PostgresSearchResults._do_searchc                 C   s   | j dd S )NT)r   )r   countrG   r>   r>   r?   	_do_count  s    zPostgresSearchResults._do_countTc                    s   | j  }|d kr@td  d   d | j jjj d  d| j | j | jd d }|	 j
tddd}t fd	d
|D S )Nz(Cannot facet search results with field "z!". Please add index.FilterField('z') to z.search_fields.rf   rb   r  z-countc                    s   g | ]}|  |d  fqS r  r>   )rE   resultr  r>   r?   r\     s     z/PostgresSearchResults.facet.<locals>.<listcomp>)r   Z_get_filterable_fieldr.   r   r{   rk   r   r   r=   rN   r   r   r   r   )r<   rf   rW   r   resultsr>   r  r?   facet  s8    
  zPostgresSearchResults.facetN)F)rk   rl   rm   r   r   r  Zsupports_facetr  r>   r>   r>   r?   r     s
   
r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )PostgresSearchRebuilderc                 C   s
   || _ d S r6   )indexr<   r	  r>   r>   r?   r@     s    z PostgresSearchRebuilder.__init__c                 C   s   | j   | j S r6   )r	  r   rG   r>   r>   r?   r     s    
zPostgresSearchRebuilder.startc                 C   s   | j jdd d S )NTr   )r	  r   rG   r>   r>   r?   finish  s    zPostgresSearchRebuilder.finishN)rk   rl   rm   r@   r   r  r>   r>   r>   r?   r    s   r  c                       s8   e Zd Z fddZ fddZdd Zdd Z  ZS )	PostgresSearchAtomicRebuilderc                    s&   t  | tj|jd| _d| _d S )Nr   F)r   r@   r
   Zatomicrs   transaction_openedr
  r   r>   r?   r@     s    z&PostgresSearchAtomicRebuilder.__init__c                    s   | j   d| _t  S NT)r
   	__enter__r  r   r   rG   r   r>   r?   r     s    
z#PostgresSearchAtomicRebuilder.startc                 C   s(   | j jdd | jd d d  d| _d S )NTr  F)r	  r   r
   __exit__r  rG   r>   r>   r?   r    s    z$PostgresSearchAtomicRebuilder.finishc                 C   s   | j rd| j_|   d S r  )r  r
   Zneeds_rollbackr  rG   r>   r>   r?   __del__  s    z%PostgresSearchAtomicRebuilder.__del__)rk   rl   rm   r@   r   r  r  r   r>   r>   r   r?   r    s   r  c                       sr   e Zd ZeZeZeZe	Z
eZ fddZdddZdd Zdd	 Zd
d Zdd Zdd Zdd Zdd Z  ZS )PostgresSearchBackendc                    sL   t  | |dd| _|d| _|dd| _|ddrH| j| _d S )NZINDEXdefaultZSEARCH_CONFIGZAUTOCOMPLETE_SEARCH_CONFIGsimpleZATOMIC_REBUILDF)r   r@   getrq   r:   r;   atomic_rebuilder_classrebuilder_class)r<   r   r   r>   r?   r@     s    zPostgresSearchBackend.__init__Nc                 C   s
   t | |S r6   )ro   )r<   r{   rs   r>   r>   r?   get_index_for_model  s    z)PostgresSearchBackend.get_index_for_modelc                 C   s   |  |jj|jjS r6   )r  r   r{   _statedbr   r>   r>   r?   get_index_for_object  s    z*PostgresSearchBackend.get_index_for_objectc                 C   s0   dd t  D D ]}tj j|jd qd S )Nc                 S   s   g | ]}|j d kr|qS )rp   )ru   )rE   rt   r>   r>   r?   r\     s   
z5PostgresSearchBackend.reset_index.<locals>.<listcomp>r   )r	   rS   r   rw   r   alias)r<   rt   r>   r>   r?   reset_index  s    
z!PostgresSearchBackend.reset_indexc                 C   s   d S r6   r>   rz   r>   r>   r?   add_type  s    zPostgresSearchBackend.add_typec                 C   s   d S r6   r>   rG   r>   r>   r?   refresh_index  s    z#PostgresSearchBackend.refresh_indexc                 C   s   |  || d S r6   )r  r   r   r>   r>   r?   add  s    zPostgresSearchBackend.addc                 C   s   |r|  |d || d S )Nr   )r  r   )r<   r{   Zobj_listr>   r>   r?   add_bulk  s    zPostgresSearchBackend.add_bulkc                 C   s   |  || d S r6   )r  r   r   r>   r>   r?   r     s    zPostgresSearchBackend.delete)N)rk   rl   rm   r   Zquery_compiler_classr   Z!autocomplete_query_compiler_classr   Zresults_classr  r  r  r  r@   r  r  r  r  r   r!  r"  r   r   r>   r>   r   r?   r    s   
r  )Ir   collectionsr   	functoolsr   Zdjango.contrib.postgres.searchr   r   r   Z	django.dbr   r   r	   r
   Zdjango.db.modelsr   r   r   r   r   r   r   Zdjango.db.models.constantsr   Zdjango.db.models.functionsr   r   Zdjango.db.models.sql.subqueriesr   Zdjango.utils.encodingr   Zdjango.utils.functionalr   r	  r   r   r   r   modelsr   r   r   r   r    r!   r"   r#   r$   utilsr%   r&   r'   r(   r)   baser+   r,   r-   r.   r0   r   r1   r2   r^   r5   ro   r   r   r   r  r  r  ZSearchBackendr>   r>   r>   r?   <module>   s>   $$  F  33