U
    {h\                     @   s  d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z
mZmZmZmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZ d
dl m!Z!m"Z" d
dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d
dl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4 ddl#m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: G dd dZ;G dd dZ<G dd dZ=G dd de=Z>G dd de2Z?G dd de?Z@G dd  d e3ZAG d!d" d"e1ZBeBZCd#S )$    )OrderedDict)reduce)DEFAULT_DB_ALIASNotSupportedErrorconnectionstransaction)AvgCountFManagerQ	TextField)
LOOKUP_SEP)CastLength)OperationalError)	force_str)cached_property   )AutocompleteFieldRelatedFieldsSearchFieldget_indexed_models)
IndexEntrySQLiteFTSIndexEntry)AndMatchAllNotOrPhrase	PlainText)ADDMULORget_content_type_pk!get_descendants_content_types_pks   )BaseSearchBackendBaseSearchQueryCompilerBaseSearchResultsFilterFieldError   )BM25AndNotLexemeMatchExpressionSearchQueryExpression	normalizec                   @   sb   e Zd ZdZdd Zdd Zdd Zedd	 Zed
d Z	edd Z
edd ZdddZdS )ObjectIndexerzW
    Responsible for extracting data from an object to be inserted into the index.
    c                 C   s   || _ | | _|j| _d S N)objget_search_fieldssearch_fieldsconfig)selfr4   backend r:   v/var/www/dating/data/www/fatepal.com/env/lib/python3.8/site-packages/wagtail/search/backends/database/sqlite/sqlite.py__init__+   s    
zObjectIndexer.__init__c                    s^   t |tr|S t |tr0d fdd|D S t |trVd fdd| D S t|S )Nz, c                 3   s   | ]}  |V  qd S r3   prepare_value.0itemr8   r:   r;   	<genexpr>5   s     z.ObjectIndexer.prepare_value.<locals>.<genexpr>c                 3   s   | ]}  |V  qd S r3   r=   r?   rB   r:   r;   rC   8   s     )
isinstancestrlistjoindictvaluesr   )r8   valuer:   rB   r;   r>   0   s    


zObjectIndexer.prepare_valuec                 c   s   t |tr"|| ||fV  nt |trD|| ||fV  npt |tr||}|d krdd S t |trx| }nt|r| }|g}|D ]"}|j	D ]}| 
||E d H  qqd S r3   )rD   r   r>   	get_valuer   r   r   allcallablefieldsprepare_field)r8   r4   fieldZsub_objZsub_objsZ	sub_fieldr:   r:   r;   rO   <   s     






zObjectIndexer.prepare_fieldc                 C   s   t | jjS )zO
        Returns the value to use as the ID of the record in the index
        )r   r4   pkrB   r:   r:   r;   idU   s    zObjectIndexer.idc                 C   sP   g }| j D ]:}| | j|D ]&\}}t|tr|jdkr|| qq
d|S )z
        Returns all values to index as "title". This is the value of all SearchFields that have the field_name 'title'
        title r6   rO   r4   rD   r   
field_nameappendrG   r8   textsrP   Zcurrent_fieldrJ   r:   r:   r;   rS   \   s    
zObjectIndexer.titlec                 C   sP   g }| j D ]:}| | j|D ]&\}}t|tr|jdks|| qq
d|S )zr
        Returns all values to index as "body". This is the value of all SearchFields excluding the title
        rS   rT   rU   rX   r:   r:   r;   bodyl   s    
zObjectIndexer.bodyc                 C   sF   g }| j D ]0}| | j|D ]\}}t|tr|| qq
d|S )zl
        Returns all values to index as "autocomplete". This is the value of all AutocompleteFields
        rT   )r6   rO   r4   rD   r   rW   rG   rX   r:   r:   r;   autocomplete|   s    

zObjectIndexer.autocompleteFc                 C   s&   dd |D }dd |D }d |S )zW
        Converts an array of strings into a SearchVector that can be indexed.
        c                 S   s   g | ]\}}|  |fqS r:   )stripr@   textweightr:   r:   r;   
<listcomp>   s     z+ObjectIndexer.as_vector.<locals>.<listcomp>c                 S   s   g | ]\}}|r||fqS r:   r:   r]   r:   r:   r;   r`      s      rT   )rG   )r8   rY   Zfor_autocompleter:   r:   r;   	as_vector   s    zObjectIndexer.as_vectorN)F)__name__
__module____qualname____doc__r<   r>   rO   r   rR   rS   rZ   r[   ra   r:   r:   r:   r;   r2   &   s   



r2   c                   @   sh   e Zd ZdddZdd Zdd Zdd	d
Zdd Zdd Zdd Z	dd Z
dd Zdd Zdd ZdS )IndexNc                 C   sV   || _ | j j| _|d krtn|| _t| j | _| jjdkrBtdt	j
| j| _d S )NsqlitezCYou must select a SQLite database to use the SQLite search backend.)r9   
index_namenamer   db_aliasr   
connectionvendorr   r   _default_managerusingentries)r8   r9   rj   r:   r:   r;   r<      s    
zIndex.__init__c                 C   s   d S r3   r:   r8   modelr:   r:   r;   	add_model   s    zIndex.add_modelc                 C   s   d S r3   r:   rB   r:   r:   r;   refresh   s    zIndex.refreshFc                 C   sn   | j jtddjddtdd }|r4| j }n| j jdd}|jtddjddj|td d d	S )
a  
        Refreshes the value of the title_norm field.

        This needs to be set to 'lavg/ld' where:
         - lavg is the average length of titles in all documents (also in terms)
         - ld is the length of the title field in this document (in terms)
        rS   )title_lengthr   )Ztitle_length__gtrt   Ztitle_length__avg      ?)Z
title_normN)ro   annotater   filterZ	aggregater   updater
   )r8   fullZlavgro   r:   r:   r;   _refresh_title_norms   s     

zIndex._refresh_title_normsc                 C   sN   |j | jjtdt dd}t|}| jj	|dj
|d}|  d S )NrQ   	object_idr|   )Zcontent_type_id__inZobject_id__in)rm   rn   rj   rv   r   r   rI   r%   ro   rw   excludedelete)r8   rq   Zexisting_pksZcontent_types_pksZstale_entriesr:   r:   r;   delete_stale_model_entries   s    
z Index.delete_stale_model_entriesc                 C   s"   t  D ]}|jjs| | qd S r3   )r   _metaparentsr   rp   r:   r:   r;   delete_stale_entries   s    
zIndex.delete_stale_entriesc                 C   s   |  |jj|g d S r3   )	add_itemsr   rq   r8   r4   r:   r:   r;   add_item   s    zIndex.add_itemc              
   C   s   i }|D ]}|j |j|jf||j< q| jj|d}t|j| djddd}|D ]*}|| \}}	}
|j|dj	||	|
d qTg }| D ]2}||kr|| \}}	}
|
t||||	|
d q| j| |   d S )	N)content_type_idr}   r|   T)Zflatr{   )rS   r[   rZ   )r   r|   rS   r[   rZ   )rS   r[   rZ   rR   ro   rw   	frozensetkeysZvalues_listrx   rW   r   Zbulk_createrz   )r8   content_type_pkindexersZids_and_dataZindexerZindex_entries_for_ctZindexed_idsZ
indexed_idrS   r[   rZ   Zto_be_createdr|   r:   r:   r;   add_items_update_then_create   sF       
z"Index.add_items_update_then_createc                    sB   |  }|sd S  fdd|D }|r>t|} j}||| d S )Nc                    s   g | ]}t | jqS r:   )r2   r9   r@   r4   rB   r:   r;   r`     s     z#Index.add_items.<locals>.<listcomp>)r5   r$   r   )r8   rq   objsr6   r   r   Zupdate_methodr:   rB   r;   r     s    zIndex.add_itemsc                 C   s   |j  j| jd d S )Nrn   )Zindex_entriesrL   _raw_deleterj   )r8   rA   r:   r:   r;   delete_item  s    zIndex.delete_itemc                 C   s   | j S r3   )ri   rB   r:   r:   r;   __str__  s    zIndex.__str__)N)F)rb   rc   rd   r<   rr   rs   rz   r   r   r   r   r   r   r   r:   r:   r:   r;   rf      s   

'rf   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )SQLiteSearchRebuilderc                 C   s
   || _ d S r3   )indexr8   r   r:   r:   r;   r<     s    zSQLiteSearchRebuilder.__init__c                 C   s   | j   | j S r3   )r   r   rB   r:   r:   r;   start  s    
zSQLiteSearchRebuilder.startc                 C   s   | j jdd d S )NTry   )r   rz   rB   r:   r:   r;   finish  s    zSQLiteSearchRebuilder.finishN)rb   rc   rd   r<   r   r   r:   r:   r:   r;   r     s   r   c                       s8   e Zd Z fddZ fddZdd Zdd Z  ZS )	SQLiteSearchAtomicRebuilderc                    s&   t  | tj|jd| _d| _d S )Nr   F)superr<   r   Zatomicrj   transaction_openedr   	__class__r:   r;   r<   $  s    z$SQLiteSearchAtomicRebuilder.__init__c                    s   | j   d| _t  S NT)r   	__enter__r   r   r   rB   r   r:   r;   r   )  s    
z!SQLiteSearchAtomicRebuilder.startc                 C   s(   | j jdd | jd d d  d| _d S )NTr   F)r   rz   r   __exit__r   rB   r:   r:   r;   r   .  s    z"SQLiteSearchAtomicRebuilder.finishc                 C   s   | j rd| j_|   d S r   )r   r   Zneeds_rollbackr   rB   r:   r:   r;   __del__4  s    z#SQLiteSearchAtomicRebuilder.__del__)rb   rc   rd   r<   r   r   r   __classcell__r:   r:   r   r;   r   #  s   r   c                       s   e Zd ZdZdZeZddgZ fddZdd Z	d	d
 Z
d!ddZd"ddZd#ddZd$ddZdd Zdd Zdd Zd%ddZdd Zdd  Z  ZS )&SQLiteSearchQueryCompilerANDFrS   rZ   c                    sD   t  j||   jd kr( _n fddjD _d S )Nc                    s   i | ]}|j | d qS ))rN   )get_search_field)r@   field_lookupZlocal_search_fieldsr8   r:   r;   
<dictcomp>L  s     z6SQLiteSearchQueryCompiler.__init__.<locals>.<dictcomp>)r   r<   get_search_fields_for_modelrN   r6   )r8   argskwargsr   r   r;   r<   A  s    
z"SQLiteSearchQueryCompiler.__init__c                 C   s   |j S r3   r7   r8   r9   r:   r:   r;   
get_configS  s    z$SQLiteSearchQueryCompiler.get_configc                 C   s   | j j S r3   )querysetrq   Zget_searchable_search_fieldsrB   r:   r:   r;   r   V  s    z5SQLiteSearchQueryCompiler.get_search_fields_for_modelNc                 C   s~   |d kr| j }t|kr(|td\}}nd }|D ]H}t|| jrR|j|krR|  S t|tr0|j|kr0| ||j  S q0d S )Nr+   )	r6   r   splitrD   TARGET_SEARCH_FIELD_TYPErV   r   r   rN   )r8   r   rN   Zsub_field_namerP   r:   r:   r;   r   Y  s    
z*SQLiteSearchQueryCompiler.get_search_fieldc                    s$  t |trp|j }|sdS | }t|jd}|D ],}t|}|j dkrZ||M }q6||O }q6t	| dS t |t
rt	|jS t |trj|j d}j|j d}	||	d}
|
S t |ttfr fdd|jD }t |t}|rtdd	 |S td
d	 |S td|jj dS )z~
        Takes a SearchQuery and returns another SearchQuery object, which can be used to construct the query in SQL.
        N)prefixr   r   NOTc                    s   g | ]}j | d qS r   )build_search_query_contentr@   subqueryr7   r8   r:   r;   r`     s   zHSQLiteSearchQueryCompiler.build_search_query_content.<locals>.<listcomp>c                 S   s   | |@ S r3   r:   abr:   r:   r;   <lambda>      zFSQLiteSearchQueryCompiler.build_search_query_content.<locals>.<lambda>c                 S   s   | |B S r3   r:   r   r:   r:   r;   r     r   3`%s` is not supported by the SQLite search backend.)rD   r    Zquery_stringr   popr.   LAST_TERM_IS_PREFIXoperatorupperr0   r   r-   r   
subquery_a
subquery_b_combiner   r   
subqueriesr   NotImplementedErrorr   rb   )r8   queryr7   ZtermsZ	last_termZlexemestermZ
new_lexemer   r   Zcombined_queryZsubquery_lexemesZis_andr:   r   r;   r   o  sR    

 




  
z4SQLiteSearchQueryCompiler.build_search_query_contentc                 C   sD   t |tr|S t |tr2|j}t| j||d}n| j||d}|S )Nr   )rD   r   r   r   build_search_queryr   )r8   r   r7   Zunwrapped_queryZbuilt_queryr:   r:   r;   r     s    

z,SQLiteSearchQueryCompiler.build_search_queryru   c                    s   t |tttfr*t } dkr&| 9 }|S t |trVt fdd|jD d S t |trt	 fdd|jD t
|jpd S td|jj d S )Nru   c                 3   s$   | ]}d j | d V  qdS )r+   r7   boostNbuild_tsrankr   r   r7   r8   vectorr:   r;   rC     s   z9SQLiteSearchQueryCompiler.build_tsrank.<locals>.<genexpr>r+   c                 3   s    | ]}j | d V  qdS )r   Nr   r   r   r:   r;   rC     s   r   )rD   r   r    r   r,   r   r"   r   r   r!   lenr   r   rb   )r8   r   r   r7   r   rank_expressionr:   r   r;   r     s.    

z&SQLiteSearchQueryCompiler.build_tsrankc                 C   s   t dt dft ddfgS )NZindex_entries__titleZindex_entries__title_normZindex_entries__bodyru   r
   rB   r:   r:   r;   get_index_vectors  s    
z+SQLiteSearchQueryCompiler.get_index_vectorsc                 C   s   |   S r3   )r   rB   r:   r:   r;   get_search_vectors  s    z,SQLiteSearchQueryCompiler.get_search_vectorsc                    s:    fdd|D }|d }|dd  D ]}||7 }q(|S )Nc                    s&   g | ]\}}j |j d | qS r   )r   r   )r@   r   r   r   r:   r;   r`     s   zDSQLiteSearchQueryCompiler._build_rank_expression.<locals>.<listcomp>r   r+   r:   )r8   vectorsr7   Zrank_expressionsr   Zother_rank_expressionr:   r   r;   _build_rank_expression  s    
z0SQLiteSearchQueryCompiler._build_rank_expressionc              
   C   s  t | j}t|tr"| j|| S t|trBt|jtrB| j S t|trX|j}d}nd}| j||d}| 	 }| 
||}	|d d }
|dd  D ]\}}|
|dd}
qt| jp| j|}tj|djt| jjd}| jr|t  }n|jjs|d	}td
}	ddlm} ddlm} |tj|d}zdd |D }W nZ t k
r } z:t t!|d |"|jd  t!|"|jd  |W 5 d }~X Y nX |s| jj|d}n| jj#|d}|d k	r|j$f ||	i}||| S )NTFr   r   r+   rT   index_entry)Zindex_entry__content_type__inz-pkrQ   )rk   )InsertQueryc                 S   s   g | ]}|j jqS r:   )r   r|   r   r:   r:   r;   r`   "  s    z4SQLiteSearchQueryCompiler.search.<locals>.<listcomp>z The original query was: )Zid__in)%r1   r   rD   r   r   r   r   noner   r   r   r   r/   rN   FTS_TABLE_FIELDSr   objectsrw   Zselect_relatedr%   rq   Zorder_by_relevanceorder_byr,   descr
   	django.dbrk   Zdjango.db.models.sql.subqueriesr   r   Zget_compilerr   rE   compiler~   rv   )r8   r7   r   stopscore_fieldZnormalized_querynegatedZsearch_queryr   r   Zcombined_vectorr   r   exprr   r   rk   r   compilerZobj_idser:   r:   r;   search  s    

 

   




z SQLiteSearchQueryCompiler.searchc                 C   s$   | | jjd | }tf ||iS )N__)Zget_attnamer   rq   r   )r8   rP   lookuprJ   lhsr:   r:   r;   _process_lookup;  s    z)SQLiteSearchQueryCompiler._process_lookupc                 C   s@   |dkrt | }n |dkr.tdd |D }nd S |r<| }|S )Nr   r#   c                 S   s   g | ]}t |qS r:   )r   )r@   Zfilr:   r:   r;   r`   D  s     z>SQLiteSearchQueryCompiler._connect_filters.<locals>.<listcomp>)r   r#   )r8   filtersZ	connectorr   qr:   r:   r;   _connect_filters?  s    
z*SQLiteSearchQueryCompiler._connect_filters)N)N)N)Nru   )N)rb   rc   rd   ZDEFAULT_OPERATORr   r   r   r   r<   r   r   r   r   r   r   r   r   r   r   r   r   r   r:   r:   r   r;   r   ;  s"   

8


Tr   c                   @   s2   e Zd ZdZeZdgZdd Zdd Zdd Z	d	S )
SQLiteAutocompleteQueryCompilerTr[   c                 C   s   |j S r3   )autocomplete_configr   r:   r:   r;   r   T  s    z*SQLiteAutocompleteQueryCompiler.get_configc                 C   s   | j j S r3   )r   rq   Zget_autocomplete_search_fieldsrB   r:   r:   r;   r   W  s    z;SQLiteAutocompleteQueryCompiler.get_search_fields_for_modelc                 C   s   t ddfgS )NZindex_entries__autocompleteru   r   rB   r:   r:   r;   r   Z  s    z1SQLiteAutocompleteQueryCompiler.get_index_vectorsN)
rb   rc   rd   r   r   r   r   r   r   r   r:   r:   r:   r;   r   O  s   r   c                   @   s2   e Zd ZdddZdd Zdd ZdZd	d
 ZdS )SQLiteSearchResultsFc                 C   s:   |rd }d }n| j }| j}| jj| j| j||| jdS )N)r   )r   r   query_compilerr   r   r9   Z_score_field)r8   	for_countr   r   r:   r:   r;   get_queryset_  s    z SQLiteSearchResults.get_querysetc                 C   s   t |  S r3   )rF   r   rB   r:   r:   r;   
_do_searchn  s    zSQLiteSearchResults._do_searchc                 C   s   | j dd S )NT)r   )r   countrB   r:   r:   r;   	_do_countq  s    zSQLiteSearchResults._do_countTc                    s   | j  }|d kr@td  d   d | j jjj d  d| j | j | jd d }|	 j
tddd}t fd	d
|D S )Nz(Cannot facet search results with field "z!". Please add index.FilterField('z') to z.search_fields.rV   rQ   r   z-countc                    s   g | ]}|  |d  fqS r   r:   )r@   resultr   r:   r;   r`     s     z-SQLiteSearchResults.facet.<locals>.<listcomp>)r   Z_get_filterable_fieldr*   r   rq   rb   r   r   r9   rI   rv   r	   r   r   )r8   rV   rP   r   resultsr:   r   r;   facetv  s8    
  zSQLiteSearchResults.facetN)F)rb   rc   rd   r   r   r   Zsupports_facetr   r:   r:   r:   r;   r   ^  s
   
r   c                       sr   e Zd ZeZeZeZe	Z
eZ fddZdddZdd Zdd	 Zd
d Zdd Zdd Zdd Zdd Z  ZS )SQLiteSearchBackendc                    s>   t  | |dd| _d | _d | _|ddr:| j| _d S )NZINDEXdefaultZATOMIC_REBUILDF)r   r<   getrh   r7   r   atomic_rebuilder_classrebuilder_class)r8   paramsr   r:   r;   r<     s    zSQLiteSearchBackend.__init__Nc                 C   s
   t | |S r3   )rf   )r8   rq   rj   r:   r:   r;   get_index_for_model  s    z'SQLiteSearchBackend.get_index_for_modelc                 C   s   |  |jj|jjS r3   )r  r   rq   _statedbr   r:   r:   r;   get_index_for_object  s    z(SQLiteSearchBackend.get_index_for_objectc                 C   s0   dd t  D D ]}tj j|jd qd S )Nc                 S   s   g | ]}|j d kr|qS )rg   )rl   )r@   rk   r:   r:   r;   r`     s   
z3SQLiteSearchBackend.reset_index.<locals>.<listcomp>r   )r   rL   r   rm   r   alias)r8   rk   r:   r:   r;   reset_index  s    
zSQLiteSearchBackend.reset_indexc                 C   s   d S r3   r:   rp   r:   r:   r;   add_type  s    zSQLiteSearchBackend.add_typec                 C   s   d S r3   r:   rB   r:   r:   r;   refresh_index  s    z!SQLiteSearchBackend.refresh_indexc                 C   s   |  || d S r3   )r  r   r   r:   r:   r;   add  s    zSQLiteSearchBackend.addc                 C   s   |r|  |d || d S )Nr   )r  r   )r8   rq   Zobj_listr:   r:   r;   add_bulk  s    zSQLiteSearchBackend.add_bulkc                 C   s   |  || d S r3   )r  r   r   r:   r:   r;   r     s    zSQLiteSearchBackend.delete)N)rb   rc   rd   r   Zquery_compiler_classr   Z!autocomplete_query_compiler_classr   Zresults_classr   r  r   r   r<   r  r  r  r	  r
  r  r  r   r   r:   r:   r   r;   r     s   
r   N)Dcollectionsr   	functoolsr   r   r   r   r   r   Zdjango.db.modelsr   r	   r
   r   r   r   Zdjango.db.models.constantsr   Zdjango.db.models.functionsr   r   Zdjango.db.utilsr   Zdjango.utils.encodingr   Zdjango.utils.functionalr   r   r   r   r   r   modelsr   r   r   r   r   r   r   r   r    utilsr!   r"   r#   r$   r%   baser'   r(   r)   r*   r,   r-   r.   r/   r0   r1   r2   rf   r   r   r   r   r   r   ZSearchBackendr:   r:   r:   r;   <module>   s4      
m   32