
    	]jK)                     @   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z#  ejH                  e%      Z&ddZ'ddZ(ddZ)d Z*d Z+ddZ,d Z-ddZ.d Z/y)    N)AsyncMigrationStatus)start_job_async_or_sync)batchbatched_iterator)iterate_queryset)ExportMixin)
DataExport)ExportDataSerializer)TaskQuerySet)settings)CountFQ)Organization)Project)
Annotation
PredictionTaskc           
      D   t        j                  t              }t        j                  j                  d      j                  dd      }|D ]A  }|j                  d|        t        t        || |dd|       |j                  d	| d
       C |j                  d       y )Nz-ididTflatz+Start recalculating stats for Organization criticaliQ )redis
queue_namejob_timeoutmigration_namezOrganization z stats were recalculatedz#All organizations were recalculated)
logging	getLogger__name__r   objectsorder_byvalues_listdebugr   redis_job_for_calculation)from_scratchr   r   loggerorganization_idsorg_ids         F/root/env/lib/python3.12/site-packages/label_studio/tasks/functions.pycalculate_stats_all_orgsr+      s    x(F $++44U;GGSWGX" GB6(KL 	 %!!)	
 	}VH,DEFG  LL67    c                    t        j                         }|j                  t         j                         t        j                  t
        j                        }|j                  t         j                         t        j                  d      }|j                  |       |j                  |       t        j                  j                  |       j                  d      j                  ddd      }|D ]  }t        j                  j!                  |d   |t        j"                        }t$        j                  j                  |d         }	|j'                  d	|d    d
|d    d|	j)                          d|d           t+        |	|      }
t        j,                  |_        |
|	j)                         d|_        |j3                          |j'                  d|d    d
|d    dt5        |
       d        y)z
    Recalculate counters for projects list
    :param org_id: ID of organization to recalculate
    :param from_scratch: Start calculation from scratch or skip calculated tasks
    z4%(asctime)s - %(name)s - %(levelname)s - %(message)s)organization_idz-updated_atr   
updated_attitle
project_idnamestatusr2   z Start processing stats project <> () with task count z and updated_at )r&   )tasks_processedtotal_project_tasksz%End processing counters for project <z), processed z tasksN)r   r   setLevelDEBUGStreamHandlersysstdout	FormattersetFormatter
addHandlerr   r!   filterr"   valuesr   createSTATUS_STARTEDr   r$   countupdate_tasks_countersSTATUS_FINISHEDr4   metasavestr)r)   r&   r   r'   handler	formatterproject_dictsproject_dict	migrationproject_tasks
task_counts              r*   r%   r%   1   s     F
OOGMM"##CJJ/GW]]#!!"XYI#
g 	v6	-	 	

  & 
(0077#D)'66 8 
	
 ++|D7I+J.|G/D.ESVZI[H\ ],22455ElS_F`Eac	

 +=|T
/??	-7P]PcPcPef	3L4I3J#l[_N`Ma bZ)1	
#
r,   c           	         t        j                  t              }t        j                  j                  |       }|j                         }t        j                  |      D cg c]  }|d   	 }}||v s
J d|        t        j                  j                  |      j                  d      j                  dd      }|j                  d|j                   d	|j                   d
|j!                          d       t#        |t$              rt'        j(                  |      }t+        j,                  |      }	g }
t/        |d      D ]  }|
t1        |fddi|	j2                  z  }
 t        j4                  ||
|t6        j8                  i       \  }}}t:        j<                  j?                  |      r t:        j<                  jA                  ||      n|}tC        |d      5 }tE        jF                  ||       d d d        |jI                          |j                  d|j                   d	|j                   d| d       |S c c}w # 1 sw Y   OxY w)N)r   r3   z+Export format is not supported, please use )projectrT   annotationspredictionszStart exporting project <r6   r7   .i  manyTwbzEnd exporting project <z) in z format.)%r   r   r    r   r!   getupperr	   get_export_formatsr   rB   select_relatedprefetch_relatedr$   r0   r   rF   
isinstancerK   jsonloadsr   _get_export_serializer_optionr   r
   datagenerate_export_filer   CONVERTER_DOWNLOAD_RESOURCESospathisdirjoinopenshutilcopyfileobjclose)r2   export_formatrg   serializer_contextr'   rT   ssupported_formatstask_idsserializer_optionstasks	_task_idsexport_file_filenamefilepathfiles                    r*   export_projectr{   `   s   x(Foo!!Z!0G!'')M,6,I,I',RSq6SS--p1\]n\o/pp- 	G,;;IFWWXegtu  LL,W]]O3wzzlJ\]e]k]k]m\nnopq $c*!ZZ(:;$BBCUV E8T* W	%iQdQ>PQVVVW  *>>x'L'Lb KH
 02ww}}T/Brww||D(+H	h	 .;-.
LL*7==/WZZLm_\defOA T4. .s   !H9H>>Ic                 d    t         j                  j                  |       j                  |        y )Ntask__project_idr5   )r   r!   rB   updater5   s    r*   _fill_annotations_projectr      s'    z:AAZAXr,   c                      t         j                  d       t        j                  j	                         j                  dd      } | D ]  }t        t        |        t         j                  d       y )Nz0Start filling project field for Annotation modelr   Tr   z3Finished filling project field for Annotation model)r'   infor   r!   allr#   r   r   )project_idsr2   s     r*   fill_annotations_projectr      s\    
KKBC//%%'33Dt3DK! G
 9:FG KKEFr,   c                    t         j                  j                         j                  dd      }|D ]  }t        j                  j                  || t        j                        }t        j                  j                  |      j                  |      }t        j                  |_        |t        j                  j                  |      j                         d|_        |j                           y )Nr   Tr   r1   r}   r5   )predictions_processedtotal_project_predictions)r   r!   r   r#   r   rD   rE   r   rB   r   rH   r4   rF   rI   rJ   )r   r   r2   rP   updated_counts        r*   _fill_predictions_projectr      s    //%%'33Dt3DK! 
(0077!'66 8 
	 #**11:1NUUakUl/??	%2)3););)B)Bj)B)Y)_)_)a
	 	r,   c                 z    t         j                  d       t        t        |        t         j                  d       y )Nz0Start filling project field for Prediction modelr   z3Finished filling project field for Prediction model)r'   r   r   r   r   s    r*   fill_predictions_projectr      s'    
KKBC5nU
KKEFr,   c                    t        ddt        d            }t        ddt        d            }t        dd      }t        | t              rYt	        |       dkD  rKt        | d   t
              r8t
        j                  j                  | D cg c]  }|j                   c}	      } t        | t              rC| j                         r3t        | d   t              r t
        j                  j                  | 	      } |s5| j                  t        d
      t        d      z  t        d      z        } | j                  dd      j                  d      j                  ddd       | j                  t        d      t        d      z        } | j                  |||      } d}t!        | j#                  dddd      t$        j&                        }t)        |t$        j&                        D ]  }g }	|D ]F  }|j*                  |_        |j.                  |_        |j2                  |_        |	j7                  |       H |	sSt
        j                  j9                  |	g dt$        j&                         |t	        |	      z  } |S c c}w )z
    Update tasks counters for the passed queryset of Tasks
    :param queryset: Tasks to update queryset
    :param from_scratch: Skip calculated tasks
    :return: Count of updated tasks
    rU   TFannotations__was_cancelleddistinctrB   rV   r   r   id__in)total_annotations__gt)cancelled_annotations__gt)total_predictions__gt)annotations__isnullpredictions__isnullr   )total_annotationscancelled_annotationstotal_predictions)r   )r   )new_total_annotationsnew_cancelled_annotationsnew_total_predictionsr   r   r   )
chunk_size)
batch_size)r   r   r_   listlenr   r!   rB   r   r   existsintexcluder"   r   annotater   onlyr   
BATCH_SIZEr   r   r   r   r   r   r   appendbulk_update)
querysetr&   r   r   r   taskr   tasks_iterator_batch
batch_lists
             r*   rG   rG      sE    md1`eCfg!-$qdhGijmd;(D!c(ma&7JxPQ{TX<Y<<&&8.L4tww.L&M(L)hoo.?JxXY{\_D`<<&&h&7##A&Q)GG!bcJdd
 OO$OGPPQUV]]1 ^ 
 qU;aTY>ZZ[H  /"7/ ! H M%d/1HJ]^&&N
 #>83F3FG -
 	$D%)%?%?D")-)G)GD&%)%?%?D"d#		$ LL$$S#.. % 
 S_,M-  _ /Ms   Jc                    | sy t         j                  }t        dt        |       |      D ],  }| |||z    }|j                  |j
                  j                  k(  rt        dd      }nt        ddt        d            }t        j                  j                  ||      j                  |	      }t        |j                  t        d
            j                  dd            }t        j                  j                  ||      j!                  d       t        j                  j                  ||      j#                  |      j!                  d       / y )Nr   annotations__completed_byTr   Fr   r   )r   rT   )annotator_countoverlap)annotator_count__gter   r   )
is_labeledr   )r   r   ranger   
skip_queue	SkipQueueIGNORE_SKIPPEDr   r   r   r!   rB   r   r   r   r#   r   r   )	tasks_idsrT   r   i	batch_idsannotator_count_exprtasks_qsfinished_task_idss           r*   !bulk_update_is_labeled_by_overlapr      sD   $$J 1c)nj1 
a!j.1	 !2!2!A!AA#()Dt#T #(+E:$  <<&&i&IRR0 S 

 !a	l!S!_!_`dko!_!pq 	#4gFMMY]M^9g>FFN_F`gg 	h 	
-
r,   )0018_manual_migrate_counters)N)0043_auto_20230825)T)0r`   r   rf   rk   r=   core.modelsr   
core.redisr   core.utils.commonr   r   core.utils.iteratorsr   data_export.mixinsr   data_export.modelsr	   data_export.serializersr
   data_manager.managersr   django.confr   django.db.modelsr   r   r   organizations.modelsr   projects.modelsr   tasks.modelsr   r   r   r   r    r'   r+   r%   r{   r   r   r   r   rG   r    r,   r*   <module>r      s      	  
 , . 5 1 * ) 8 .   ( ( - # 5 5			8	$82,
^&RYG&G;|
r,   