
    	]jc                        d dl Z d dlmZ d dlmZmZmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZ d d	lmZ d d
lmZ d dlm Z  d dl!m"Z"m#Z# d dl$m%Z%  e jL                  e'      Z(de%de de)fdZ* eejV                        Z, eejZ                        xs e*Z.de%de/fdZ0d,dee#   de%dee#df   fdZ1dee#   dee#df   fdZ2dee#   de de%dee#df   fdZ3dee#   deee#df   ee#   f   fdZ4dee#   de%de dee#df   fdZ5dee#   de dee/   de%dee#   dee#df   fdZ6de%de de)fdZ7dee#   dee#   fdZ8de%de dee#   d ee)df   d!e9deee#   ee/   e9e)f   fd"Z:d# Z;de%de d$ed ee)df   d%e)deee#df   e)e9f   fd&Z<d' Z=d( Z>d$ee#   dee/   dede%de d!e9deee#df   e9f   fd)Z?	 d,de%dede d*ee)df   d ee)df   deee#df   e9f   fd+Z@y)-    N)Counter)ListTupleUnion)flag_set)conditional_atomicdb_is_not_sqlite	load_func)
fast_first)settings)	CaseCountExistsFMaxOuterRefQQuerySetWhen)DecimalField)add_stream_history)Project)
AnnotationTask)Useruserprojectreturnc                 ,    t        |j                        S N)boolannotator_evaluation_enabled)r   r   s     S/root/env/lib/python3.12/site-packages/label_studio/projects/functions/next_task.py_lso_should_attempt_gt_firstr$      s    4455    c                 `    t         j                  }t        d|       rt         j                  }|S )N9fflag_fix_back_dev_4185_next_task_additional_logging_longr   )loggingDEBUGr   INFO)r   levels     r#   get_next_task_logging_levelr-      s%    MMEKRVWLr%   
task_queryc                    | j                  d      j                  d      d t        j                   D ]R  }	 t        j
                  j                  d      j                  |j                        }|j                  |      s|c S T y # t        j                  $ r1 t        j                  dj                  |j                               Y w xY w)N?idTskip_lockedpkTask with id {} locked)order_byonlyr   RANDOM_NEXT_TASK_SAMPLE_SIZEr   objectsselect_for_updategetr1   has_lockDoesNotExistloggerdebugformat)r.   r   upper_limittasks       r#   _get_random_unlockedrD   &   s    ##C(--d34[h6[6[\ C	C<<11d1CGG477GSD==& 'C
    	CLL188AB	Cs   ABACCtasks_queryc                 :   | j                  dd      D ]H  }	 t        j                  j                  d      j	                  |      }|j                  |      s|c S J y # t        j                  $ r' t        j                  dj                  |             Y w xY w)Nr1   Tflatr2   r4   r6   )
values_listr   r:   r;   r<   r=   r>   r?   r@   rA   )rE   r   task_idrC   s       r#   _get_first_unlockedrK   0   s    **4d*; C	C<<11d1CGG7GSD==& 'C    	CLL188AB	Cs   AA  7BBtasksc                     t        ||      syt        |       j                  d      }|j                         r1|j                  |j
                  k(  rt        ||      S t        ||      S y)z`Returns task from ground truth set for onboarding. Continuous GT flows through regular ordering.NThas_ground_truths)!should_attempt_ground_truth_first_annotate_has_ground_truthsfilterexistssamplingSEQUENCErK   rD   )rL   r   r   #not_solved_tasks_with_ground_truthss       r#   _try_onboarding_ground_truthrW   <   sj     -T7;*Ee*L*S*Sfj*S*k'*113w///&'JDQQ#$GNNr%   c                 v    | j                  d      }|j                         rd|fS d| j                  d      fS )z;Filter out tasks without overlap (doesn't return next task)   )overlap__gtN)overlap)rR   rS   )rL   tasks_with_overlaps     r#   _try_tasks_with_overlapr]   J   sA    !4  "'''U\\!\,,,r%   c                 P   |j                   rt        |       } | j                  d      } | j                  t	        dt        |                   } | j                  t        d            d   }|d	k(  s|y
| j                  |      }|j                         rt        ||      }|S y
)zlTry to find tasks with maximum amount of annotations, since we are trying to label tasks as fast as possibleFrN   annotations)annotations__completed_by)rR   )annotations_countra   annotations_count__maxr   N)
r"   rQ   rR   annotater   r   	aggregater   rS   rD   )rL   r   r   max_annotations_count
candidatesresults         r#   _try_breadth_firstrh   S   s     ++ ,E2u5NNU=!fjJkIk-lNmE!OOC0C,DEF^_!%:%B 0EFJ%j$7r%   user_solved_tasks_arrayprepared_tasksc                 H   | j                  |j                        }|j                         r6t        j	                  d       |j                  |      j                  t        d            j                  dd      }t        |      }|j                         D cg c]  \  }}t        ||	       }	}}|j                         }
|	r7|j                  t        |	d
t               d      }|j                  dd      }n|j                  d      }|j                         j                         }|dkD  r"|
d
kD  rt!        ||t#        |dz   |
            }|S t%        ||      }|S t        j	                  dt'        |j                         d       t!        | |      }|S c c}}w )N)predictions__model_versionzUse uncertainty samplingpk__inpredictions__cluster)clusterrp   TrG   )ro   thenr   )defaultoutput_field)cluster_num_solvedrt   predictions__scorerY   )rB   zQUncertainty sampling fallbacks to random sampling (current project.model_version=))rR   model_versionrS   r?   r@   rc   r   rI   r   itemsr   countr   r   r7   
annotatorsrD   minrK   str)rL   r   ri   r   rj   task_with_current_predictionsuser_solved_clusterskvcluster_num_solved_map"num_tasks_with_current_predictionspossible_next_tasksnum_annotators	next_tasks                 r#   _try_uncertainty_samplingr   o   s    %*LLGLaLaL$b!$++-/0 !!)@!AXc"89X:[[. 	
  '';<SgSmSmSo!p41a$AA"F!p!p .K-P-P-R*!,I,R,R#')?YeYg#h -S -) #@"H"HI]_s"t"?"H"HI]"^ ++-335A"Dq"H,#Ts>A;MOq7rI  ,,?FI  	..1'2G2G.H-IL	
 )5	9 "qs   Fc                    |j                   syt        |dd      }|syt        |dd      }t        |dd      }||z   }|dk(  ryt        j                  j	                  |d      j                  d	d
      }t        j                  j	                  || d|      j                  d      j                         j                         }||k  S )z
    Check if GT tasks should be included in the task pool for this user.

    Returns True if user hasn't reached their GT limit (onboarding + continuous tasks).
    Flse_projectN%annotator_evaluation_onboarding_tasksr   %annotator_evaluation_continuous_tasksT)r   annotations__ground_truthr5   rG   )r   completed_bywas_cancelledtask_id__inrJ   )
r"   getattrr   r:   rR   rI   r   valuesdistinctry   )r   r   r   onboarding_taskscontinuous_taskstotal_gt_limitgt_task_idsuser_gt_completeds           r#   _should_include_gt_tasksr      s     //'=$7K{,SUVW{,SUVW%(88N ,,%%gQU%VbbcgnrbsK 	!!'TYgr!s						  ~--r%   c                     t         j                  j                  t        d      d      }| j	                  t        |            S )Nr5   T)rC   ground_truthrN   )r   r:   rR   r   rc   r   )rL   r   s     r#   rQ   rQ      s7    %%,,(4.t,TL>>F<,@>AAr%   assigned_flag
queue_infoc           	         | j                   j                  |d      }|j                         j                  dd      }|j	                  |      }| j
                  j                  |d      }|j                         r3|j                         j                  dd      }|j	                  |      }d}	t        | |      }
|s?t        |dd       }|r|j                  t        r| j                  |      rt        |      }|j                  t        d	d
            }t        |j                  d      t        d      z  }t        t        d      |j                   xs dz         }|
r-t#        |      }|j                  t        d      ||z  z        }n|j                  ||z        }t%        ||      \  }	}nH|
r4t#        |      }|j                  t        d      t        d      z        }n|j                  d      }t'        d      sA|j(                  r5|	s3t*        j-                  d|  d       t/        |      \  }}||rdnddz   z  }t'        d|       r|st        |dd       }t        | dd      xs t        | dd      }|rt        |dd      r|rd}|j                  |j                   xs d}t0        j2                  j                  |      j                  t        d	t        dd      d             j                  t        d      |z   !      j                  d"d      }|j	                  |      }||||	fS )#NF)r   task__isnulltask__pkTrG   rm   )task__projectwas_postponedr   r`   )r   )rz   _agreement__lt
is_labeled)r   r[   r   )annotators__ltrN   @fflag_fix_back_lsdv_4523_show_overlap_first_order_27022023_shortUser=( tries overlap first from prepared tasks &  Show overlap first&fflag_feat_all_fit_1304_strict_overlapr(   is_annotatoris_reviewerstrict_task_overlap)r   )annotations__was_cancelledr   )rR   r   )distinct_annotators)distinct_annotators__gter5   )r_   rR   r   rI   excludedraftsrS   r   r   agreement_thresholdget_tasks_agreement_querysetis_project_annotatorrc   r   r   r   $max_additional_annotators_assignablerQ   _prioritize_low_agreement_tasksr   show_overlap_firstr?   r@   r]   r   r:   )r   r   rj   r   r   ri   not_solved_taskspostponed_draftsuser_postponed_tasksprioritized_on_agreement
include_gtr   qslow_agreement_predcapacity_pred_is_restricted_rolemax_additionaltasks_at_overlaps                      r#   get_not_solved_tasks_qsr      sU    #..55gTY5Z5>>@LLZ^bLc%--5L-M {{))t)T /88:FFzX\F]+33;O3P$)$8J  g}d;//;,))'2-.>?B.ITX(YZB!"+2Q2Q^b!cfg g " Qy\[=m=m=rqr-stM04#%99Q-FJ\_lJl-m#n #%99-?--O#P 9XYikv9w6$&6 #>?O#P #3#:#:1;NQReiQj;j#k #3#:#:e#:#L VW%%.FLL5&NOP"9:J"KAJ5B:NNNJ 8tD]g}d;$T>5AhWTS`bgEh7;0EuMRd N..:!,!Q!Q!VUV
 ##G#4(-3 E]bc!%)   91NOT-   077?O7P4jBZZZr%   c                     | j                  |j                  d      }|j                         rd| j                  dd      fS d| fS )NTr   z-is_labeled
_agreementF)rR   r   rS   r7   )rL   r   prioritized_low_agreements      r#   r   r   *  sJ     !&K<[<[hl m '')U^^M<@@@%<r%   r   r   c                 v   d }d}d}|r7t         j                  d|  d       |j                         }d}||rdnddz   z  }|sBt        j                  | |      }|r)t         j                  d|  d	|        d}||rdndd
z   z  }|s4t         j                  d|  d       t        |||       }|r||rdnddz   z  }|s5|r3t         j                  d|  d       t        ||       }|r||rdnddz   z  }|sC|j                  dkD  r4t         j                  d|  d       t        || |      }|r||rdnddz   z  }|||fS )NTr   r   z try to get task from assignedFr   zManually assigned queue)rL   z got already locked for them z	Task lockz' tries ground truth from prepared taskszOnboarding ground truth queuez( tries low agreement from prepared taskszLow agreement queuerY   z& tries depth first from prepared taskszBreadth first queue)	r?   r@   firstr   get_locked_byrW   rK   maximum_annotationsrh   )r   r   r   r   r   r   use_task_lockr   s           r#   get_next_task_without_dm_queuer   5  s    IMJ uTF"@AB$**,	
u6OOO
 &&t3CD	LL5&CI;OP!MJ5B+EEJ uTF"IJK01A7DQ	J5B:YYYJ 2uTF"JKL'(8$?	J5B:OOOJ 44q8uTF"HIJ&'7wG	J5B:OOOJmZ//r%   c           
         | s|j                   |j                  j                  k(  rt        |ddd      }|j                  j                  |      j                  d      j                  dd      }|j                         rlt        t        |      D 	cg c]  \  }}	t        |	|       c}	} }
|j                  |      j                  |
      }|rt        |      } nt        ||      } d	}| |fS c c}	}w )
NFT)r   r   r   task__is_labeled
updated_atr   rG   r5   rq   rm   zSkipped queue)
skip_queue	SkipQueueREQUEUE_FOR_MEr   r_   rR   r7   rI   rS   r   	enumerater   r   rK   )r   rj   r   r   r   r   qskipped_tasksposr5   preserved_orders              r#   skipped_queuer   h  s    ++w/@/@/O/OOgEX]^((//2;;LIUUV`gkUl!")TaJb$cwsBTRc%:$cdO*111GPPQ`aM &}5	/tD	(Jj   %ds   C0
c           
         | st        |ddd      }|j                  j                  |      j                  d      j	                  dd      }|j                         rut        t        |      D 	cg c]  \  }}	t        |	|       c}	} }
|j                  |      j                  |
      }|rt        |      } nt        ||      } | d| _        d	}| |fS c c}	}w )
NFT)r   r   r   r   r   r   rG   r   rm   zPostponed draft queue)r   r   rR   r7   rI   rS   r   r   r   r   rK   allow_postpone)r   rj   r   r   r   r   r   postponed_tasksr   r5   r   s              r#   postponed_queuer   }  s    G%t^cd++,,Q/88FRRS]dhRi!!#")TcJd$ewsBTRc%:$efO,33?3KTTUdeO &7	/F	$+0	(0Jj   %fs   0C
c                    d }|j                   |j                  k(  r7t        j                  d| d       t	        | |      }|r||rdnddz   z  }||fS |j                   |j
                  k(  r:t        j                  d| d       t        | ||||      }|r||rdnddz   z  }||fS |j                   |j                  k(  r3t        j                  d| d       t        | |      }|r||rdndd	z   z  }||fS )
Nr   , tries sequence sampling from prepared tasksr   r   zSequence queuez/ tries uncertainty sampling from prepared taskszActive learning or random queuez* tries random sampling from prepared taskszUniform random queue)	rT   rU   r?   r@   rK   UNCERTAINTYr   UNIFORMrD   )r   ri   rj   r   r   r   r   s          r#   get_task_from_qs_with_samplingr     s#    I7+++uTF"NOP'(8$?	J5B:JJJJ j   
		W00	0uTF"QRS-.>I`bfhvw	J5B:[[[J j   
		W__	,uTF"LMN()94@	J5B:PPPJj  r%   dm_queuec           
         t         j                  d|  d| d|        t        t              5  d }d}d}t	        | ||||      \  }}	}}
|st        | ||||
      \  }}}t        d      rT|sR|j                  rFt         j                  d|  d	       t        |      \  }}||rd
nddz   z  }t        ||	|| ||      \  }}|sK|r6||rd
nddz   z  }t         j                  d|  d       |j                         }nt        ||	|| ||      \  }}t        |||| ||      \  }}t        |||| ||      \  }}|r|r|j                  |        t         j                  t        |       d| d|        |rt        d|       r	 |j                   j#                  d      j%                         }||j&                  k\  }||j(                  k\  }|j*                  j%                         |j(                  |j                   j%                         z
  kD  }|j,                  s|s|s|rOddlm} t3        t5                     }|j7                  dd        |j7                  dd        |j7                  dd         ||      j8                  }|j7                  dd        |j7                  dd        t;        |d         D ]4  \  }}t3        |d   |         |d   |<   |d   |   j7                  dd        6 |j<                  }|j(                  |j>                  |j@                  |jB                  |j                  |jD                  |jF                  |jH                  d}t         jK                  d| d| d|        tS        || |       ||fcd d d        S # tL        $ r+}t         jO                  d tQ        |              Y d }~Id }~ww xY w# 1 sw Y   y xY w)!Nzget_next_task called. user: z, project: z, dm_queue: )	predicateTr   r   r   r   r   r   zData manager queuer   z#get_next_task finished. next_task: z, queue_info: r'   F)r   r   )TaskSimpleSerializerrj   ri   r   datapredictionsr_   rg   )r   r   rT   r"   r   overlap_cohort_percentage
project_idtitlez9DEBUG INFO: get_next_task is_labeled/overlap: LOCALS ==> z :: PROJECT ==> z :: NEXT_TASK ==> z-get_next_task is_labeled/overlap try/except: )*r?   r@   r   r	   r   r   r   r   r]   r   r   r   r   set_locklogr-   r_   rR   ry   r[   r   locksr   tasks.serializersr   dictlocalspopr   r   r   r   rT   r"   r   r1   r   info	Exceptionerrorr|   r   )r   rj   r   r   r   r   r   r   r   ri   r   r   r\   ry   task_overlap_reachedglobal_overlap_reachedr   r   localrC   iaproject_dataes                           r#   get_next_taskr     s    LL/v[	V^U_`a	&6	7 \%	
[r'>=*\
X1:?X 3Qg/@Y40I}j VW!;!;uTF*RST(?@P(Q%%
u>RRR
(F&(?QUW^`j)%	: 
u>RRR
uTF*VWX,224	 )G$&=~tU\^h)%	: !0	>7TXZgis t	: -iRVXegq r	:t$

'-1)N:,W	
 "]_cd&!--4454IOOQ',	0A0A'A$).'2M2M)M&!--/'2M2MPYPePePkPkPm2mm''+?CY]bF NEII.5II7>II0$7/	:??DHHVT*HH]D1 )$}*= > C115d=6I!6L1M]+A.]+A.228TBC (//G/6/J/J&-&8&8$+$4$48?8\8\.5.H.H5<5V5V&-jj!(	$L KK&&+W,<\N K))-0 	9dG4*$y\% \%n  LSQRVHUVo\% \%s7   EN9<G,N(N9	N6!N1,N91N66N99Or    )Ar)   collectionsr   typingr   r   r   core.feature_flagsr   core.utils.commonr   r	   r
   core.utils.dbr   django.confr   django.db.modelsr   r   r   r   r   r   r   r   r   django.db.models.fieldsr   !projects.functions.stream_historyr   projects.modelsr   tasks.modelsr   r   users.modelsr   	getLogger__name__r?   r!   r$   GET_TASKS_AGREEMENT_QUERYSETr   !SHOULD_ATTEMPT_GROUND_TRUTH_FIRSTrP   intr-   rD   rK   rW   r]   rh   r   r   rQ   r|   r   r   r   r   r   r   r    r%   r#   <module>r     s     % % ' M M $   U U U 0 @ # ) 			8	$6t 6g 6$ 6
  ))N)NO h889Y=Y "
d s CXd^ C4 CV[\`bf\fVg C	CXd^ 	CeD$J>O 	C  PT Y^_cei_iYj -8D> -eE$*<MxX\~<]6^ -htn D 7 uUY[_U_O` 8.D>.. "#Y. 	.
 TN. 4:.b.4 .' .d .BBx~ B(4. B
a[
a[a[ TNa[ t$	a[
 a[ 8D>49c4/0a[H00
0000 00 t$	00
  $00 5tdC'(00f!*!.!tn!!#Y! ! 	!
 ! ! 5tc!"!F (,e%
e%e% e% D$J	e%
 t$e% 5tc!"e%r%   