]j-n~ddlZddlZddlZddlmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZdd lmZmZmZdd lmZdd lmZdd lmZdd lmZddlmZddl m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'ejPe)Z* ddeedfdZ+dZ,dZ-ddZ.e e j^Z0dZ1dZ2dZ3y)N)CallableOptional)flag_set) load_func)%load_tasks_for_async_import_streaming)settings) transaction)LabelInterface) ProjectImportProjectReimportProjectSummary)ValidationError)Task)"sanitize_prediction_import_payload)User) WebhookAction)emit_webhooks_for_instance) FileUpload)ImportApiSerializer)load_tasks_for_async_importrecalculate_stats_func).Nc  tj5 tjj |}|jtjjk7r#t jd|d dddytjj|_|jdgdddtjj |}tdd r9t jd j j"t%||yt'j&}j }d}t)||\}} } } |j*rQtd |j,j. } t jd | t1||j*|| }|j2rJtd |j,j. r(g} t5|j6}t9|D]b\}}d|vs t9|dD]G\}} t;|}|j=|d}|r!|D]}| j?d|d|d|Id| rdtE| d}| D] }|d|dz } td |j,j. r;||_tjjF|_|jddgyt jdtE| d||jHr)tj5tJjjMj |}tO|dd|i}|jQd |j|j" }tS|jT|tVjX|tE|}tE|jZ}tE|j\}|||d!}|j_|d"d"d|#t jd$|ja| dddntE|}d}d}t'j&|z }xsd&|_1xsd&|_2xsd&|_3||_4| |_5| |_6| |_7|jpr|Dcgc]}|j"c}|_9tjjt|_|jy#tj $r$t jd|dYdddywxYw#1swYxYw#t@$rH}d|d|dtC|}| j?|t jd|Yd}~td}~wwxYw#t@$rZ}d%tC|}||_tjjF|_|jddgYd}~dddyd}~wwxYw#1swYxYwcc}w)'NidzProjectImport with id $ not found, import processing failedzProcessing import with id  already startedstatus update_fieldsEfflag_fix_back_plt_902_async_import_background_oom_fix_22092025_shortautouserz#Using streaming import for project 1fflag_feat_utc_210_prediction_validation_15082025,Reformatting predictions with raise_errors: predictionsT return_errorsTask , prediction : : Error validating prediction - Exception during validation: Prediction validation failed ( errors): -  error3Prediction validation failed, not raising error - (projectr6datamanycontextraise_exception project_id task_countannotation_countprediction_countFtasks_querysetmaximum_annotations_changed!overlap_cohort_percentage_changedtasks_number_changedrecalculate_stats_countsz)Tasks bulk_update finished (async import)zError creating tasks: r);r atomicr objectsget DoesNotExistloggerr3rStatusCREATED IN_PROGRESSsaverrinfor6r"_async_import_background_streamingtimerpreannotated_from_fields organization created_byreformat_predictionslabel_config_is_not_defaultr label_config enumeratervalidate_predictionappend ExceptionstrlenFAILEDcommit_to_projectr select_for_updateris_validractive_organizationr TASKS_CREATEDdb_annotationsdb_predictions%update_tasks_counters_and_task_statesupdate_data_columnsr@rArBdurationfile_upload_ids found_formats data_columnsreturn_task_idstask_ids COMPLETED) import_iduser_idrkwargsproject_importr$startr6tasksrlrmrn raise_errorsvalidation_errorsliitaskj predictionvalidation_errors_listr3e error_msg error_messagesummary serializerr@rArBrHrks L/root/env/lib/python3.12/site-packages/label_studio/data_import/functions.pyasync_import_backgroundrs     6 *2266)6DN  M$8$8$@$@ @ LL5i[@PQ R  6 6!. 4 4 @ @8*5 6 <<  w  'DW^de 9.:P:P:S:S9TUV*>4@ IIKE$$G E;VVdfj:k7E?M<.. ?gFZFZFeFe   B<.QR$UN,S,SU\^jk**x;'BVBVBaBa0 G00 1 ' RGAt$%.tM/B%C RMAz R%G %S 131G1G bf1G1g.1)?_ 1 8 85=QRPSSUV[U\9] ^_ R R >@DDWDUG-%dYX_L`aJ     5! "7::>*4+C+CWmNiNikpq Z #&z'@'@#A #&z'@'@#A #-(8(8,( ==#(056;)--E > GH++E2C) ) XZ yy{U"H *aN&6&;!N#&6&;!N#&N%4N"#0N ".N%%7<"=t477"=)00::NC))  LL1)<`a b  6 6  6 6d%R&+A3mA3>^_bcd_e^f$g )00; 'DYK%PQQRp "8Q A '4$(5(<(<(C(C%##7H2E#FS) ) F G) ) t#>sT6 S<AT62T6"AU)AW=;B9V1X <,T3(T62T33T66U V =VV W: AW5(W=5W::W==Xc|jd}tjj|j tj j tjt|yNrr)r tracebackr3) argsr rJfilterupdaterNrar format_excr_)job connectiontypevalue_rrs rset_import_background_failurersY I  I .55##**i6J6J6LTWX]T^6c|jd}tjj|j tj j tjt|yr) rr rJrrrNrarrr_)rrrrr reimport_ids rset_reimport_background_failurersX((1+K""k"299%%,,&&(%j:rc xg}g}d}|r t|j}t |D]0\}} d| vr| d} n| } g} |D]} | | vr|jd|d| d!| | } | )d}d }|ry |j| }t|d r[|jrOt|jtr|jd n |j}|jj}t| tr| }n)|}|d k(rd}|d vr|| i}n|t| ts| gn| i}| |||dgddd}| j|| | d}|j|3|r|r t!d|i|S#t$r0}tj d|j d|Yd}~d}~wwxYw#t$rYwxYw)a Transform flat task JSON objects into proper format with separate data and predictions fields. Also validates the predictions to ensure they are properly formatted using LabelInterface. Args: tasks: List of task data preannotated_from_fields: List of field names to convert to predictions project: Optional project instance to determine correct to_name and type from label config Nz,Could not create LabelInterface for project r,r8r*z: Preannotated field 'z' not found in task datatextchoicesto_namertextarea)ratingnumberdatetime) from_namerrrg? preannotated)resultscore model_version)r8r'preannotated_fields)r rZr^rMwarningrr[r] get_controlhasattrr isinstancelisttaglowerdictr)rwrUr6rx new_tasksryrzr task_indexr| task_datar'fieldrrprediction_type control_tagprediction_value value_keyr~new_tasks rrXrXsAI B ] 4 45B&e,F# D T>V II -; /EI%!((5 K@P?QQefnfqfqers t##% <(00BBDHHQXHY1$4WVZ9[ ##D#9!+GJJ!G%++>,JaQTT,JK$'~#6 ),Z-F-F)G&),Z-F-F)G& $44 &*@@&&*@@&"((7 ++8((M9(++N;A rtAjB5tDjFjIjK|}tM|d d|i}|jOd |j=|j} |jQ| D!cgc]}!|!jc}!t| }"t|jR}#t|jT}$||"z }||#z }||$z }|jW| dddn|t|z }t jd| d t|d!tY| } |j>r|rt jd"t|d!t[|j\|t^j`||||d#}%tbjFje|$}&|jg|&d%d%d |%&t jd'tj|z }'|xsd|_4|xsd|_5|xsd|_6|'|_7| |_8| |_9| |_:|jvr||_<t4j6jz|_|j=t jd(|jd)|d*y#t.$rK}d ||zd|dt1|}|j-|t j3d|Yd}~d}~wwxYwcc}!w#1swYxYw#t.$r}t j3d+|jdt1|d ,t4j6j8|_t}j~|_>t1||_|j=d}~wwxYw)-Nrz Empty batch received for import rrrz tasks for import r%r#r&r'Tr(r*r+r,r-r.r/r0r1r2r3rrr4r5r6r7r;r=rrrzBFinalizing import: emitting webhooks and updating task states for r?rFrCz3Tasks bulk_update finished (async streaming import)zStreaming import rrzError in streaming import r)@rIMPORT_BATCH_SIZEr6rTrrrMrRrr`rrUrrVrWrXrYr rZr[rr\r]r^r_r3r rNrarrQrbr rIr rJrcrKrrdrrgrhrjrrrerrfrrrir@rArBrkrlrmrnrorprqrr)(rur$rrrrrr6rvrstreaming_generatorfinal_file_upload_idsfinal_found_formatsfinal_data_columnsrrlrmrnrxryrzr{r|r}r~rr3rrrrrrrrrrrHrrks( rrSrSs\// !"!"! ((  CNTXZde "  UI\V b EK- >~?P?P>QRS A L KK#L>K8H7II[\j\m\m[no ?:O#O(7%  & &} 5  % %l 366'GgNbNbNmNm   J<.YZ2!H!H'S_ 22xC'J^J^JiJi8%'!#G$8$89(5ZGAt$,-6tM7J-K ZMAz Z-OPZ-[ 9;9O9OPZjn9O9o 6#91G%*(9(@(@.34Dq4H3IWXVYY[\a[b,c)*%* ZZ %&DSIZE[D\\g$hM!28%2eWB7 8 KRYRfRfRqRq0=,0=0D0D0K0K-&++7H:M+N QRUVgRhQiitvCuDE// '')@,44FFHLLU\L]G!4+D[dfmZn!oJ'''=%/__ _%KN(//~0N!0NO'*>':$-01J1J-K*-01J1J-K*$(88$*.DD**.DD*//?#@@(!C $44 KK&.GKHXGYY_` amV bp""45  + +0D KKTUXYmUnToouv  '(('=3N3NPd  /$:$:( $ "&!4!4&':$&8#  ) )&:N # - 4 4 > > '(9(9':,GWFXXghis$-Z.34Dq4H3IWXVYYyz}~{AzB-C 1 8 8 C & /LYK-X Y YZ<1O@@H  1.2C2C1DBs1vhOZ^ _ - 4 4 ; ;#,#7#7#9 "1v sFW W5AU9BW;AWA} t=| j@} | | | d}|jC|ddd|t jd| jE|ddd |_# |_$ |_%||_&tO||_(tjjR|_|jtU|y#tj $r$t jd|dYdddywxYw#1swYgxYw#1swYxYw)NrzProjectReimport with id rzProcessing reimport with id rrrAfflag_fix_back_plt_838_reimport_memory_improvement_05082025_shortr"r#z%Using streaming reimport for project )rr5Trr7r;r=r?FrCz+Tasks bulk_update finished (async reimport))+r rIr rJrKrLrMr3rrNrOrPrQr6rrRrrrload_tasks_from_uploaded_filesrlrr rcrrrdrrrfr`rgrhrirjr@rArBrmrrnrqr)rrr$rtrr6rwrmrnrrr@rArBrHs rasync_reimport_backgroundr3s     0 &..22k2BH ??o44<< < LL7 }DTU V  0 0*00<< XJ / 0GSZ`a ;GJJ<HI,XwQUV.8-V-V   h66HLhLh. *}l   ! /$,,>>@DDWDUG  0 01I1I J,%dX_imLnoJ     5OOwzzO:E &A\A\^c dUJ":#<#<= ":#<#<= )$4$4( $  9 9$,127%))A :  KKE F  ' ' .? /D)$4!$4!!. $\ 2)00:: h'++  LL3K=@de f  0 0  0 00 / /sAL KAL2LDL%,L LLLL"%L.)N)NF)4loggingrTrtypingrrcore.feature_flagsrcore.utils.commonrdata_import.uploaderr django.confr django.dbr label_studio_sdk.label_interfacer projects.modelsr r r rest_framework.exceptionsr tasks.modelsrtasks.serializersr users.modelsrwebhooks.modelsrwebhooks.utilsrmodelsr serializersruploaderr getLogger__name__rMrrrrXPOST_PROCESS_REIMPORTrrrSrrrrs %''F !;JJ5@)5,1   8 $QUG08)9L0MGTaH"("@"@Atn]@D(r