
    	]j                        d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z
ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZ ddlZddlZddlZddlmZ dd	lmZmZ dd
lmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z>m?Z? ddl@mAZAmBZB ddlCmDZD ddlEmFZF ddlGmHZH  ej                  eJ      ZK G d de(j                        ZM G d  d!eM      ZN G d" d#eN      ZO G d$ d%e(j                        ZPe$j                  fd&ZRd' ZSd( ZTd) ZUd* ZV G d+ d,eNeP      ZW G d- d.e(j                        ZX G d/ d0e(j                        ZYy)1zThis file and its contents are licensed under the Apache License 2.0. Please see the included NOTICE for copyright information and LICENSE for a copy of the license.    N)ThreadPoolExecutor)asdict)datetime)AnyIteratorUnion)urljoin)flag_set)redis_connectedstart_job_async_or_sync)	load_func)iterate_queryset)ExportDataSerializer)settings)AnonymousUser)modelstransaction)	JSONField)reverse)timezone)gettext_lazy)backfill_fsm_states_for_tasks)StorageObjectget_uri_via_regexparse_bucket_uri)ValidationError)Job)
AnnotationTask)AnnotationSerializerPredictionSerializer)WebhookAction)emit_webhooks_for_instance   )UnsupportedFileFormatErrorc                      e Zd ZdZ G d dej
                        Z G d d      Z ej                   e	d      ddd	      Z
 ej                   e	d
      ddd	      Z ej                   e	d      dddd      Z ej                  dej                  ej                         Z ej$                  ddd	      Z edded      Zd Zd Zd Zd Zed        Zd Zd Zd Zd Z e!d        Z"d  Z#d! Z$y")#StorageInfozi
    StorageInfo helps to understand storage status and progress
    that happens in background jobs
    c                       e Zd Zd ed      fZd ed      fZd ed      fZd ed      fZd	 ed
      fZd ed      fZ	y)StorageInfo.StatusinitializedInitializedqueuedQueuedin_progresszIn progressfailedFailed	completed	Completedcompleted_with_errorszCompleted with errorsN)
__name__
__module____qualname___INITIALIZEDQUEUEDIN_PROGRESSFAILED	COMPLETEDCOMPLETED_WITH_ERRORS     N/root/env/lib/python3.12/site-packages/label_studio/io_storages/base_models.pyStatusr)   4   s[    #Q}%551X;&#Q}%551X;&;/	 7;R9S Sr?   rA   c                       e Zd ZdZy)StorageInfo.MetaTNr4   r5   r6   abstractr>   r?   r@   MetarC   <       r?   rF   z	last syncTzLast sync finished timenullblank	help_textzlast sync countzCount of tasks synced last timelast_sync_job   zLast sync job IDrI   rJ   
max_lengthrK   @   )rO   choicesdefaultz)Traceback report for the last failed syncmetaz2Meta and debug information about storage processes)rI   rR   rK   c                 8    || _         | j                  dg       y )NrL   update_fields)rL   save)selfjob_ids     r@   info_set_jobzStorageInfo.info_set_jobO   s    #		 1	2r?   c                    d | _         d | _        d | _        | j                  j                  | _        | j                  j                  dd      dz   t        t        j                               d| _        | j                  g d       y )Nattemptsr   r$   )r\   time_queued)rL   	last_synclast_sync_countstatusrS   rU   )r^   r_   rL   rA   r9   r`   rS   getstrr   nowrW   rX   s    r@   _update_queued_statusz!StorageInfo._update_queued_statusS   sk    #!kk(( "&z1!=!ARUV^VbVbVdRef			 c	dr?   c                 X   t         j                  t         j                  k(  r| j                          yt	        j
                         5  	 | j                  j                  j                         j                  | j                        }|j                  | j                   j"                  | j                   j$                  fv rVt        j                  d| d|j&                   d|j                   d	|j(                   d
|j*                   
       	 d d d        y|j                          | j-                          	 d d d        y# | j                  j                  $ rE t        j                  d| j                  j                   d| j                   d       Y d d d        yw xY w# 1 sw Y   y xY w)NT)pkStorage z	 with pk=z does not existFz (id=z) is already in status "z(". Cannot set to QUEUED. Last sync job: z, Meta: )r   	DJANGO_DBDJANGO_DB_SQLITEre   r   atomic	__class__objectsselect_for_updatera   rg   DoesNotExistloggererrorr4   r`   rA   r9   r:   idrL   rS   refresh_from_db)rX   locked_storages     r@   info_set_queuedzStorageInfo.info_set_queued^   s   !:!::&&(! 	!%!7!7!I!I!K!O!OSWSZSZ!O![
 $$););T[[=T=T(UU~.eN4E4E3F G&--. /&&4&B&B%C D+0013 	 	  002  "'	 	 >>.. x(?(?'@	$''Rabc	 		 	s8   F 	>D?BF !F ?AFF FF  F)c                 b   | j                   | j                  j                  k7  rt        d| j                    d      | j                  j                  | _         t        j                         }t        |      | j                  d<   t        |      | j                  d<   | j                  ddg       y )NzStorage status (z') must be QUEUED to move it IN_PROGRESStime_in_progresstime_last_pingr`   rS   rU   )
r`   rA   r9   
ValueErrorr:   r   rc   rb   rS   rW   )rX   dts     r@   info_set_in_progressz StorageInfo.info_set_in_progressx   s    ;;$++,,,/}<cdeekk--\\^(+B		$%&)"g		"#		6 2	3r?   c                     d| j                   vr"t        j                  | j                   d         S t        j                  | j                   d         S )Ntime_failurerw   )rS   r   fromisoformatrd   s    r@   rw   zStorageInfo.time_in_progress   sC    *))$))4F*GHH))$))N*CDDr?   c                    | j                   j                  | _        t        j                         | _        || _        t        j                         }t        |      | j                  d<   || j                  z
  j                         | j                  d<   | j                  j                  |       | j                  g d       y )Ntime_completedduration)r`   rS   r^   r_   rU   )rA   r<   r`   r   rc   r^   r_   rb   rS   rw   total_secondsupdaterW   )rX   r_   kwargsr   s       r@   info_set_completedzStorageInfo.info_set_completed   s    kk++!.!&).&9		"#!/$2G2G!G V V X		*		 		 R	Sr?   c                    | j                   j                  | _        t        j                         | _        || _        dj                  |      | _        t        j                         }t        |      | j                  d<   || j                  z
  j                         | j                  d<   t        |      | j                  d<   | j                  j                  |       | j                  g d       y )N
r   r   tasks_failed_validation)r`   rS   r^   r_   	tracebackrU   )rA   r=   r`   r   rc   r^   r_   joinr   rb   rS   rw   r   lenr   rW   )rX   r_   validation_errorsr   r   s        r@   info_set_completed_with_errorsz*StorageInfo.info_set_completed_with_errors   s    kk77!.#45!&).&9		"#!/$2G2G!G V V X		*/23D/E		+,		 		 _	`r?   c                    | j                   j                  | _        t        j                         \  }}}|rt        |t              rg }t        |d      rCt        |j                  t              r|j                  j                         D ]y  \  }}t        |t              rJ|D ]D  }t        |d      r|j                  |j                         +|j                  t        |             F `|j                  t        |             { nt        |j                  t              rT|j                  D ]D  }t        |d      r|j                  |j                         +|j                  t        |             F n$|j                  t        |j                               |rdj!                  |      | _        nEt        t%        j&                               | _        n"t        t%        j&                               | _        t)        j*                         }t        |      | j,                  d<   || j.                  z
  j1                         | j,                  d<   | j3                  g d       y )Ndetailstringr   r}   r   )r`   r   rS   rU   )rA   r;   r`   sysexc_info
issubclassr   hasattr
isinstancer   dictitemslistappendr   rb   r   r   tb
format_excr   rc   rS   rw   r   rW   )	rX   exc_type	exc_valueexc_tracebackerror_messagesfielderrorsrq   r}   s	            r@   info_set_failedzStorageInfo.info_set_failed   s   kk(( .1\\^*)] 
8_=Ny(+i..5)2)9)9)?)?)A ?v%fd3)/ F#*5(#;$2$9$9%,,$G$2$9$9#e*$E	F +11#f+>?  	 0 0$7!*!1!1 >"5(3*11%,,?*11#e*=	> #))#i.>.>*?@ !%>!:!$R]]_!5 !1DN||~$'$5		.!!-0E0E!E T T V		*		 ?	@r?   c                    t        j                         }t        j                  | j                  d         }||z
  j                         }|t        j                  kD  ry|| _        t        |      | j                  d<   || j                  z
  j                         | j                  d<   | j                  j                  |       | j                  ddg       y y )Nrx   r   r_   rS   rU   )r   rc   r   r~   rS   r   r   STORAGE_IN_PROGRESS_TIMERr_   rb   rw   r   rW   )rX   r_   r   rc   	last_pingdeltas         r@   info_update_progressz StorageInfo.info_update_progress   s    lln**4995E+FG	y//18555#2D *-c(DII&'%(4+@+@%@$O$O$QDIIj!IIV$II%6$?I@ 6r?   c                 Z    | j                  dddd      } | D ]  }|j                           y)zCheck failed jobs and set storage status as failed if job is failed

        :param storages: Import or Export storages
        rr   rL   r`   rS   N)onlyhealth_check)storagesstorages     r@   ensure_storage_statusesz#StorageInfo.ensure_storage_statuses   s3     ==&I 	#G  "	#r?   c                 $   t        j                         }t        j                  | j                  j                  dt        |                  }||z
  j                         }t               r| j                          | j                  | j                  j                  k(  rt|t        j                  dz  kD  r]| j                  j                  | _
        d| _        | j#                  ddg       t$        j'                  d|  d| j(                   d	       y y y )
Nrx      zIt appears the job was failed because the last ping time is too old, and no traceback information is available.
This typically occurs if job was manually removed or workers reloaded unexpectedly.r`   r   rU   rh   * status moved to `failed` because the job z has too old ping time)r   rc   r   r~   rS   ra   rb   r   r   job_health_checkr`   rA   r:   r   r   r;   r   rW   rp   inforL   )rX   rc   r   r   s       r@   r   zStorageInfo.health_check   s    lln**499==9I3s8+TU	y//1 !!# ;;$++111eh>`>`cd>d6d++,,DK4 N IIX{$;I<KK4& J4K]K]J^^tu 7e1r?   c                    | j                   }| j                  |j                  |j                  fvry t	        j
                  d      }	 t        j                  | j                  |j                        }|j                         }|dk(  rR|j                  | _        d| _        | j!                  ddg       t"        j%                  d	|  d
| j                          y |dk(  rS|j                  | _        d| _        | j!                  ddg       t"        j%                  d	|  d| j                   d       y y # t        j                  j                  $ r d}Y w xY w)Nlow)
connectionz	not foundr/   zIt appears the job was terminated unexpectedly, and no traceback information is available.
This typically occurs due to an out-of-memory (OOM) error.r`   r   rU   rh   z4 status moved to `failed` because of the failed job zIt appears the job was not found in redis, and no traceback information is available.
This typically occurs if job was manually removed or workers reloaded unexpectedly.r   z was not found)rA   r`   r:   r9   	django_rq	get_queuer   fetchrL   r   
get_statusrq
exceptionsNoSuchJobErrorr;   r   rW   rp   r   )rX   rA   queuesync_job
job_statuss        r@   r   zStorageInfo.job_health_check   sA   ;;v116==AA##E*	%yy!3!3@P@PQH!,,.J ! --DKM N
 IIX{$;I<KK(4&(\]a]o]o\pqr ;& --DK4 N IIX{$;I<KK(4&(RSWSeSeRfftuv '! }}++ 	%$J	%s   ;D3 3EEN)%r4   r5   r6   __doc__r   TextChoicesrA   rF   DateTimeFieldr7   r^   PositiveIntegerFieldr_   	CharFieldrL   rQ   r8   r`   	TextFieldr   r   r   rS   rZ   re   ru   r{   propertyrw   r   r   r   r   staticmethodr   r   r   r>   r?   r@   r'   r'   .   s?   
T## T  %$$Q{^$dVopI1f11	
4t?`O %F$$	/TcM_M V""F
 !  d$BmnIV$@tuD3	e4
4 E E
T
a+AZA # #0"wr?   r'   c                   
   e Zd ZdZ ej
                   ed      dddd      Z ej                   ed      ddd	      Z	 ej                   ed
      dd      Z ej                   ed      dd      ZddZ G d d      Zy)Storage titleTrM   zCloud storage titlerN   descriptionzCloud storage descriptionrH   
created atCreation timeauto_now_addrK   synchronizablezIf storage can be synced)rR   rK   Nc                     t        d      )Nz&validate_connection is not implementedNotImplementedError)rX   clients     r@   validate_connectionzStorage.validate_connection.  s    !"JKKr?   c                       e Zd ZdZy)Storage.MetaTNrD   r>   r?   r@   rF   r   1  rG   r?   rF   N)r4   r5   r6   
url_schemer   r   r7   r   r   r   r   
created_atBooleanFieldr   r   rF   r>   r?   r@   r   r   %  s    JFQwZd$3ZopE"&""1]#3$dVqrK%%%aoDTcdJ(V((+;)<dVpqNL r?   r   c                       e Zd Zdee   fdZdee   fdZdedefdZ	de
e   fdZd Zd Zd	eed
f   defdZd	eed
f   defdZddZd Zedefd       Zd Zd Zd Z G d d      Zy
)ImportStoragereturnc                     t         )z]
        Returns:
            Iterator[Any]: An iterator for objects in the storage.
        r   rd   s    r@   iter_objectszImportStorage.iter_objects6  
    
 "!r?   c                     t         )zi
        Returns:
            Iterator[str]: An iterator of keys for each object in the storage.
        r   rd   s    r@   	iter_keyszImportStorage.iter_keys=  r   r?   objc                     t         )z
        Args:
            obj: The storage object to get metadata for
        Returns:
            dict: A dictionary of metadata for the object with keys:
            'key', 'last_modified', 'size'.
        r   )rX   r   s     r@   get_unified_metadataz"ImportStorage.get_unified_metadataD  s
     "!r?   c                     t         r   r   )rX   keys     r@   get_datazImportStorage.get_dataN      !!r?   c                     t         r   r   rX   urls     r@   generate_http_urlzImportStorage.generate_http_urlQ  r   r?   c                     t         )zGet file bytes from storage as a stream and content type.

        Args:
            uri: The URI of the file to retrieve

        Returns:
            Tuple of (BytesIO stream, content_type)
        r   )rX   uris     r@   get_bytes_streamzImportStorage.get_bytes_streamT  s
     "!r?   r   Nc                 $    | j                  |      S r   )can_resolve_schemer   s     r@   can_resolve_urlzImportStorage.can_resolve_url_  s    &&s++r?   c                 ^   |syt        || j                  f      \  }}t        ||       }|| j                  k(  rt|rrt        | d      r|j                  | j                  k(  ryt        | d      r|j                  | j
                  k(  ryt        | d      r|j                  | j                  k(  ryy)NFprefixesbucketT	containerpath)r   r   r   r   r   r   r   )rX   r   r7   prefix
bucket_uris        r@   r   z ImportStorage.can_resolve_schemeb  s    %cT__4FG	6%c40
 T__$tX&:+<+<+Kt[)j.?.?4>>.QtV$):):dii)Gr?   c           	      |   t        |t              r2g }|D ])  }| j                  ||      }|j                  |r|n|       + |S t        |t              r:i }|j                         D ]#  }| j                  ||   |      }|r|n||   ||<   % |S t        |t              r| j                  |v r	 t        || j                  f      \  }}| j                  |      st        j                  d|        y |(t        j                  d| d       t        d|       t        t        j                   t#        dd|j$                  i      d	t'        j(                  |j+                               j-                          z         }	|j/                  ||	      S y y # t0        $ r t        j3                  d
| d       Y y w xY w)Nr   zNo storage info found for URI=z Task is required to resolve URI=Tr   z"storages:task-storage-data-resolvetask_id)r   z	?fileuri=zCan't resolve URI=)r   r   resolve_urir   r   keysrb   r   r   r   rp   debugrq   ry   r	   r   HOSTNAMEr   rr   base64urlsafe_b64encodeencodedecodereplace	Exceptionr   )
rX   r   taskresolveditemresultr   extracted_urir7   	proxy_urls
             r@   r  zImportStorage.resolve_uriw  s   c4 H <))$5&d;< O T"Hxxz ?))#c(D9*0c#h? O S!doo&<G#4SDOOCU#V q++M:LL#A#!GH<LL#CC5!ITXLY$'Gu%MNN#%%@)UYU\U\I]^!&":":=;O;O;Q"R"Y"Y"[!\]^	
 {{=)<<# '=!$  G06FGs   ,AF 1B!F #F;:F;c                     t         r   r   rd   s    r@   _scan_and_create_links_v2z'ImportStorage._scan_and_create_links_v2  s
     "!r?   link_objectc                    t        |      }|j                  dd       }|j                  dd      }	|j                  d      xs g }
|
rd|vrt        d      |j                  d      xs g }d}|r;d|vrt        d	      t	        |D cg c]  }|j                  d
d      s| c}      }d|v r/t        |d   t              r|d   |d   }n|j                  d       t        j                         5  t        |||t	        |      |k\  t	        |
      t	        |      |z
  |||		      }|j                  d        |j                  |fd|i| t        j                  d|j                  j                   d| d|       t!        dt#                      }t        j                  dt	        |
       d|        |
D ]   }|j$                  |d<   |j$                  |d<   " t'        |
d      }t!        d|j(                  j*                        xs |}|j-                  |      r|j                          t        j                  dt	        |       d|        |D ]   }|j$                  |d<   |j$                  |d<   " t/        |d      }|j-                         r|j                          nFt        j1                  d|j$                   d|j2                          |rt5        |j2                        d d d        |S c c}w # 1 sw Y   S xY w)N	task_data
allow_skipTpredictionsdatazUIf you use "predictions" field in the task, you must put "data" field in the task tooannotationsr   zUIf you use "annotations" field in the task, you must put "data" field in the task toowas_cancelledF)	r  projectoverlap
is_labeledtotal_predictionstotal_annotationscancelled_annotationsinner_idr  )skip_fsmr   zCreate z link with z
 for task=:ff_fix_back_dev_3342_storage_scan_with_invalid_annotationsuserz predictions for task=r  r  )r  many1fflag_feat_utc_210_prediction_validation_15082025)raise_exceptionz annotations for task=zInvalid annotations for task : )r   popra   ry   r   r   r   r   rk   r   rW   createrp   r  rl   r4   r
   r   rr   r!   organization
created_byis_validr    rq   r   r   )clsr  maximum_annotationsmax_inner_idr   r  
link_classlink_kwargsr  r  r  r  r   ar  r(  
predictionprediction_serraise_prediction_exception
annotationannotation_sers                        r@   add_taskzImportStorage.add_task  s%   [){D1XXlD1
 hh}-3T! k 
 hh}-3 !T! k  %(K(aq155Z_C`(a$b!T>jft<F|'F| ! 4	A+{+/BB"%k"2"%k"25J"J&;%%
D IItI$JdCGC{CLL77#4#4#=#=">k+Va\`[bcd"*LS`Sb# O
 LL73{#3"44J4&QR) 3
%)WW
6"(/


9%3 2{NN LSZSgSgSrSrs #" ' &&7Q&R##% LL73{#3"44J4&QR) 3
%)WW
6"(/


9%3 2{NN &&(##% <TWWIRH]H]G^_`").*?*?@@i4	Aj { )b4	Aj s   K:K:1G?K??L	c                 r   | j                          dx}}| j                  j                  }| j                  j                  j	                  d      j                         }|r|j                  dz   nd}g }t        d| j                  j                        }t        d| j                  j                        }	g }
t        | j                         |	rt        j                  nd      D ]  }t        t        j                  |            }|D ]  }t         j#                  d|         |j%                  ||       }||j&                  j)                  || j*                        j-                         z  }| j/                  ||	       |D ]s  }||v r/t         j#                  | j0                  j2                   d
|       7t         j#                  |  d|        |rU| j4                  sIt6        j8                  j;                  |j=                               \  }}h d}|r||vrt?        d| d      	 | jA                  |      }|D ]  }	 | jM                  | j                  ||| ||      }|dz  }|dz  }|
jO                  |j*                         tW        |
      t        jX                  k\  sgt[        | j                  j                  | j                  t\        j^                  |
       g }
 | j/                  ||	       v  |
r:t[        | j                  j                  | j                  t\        j^                  |
       ta        | j*                  ||       | j                  jc                  ddd       |r| je                  |||       y| jg                  ||	       y# tB        tD        jF                  jH                  f$ r+}t         j#                  |d       tK        d| d      d}~ww xY w# tP        $ rC}d|jR                   d| }t         jU                  |       |jO                  |       Y d}~d}~ww xY w)z
        TODO: deprecate this function and transform it to "pipeline" version  _scan_and_create_links_v2,
        TODO: it must be compatible with opensource, so old version is needed as well
        r   z	-inner_idr$   :fflag_fix_back_plt_804_check_file_extension_11072025_short)r,  .fflag_root_212_reduce_importstoragelink_countszScanning key key__inr   )r_   tasks_existedz! already has tasks linked to key=z: found new key >   .jsonl.parquet.jsonzFile "z" is not a JSON/JSONL/Parquet file. Only .json, .jsonl, and .parquet files can be processed.
If you're trying to import non-JSON data (images, audio, text, etc.), edit storage settings and enable "Tasks" import methodTr   zError loading JSON from file "z".
If you're trying to import non-JSON data (images, audio, text, etc.), edit storage settings and enable "Tasks" import methodN)r2  zValidation error for task from r)  F)maximum_annotations_changed!overlap_cohort_percentage_changedtasks_number_changed)r_   r@  r   )4r{   r  r0  tasksorder_byfirstr!  r
   r,  _batchedr   r    STORAGE_EXISTED_COUNT_BATCH_SIZEr   r   fromkeysrp   r  existsrm   filterrr   countr   rl   r4   use_blob_urlsosr   splitextlowerr%   r   UnicodeDecodeErrorjsondecoderJSONDecodeErrorry   r:  r   r   r   rq   r   WEBHOOK_BATCH_SIZEr#   r"   TASKS_CREATEDr   update_tasks_statesr   r   )rX   r2  r@  tasks_createdr0  r  r1  r   check_file_extensionexisted_count_flag_settasks_for_webhook
keys_batchdeduplicated_keysr   existing_keysr7   extjson_extensionslink_objectsexcr  eerror_messages                          r@   _scan_and_create_linksz$ImportStorage._scan_and_create_links  s%    	!!#())"ll>>||!!**;7==?.2)  (HW[WcWcWpWp 
 "*<4<<KdKd"
 "NNKahGGgh
 U	fJ !%T]]:%> ?( 4}SE234 '--.?FMZ//66}VZV]V]6^ddffM%%mS`%a( Ef-'LLDNN$;$;#<<^Z]Y_!`av%5cU;< (0B0BWW--ciik:FAs&EOs/98$SE *U V #'==#5L $0 $/K!#}} LL/( ''1  -   %) &* *009 ,-1L1LL2 LL55t||]E`E`bs -/)I$/L ))-Wd)eKEf!U	fn &))4<<9T9TVg
 	&dgg}jI(((-QVmq 	) 	
 // -]^o 0 
 ##MQ^#_E +DLL,H,HI LLtL4$8 >0 1 6 + !*I+//IZZ\]^\_(`]3)00? !s1   ,NAO*#O'<&O""O'*	P6	38P1	1P6	c                 .    | j                  t               y)z_This is proto method - you can override it, or just replace ImportStorageLink by your own modelN)rh  ImportStorageLinkrd   s    r@   scan_and_create_linksz#ImportStorage.scan_and_create_links  s    ##$56r?   c                    t               r| j                         sy t        t        | j                  | j
                  d| j                  j
                  | j
                  d| j                  j
                  | j                  j                  j
                  t        t        j                  	      }| j                  |j
                         t        j                  d|j
                   d|  d       y 	 t        j                  d|         | j                         sy t        | j                  | j
                         y # t        $ r) t        j                  d|  d	d
       t        |        Y y w xY w)Nr   )r  r   )
queue_namerS   
project_idorganization_id
on_failurejob_timeoutStorage sync background job  for storage z has been startedStart syncing storage rh   z failedTr   )r   ru   r   import_sync_backgroundrl   rr   r  r,  storage_background_failurer   RQ_LONG_JOB_TIMEOUTrZ   rp   r   r
  r  )rX   r   s     r@   synczImportStorage.sync  s   '').& !%TWWE<<?? $ 9 9 < <5$88
H hkk*KK6x{{m=QUPVVghi14TF;<++-&t~~tww? 1xvW5E*401s   0(D:  D: :/E,+E,c                       e Zd ZdZy)ImportStorage.MetaTNrD   r>   r?   r@   rF   rz    rG   r?   rF   r   )r4   r5   r6   r   r   r   rb   r   r   r   r   r   r   r   r   r   boolr   r   r  r  classmethodr:  rh  rk  rx  rF   r>   r?   r@   r   r   5  s    "hsm ""8C= "" " ""tM2 ""	",5d#3 , ,eCI&6 4 *%GN
" SXe S Sl@`D718 r?   r   c                   f    e Zd Z ej                  ddej
                  d      Zd Z G d d      Zy)	ProjectStorageMixinzprojects.Projectz%(app_label)s_%(class)ssz0A unique integer value identifying this project.)related_name	on_deleterK   c                 ^    | j                   |_         | j                   j                  |      ryyNTF)r  has_permissionrX   r%  s     r@   r  z"ProjectStorageMixin.has_permission  s%    ||<<&&t,r?   c                       e Zd ZdZy)ProjectStorageMixin.MetaTNrD   r>   r?   r@   rF   r    rG   r?   rF   N)	r4   r5   r6   r   
ForeignKeyCASCADEr  r  rF   r>   r?   r@   r~  r~    s3    f/..D	G r?   r~  c                     | j                   j                  |      }	 |j                          y # t        $ r |j	                          Y y w xY wNrr   )rm   ra   rk  r%   r   )storage_class
storage_idtimeoutr   r   s        r@   ru  ru    sL    ##'':'6G%%'%  	!s   / A
Ac                 \    | j                   j                  |      }|j                          y r  )rm   ra   save_all_annotationsr  r  r   r   s       r@   export_sync_backgroundr    s'    ##'':'6G  "r?   c                 \    | j                   j                  |      }|j                          y r  )rm   ra   save_only_new_annotationsr  s       r@   export_sync_only_new_backgroundr    s'    ##'':'6G%%'r?   c            	          t        | d   t        j                  j                        ro| d   }|j                  d   }|j                  d   }|j
                  j                  |      j                         }|t        j                  d| d| d| d       y t        | d   t              rG| d   j                  }| d   j                  j
                  j                  |      j                         }nt        d|        |j                          y )	Nr   r$   r  rh    z not found at job z failurezUnknown storage in )r   r   jobr   argsrm   rN  rI  rp   r   r   rr   rl   ry   r   )r  r   r   _classr  r   s         r@   rv  rv    s    $q'266::&7q!]]1%
..'':'6<<>?KK(6(!J<7I(S[\] 
DGW	% !WZZ
q'##++22j2AGGI.tf566 r?   c              #      K   |dk  rt        d      t        |       }t        t        j                  ||            x}r'| t        t        j                  ||            x}r&y y w)Nr$   zn must be at least one)ry   itertuple	itertoolsislice)iterablenitbatchs       r@   rJ  rJ    sc     1u122	hB))"a01
1%
1 ))"a01
1%
1s   A"A'%A'c                       e Zd Z ej                   ed      ddd      Z ed ej                         xs ddz        Z
d Zd	 Zd
ej                  e   fdZd Zd ZddefdZ G d d      Zy)ExportStoragecan_delete_objectsTzDeletion from storage enabledrH            c                 \   | j                   j                  j                  }t        d|d      }t        j
                  s|r4ddg}d| j                   i}t        |j                  ||      j                  S t        t        j                        } ||d| j                   i      j                  S )	N4fflag_feat_optic_650_target_storage_task_format_longF)r%  override_system_defaultzannotations.reviewszannotations.completed_byr  )contextexpand)r  )r  r,  r-  r
   r   FUTURE_SAVE_TASK_TO_STORAGEr   r  r  r   STORAGE_ANNOTATION_SERIALIZER)rX   r8  r%  flagr  r  serializer_classs          r@   _get_serialized_dataz"ExportStorage._get_serialized_data  s    ||((33Bgl
 //4 ,-GHF $,,/G'
QWX]]]()O)OP#JDLL8QRWWWr?   c                     t         r   r   )rX   r8  s     r@   save_annotationzExportStorage.save_annotation
  r   r?   r  c                    d}|j                         }| j                          | j                  j                  j                  | _        | j                  j                         }t        d|| j                  z        }t        j                  d| j                   d| d| d| j                   d	       t        | j                        5 }t        t        ||	      |      D ]  }g }|D ]>  }	| j
                  |	_        |j                  |j!                  | j"                  |	             @ t$        j&                  j)                  |      D ]  }
|dz  }| j+                  ||
         	 d d d        | j-                  ||
       y # 1 sw Y   xY w)Nr   r$   zExport storage z: using chunk_size=z (project_batch_size=z, max_workers=))max_workers)
chunk_size)r_   r  )rO  r{   r  r,  r-  cached_userget_task_batch_sizemaxr  rp   r   rr   r   rJ  r   r   submitr  
concurrentfuturesas_completedr   r   )rX   r  annotation_exportedr  project_batch_sizer  executorannotation_batchr  r8  futures              r@   save_annotationszExportStorage.save_annotations  s   '--/!!#<<44?? "\\==?.$2B2BBC
dggY&9* F##5"6nTEUEUDVVWY	

  D,<,<= 	x %- D% x  "2 VJ-1-=-=J*NN8??43G3G#TUV )00==gF xF'1,'-->Qev-wxx		x" 	0CWhi#	x 	xs   BFFc                 v    | j                  t        j                  j                  | j                               y )Nr  )r  r   rm   rN  r  rd   s    r@   r  z"ExportStorage.save_all_annotations.  s(    j00777MNr?   c                 .   | j                   j                  }t        j                  j	                  | j
                        j                  |j                  j	                  | | j
                        j                  d            }| j                  |       y)z^Do not update existing annotations, only ensure that all annotations have an ExportStorageLinkr  )r   annotation__projectannotation_id)id__inN)	linksmodelr   rm   rN  r  excludevaluesr  )rX   storage_link_modelnew_annotationss      r@   r  z'ExportStorage.save_only_new_annotations1  s     "ZZ--$,,33DLL3IQQ%--44TW[WcWc4dkk R 

 	o.r?   r  c           
      t   |rt         }nt        }t               r| j                         sy t	        || j
                  | j                  dt        j                  | j                  j                  | j                  j                  j                  t              }| j                  |j                         t        j                  d|j                   d|  d       y 	 t        j                  d|         | j                         sy  || j
                  | j                         y # t        $ r t        |        Y y w xY w)Nr   )rm  rq  rn  ro  rp  rr  rs  z has been queuedrt  )r  r  r   ru   r   rl   rr   r   rw  r  r,  rv  rZ   rp   r   r
  )rX   r  export_sync_fnr   s       r@   rx  zExportStorage.sync<  s    $<N3N''). $88<<?? $ 9 9 < <5	H hkk*KK6x{{m=QUPVVfgh14TF;<++-t~~tww7 1*401s   (D  D   D76D7c                       e Zd ZdZy)ExportStorage.MetaTNrD   r>   r?   r@   rF   r  Z  rG   r?   rF   N)F)r4   r5   r6   r   r   r7   r  minrQ  	cpu_countr  r  r  QuerySetr   r  r  r  r{  rx  rF   r>   r?   r@   r  r    s    ,,,	
d$Ba
 a,",,.-A23KX""jFOOJ,G jBO	/1d 1< r?   r  c                      e Zd Z ej                  dej
                  d      Z ej                   ed      dd      Z	 ej                   ed      d	d
      Z ej                   ed      d
d      Z ej                  d
d
d      Z ej                  d
d
d      Zedee   fd       Zedd       Z G d d      Zy)rj  z
tasks.Task%(app_label)s_%(class)sr  r  r   FzExternal link key)rI   rK   object exists/Whether object under external link still existsTrK   rR   r   r   r   zParquet row grouprH   z*Parquet row index, or JSON[L] object indexr   c                     t        | j                  j                  ||j                        j	                  dd      j                               S )Nr>  r   T)flat)setrm   rN  rr   values_listdistinct)r/  r  r   s      r@   rM  zImportStorageLink.existsl  s@    3;;%%dGJJ%GSSTY`dSennpqqr?   Nc                 b    | j                   j                  |j                  ||||d      \  }}|S )NT)r   r   	row_index	row_groupr   object_exists)rm   get_or_createrr   )r/  r  r   r   r  r  linkcreateds           r@   r+  zImportStorageLink.createp  s9    11GG	YX_os 2 
g r?   c                       e Zd ZdZy)ImportStorageLink.MetaTNrD   r>   r?   r@   rF   r  w  rG   r?   rF   )NN)r4   r5   r6   r   OneToOneFieldr  r  r   r7   r   r   r  r   r   IntegerFieldr  r  r|  r  rb   rM  r+  rF   r>   r?   r@   rj  rj  ^  s    6UnoD
&

1U8%;N
OC (F''	/&WaeM &%%aoDTcdJ###TEXYI###TEqrIrc#h r r   r?   rj  c                   F   e Zd Z ej                  dej
                  d      Z ej                   ed      dd      Z	 ej                   ed      dd	
      Z ej                   ed      dd      Zed        Zed        Zed        Zed        Zd Z G d d      Zy)ExportStorageLinkztasks.Annotationr  r  r  r  Tr  r   r   r   z
updated atzUpdate time)auto_nowrK   c                 8   t        | dd       }| | j                  j                  j                  }t	        d|      }t
        j                  s|r8t
        j                  s|rdnd}t        | j                  j                        |z   S t        | j                        S )Nr  r  r$  rC  r   )getattrr  r,  r-  r
   r   r  $FUTURE_SAVE_TASK_TO_STORAGE_JSON_EXTrb   r  rr   )r8  r%  r  rb  s       r@   get_keyzExportStorageLink.get_key  s     z=$7<%%22==DNUYZ//4%JJd'XZCz))*S00z}}%%r?   c                 8    | j                  | j                        S r   )r  r8  rd   s    r@   r   zExportStorageLink.key  s    ||DOO,,r?   c                     | j                   j                  |j                  |j                        j                         S )N)r8  r   )rm   rN  rr   rM  )r/  r8  r   s      r@   rM  zExportStorageLink.exists  s-    {{!!Z]]GJJ!OVVXXr?   c                 l    | j                   j                  ||d      \  }}|s|j                          |S )NT)r8  r   r  )rm   r  rW   )r/  r8  r   r  r  s        r@   r+  zExportStorageLink.create  s4    11ZQXhl1mgIIKr?   c                 r    | j                   j                  |_        | j                   j                  |      ryyr  )r8  r  r  r  s     r@   r  z ExportStorageLink.has_permission  s+    ..??))$/r?   c                       e Zd ZdZy)ExportStorageLink.MetaTNrD   r>   r?   r@   rF   r    rG   r?   rF   N)r4   r5   r6   r   r  r  r8  r   r7   r  r   r   
updated_atr   r  r   r   r|  rM  r+  r  rF   r>   r?   r@   r  r  {  s    """fnnC\J (F''	/&WaeM &%%aoDTcdJ%%%aoP]^J& & - - Y Y   r?   r  )Zr   r  concurrent.futuresr  r  rU  loggingrQ  r   r   r   r   dataclassesr   r   typingr   r   r   urllib.parser	   r   r   rq.exceptionscore.feature_flagsr
   
core.redisr   r   core.utils.commonr   core.utils.iteratorsr   data_export.serializersr   django.confr   django.contrib.auth.modelsr   	django.dbr   r   django.db.modelsr   django.shortcutsr   django.utilsr   django.utils.translationr   r7   fsm.functionsr   io_storages.utilsr   r   r   rest_framework.exceptionsr   rq.jobr   tasks.modelsr   r   tasks.serializersr    r!   webhooks.modelsr"   webhooks.utilsr#   r   r%   	getLoggerr4   rp   Modelr'   r   r   r~  rw  ru  r  r  rv  rJ  r  rj  r  r>   r?   r@   <module>r     s)   l      	 
  1   ' '    	  ' ? ' 1 8   4 ) & $ ! 6 7 P P 5  ) H ) 5 2			8	$tw&,, twnk  oG od&,, $ ?G>Z>Z 	#
(
2jG0 jZ :0 0r?   