]j?u d Z ddlZddlZddlZddlmZ ddlmZ ddlm Z ddl
m Z m Z m
Z
mZ ddlmZ dd lmZ ed
d Zd
dddddgZddddgddddgddddddgddddgddgZd Zej0 j3 e d ej0 j5 d ee eddddgddddgd ! eddddgddddgd ! gedd"d#fee eddddgddddgd ! eddddgddddgd ! gedd"d fg ej0 j6 d$ Z ej0 j3 e d ej0 j5 d%d& ed
d'd() d"ddiddigddgd*d+d,d,dgidgdd'd-d*d+d,d,dgidgdd'd-gd.dddf
d/ ed
d'd0) d"ddiddigddgd*d+d,d,dgidgdd"d-d*d+d,d,dgidgdd"d-gddddf
d1 ed
d"d0) d"ddiddigddgd*d+d,d,dgidgdd"d-d*d+d,d,dgidgdd'd-gddddf
d2 ed
d'd3) d"ddiddigddgd*d+d,d,dgidgdd"d-d*d+d,d,dgidgdd'd-gddddf
d4 ed
d"d3) d"ddiddigddgddgd.dddf
d5 ed
d"d6) d"ddiddigddgd*d+d,d,dgidgdd"d-dgddddf
d7 ed
d"d8) d"ddiddigdd*d+d,d,dgidgd d!gd*d+d,d,dgidgdd"d-dgddddf
d9 ed
d"d8) d"ddiddigddgd*d+d,d,dgidgdd'd-dgd.dddf
d: ed
d'd;) d"ddiddigddgd*d+d,d,dgidgdd'd-dgd.dddf
d< ed
d'd=) d"ddiddiddigg d>d*d+d,d,dgidgdd'd-d*d+d,d,dgidgdd"d-dgd.dddf
g
ej0 j6 d? Z ej0 j6 d@ Z ej0 j6 dA Z ej0 j6 dB Z ej0 j6 dC Z!y)DzThis file and its contents are licensed under the Apache License 2.0. Please see the included NOTICE for copyright information and LICENSE for a copy of the license.
N)redis_healthcheck) MLBackend)Project)
AnnotationAnnotationDraft
PredictionTask)User ) make_projectTestaB
)title label_configz meta info Aztext A) meta_infotextz meta info Bztext B
text_classr labelsclass_A)r startend from_nameto_nametypevalueffffff?resultscoreclass_BgzG?c G d d }t j j |j j d j }| j d|j d S )Nc e Zd Zd Zy)%run_task_predictions..TestJobc || _ y )Nid)selfjob_ids M/root/env/lib/python3.12/site-packages/label_studio/tests/test_predictions.py__init__z.run_task_predictions..TestJob.__init__= s DG N)__name__
__module__ __qualname__r* r+ r) TestJobr# < s r+ r0 )projecthttp://localhost:8999url/api/ml//predict)r objectsfilterr& firstpost)clientr1 mockerr0 ms r) run_task_predictionsr> ; s_ 4;;@W;X^^`A ;;!$$x0 11r+ z2Starting predictions requires Redis server enabled)reasonzpproject_config, tasks, annotations, prediction_results, log_messages, model_version_in_request, use_ground_truthT)r ground_truth12345Fc t || j } t || D ]J \ }
} t j j |
| } |s*t
j j
dd| i| L t j 5 }
|
j dt j d|i |
j dt j |d d |d t | | | }|j d k( sJ |
j sJ d d d t j j! }t" j j% | j&
} t( j j% d }|j+ t- | k( sJ t || D ]N \ }}|j. |d
d k( sJ |j0 |d
d k( sJ |j2 |j2 k( rNJ y # 1 sw Y xY w)N)datar1 taskhttp://localhost:8999/setup
model_versionr http://localhost:8999/predictr resultsrF r% r2 r3 r r r r/ )r userzipr r7 creater
requests_mockMockerr: jsondumpsr> status_codecalledr allr getr& r countlenr r rF )business_clientproject_configtasks annotationsprediction_results log_messagesmodel_version_in_requestuse_ground_truthr< r1 rD
annotationtr= r predictions
ml_backendactual_predictionexpected_prediction_results r) test_predictionsrh D s @ >?+?+?@G { 3 <j LL T7 ; % % ;1 ;
;<
1 ,4::Ph>i3jk +(:2A(>Qi jk
! '6 B}}###xxx $$((*Koo!!WZZ!0G""&&+B&CJ #e* ,, ,9<[J\9] K55 ''+=a+@+JJJJ &&*
)r rF r clstxtchoices)r r rF z All predictions are up-to-datez
zXSome predictions are outdated, other are up-to-date. project.model_version is up-to-datezVSome predictions are outdated, other are up-to-date. project.model_version is outdatedz
z All tasks has no predictionsz4Some tasks has no predictions, others are up-to-datea'
zXSome tasks has no predictions, others are up-to-date, labeled task contains ground_truthz
z2Some tasks has no predictions, others are outdatedzUSome tasks has no predictions, others are outdated, project.model_version is outdatedz
zISome tasks has no predictions, others are outdated, others are up-to-datez
)NNNc N t || j } t j j d }
|d |
_ |
j
t ||| D ]y \ }}}t j j | | }|!t j j dd|i| |Nt j j d||j d| { t j 5 }|j ddt! j" d|i |j dd
t! j" d d d
d
dgidgddg|d t% | | | }|j& dk( sJ t) t+ t- d |j. |k( sJ t j j- | j0 | j3 t) | k( sJ t j j d j
|k( sJ d d d y # 1 sw Y y xY w)Nr2 r3 rF )r1 rC rD )rD r1 POSTrE rG rH rj rk rl r r r r rI rK c 8 | j j d S )Npredict)r4 endswith)hs r) zAtest_predictions_with_partially_predicted_tasks.. s )B r+ )r1 rF r/ )r rL r r7 rV rF saverM r rN r r r1 rO rP register_urirQ rR r> rS rX listr8 request_historyr& rW )rY test_namesetup_returns_model_versionrZ r[ r\ input_predictionsprediction_call_countnum_project_statsnum_ground_truth_in_stats num_ground_truth_fit_predictionsr< r1 re rD ra
predictiontask_objr= rc s r) /test_predictions_with_partially_predicted_tasksr s7 J >?+?+?@G""&&+B&CJ -o >J OO(+E;@Q(R ]$j*<<&&wT&B ! % % B8 Bz B ! % % \8X=M=M \Q[ \ ]
"o1 1
OUpCq8r
+ 27/4,5.7)-E !"' &'
&A
.
! '6 B}}###4 BADUDUVWX\qqqq!!((Kf(gmmosv t
$$)@$AOOSnnnnE"o "o "os
.D#HH$c . |j j }d|_ |j |j j }t j d 5 }|j d|j ddddigid | j d |j d
t j |j d d id d
}d|_ |j }d|v sJ d|d v sJ |d d dk( sJ d d d y # 1 sw Y y xY w)NT real_httpro r6 rJ xrK rQ rS r5 /interactive-annotatingyrD contextapplication/jsonrC content_typerC ) ml_backendsr9 is_interactiveru r[ rO rP rv r4 r: pkrQ rR r& rS )rY configured_projectre rD r= rc r s r) test_interactive_annotatingr s% $//557J $J OO # # ) ) +D - * v*..!1 :)sTWj\AZhkl z}}o%< = GG #Sz ,
!
fVn$$$f~c"c)))'* * *s
"B D Dc |j j }d|_ |j |j j }| j d|j dt j |j ddid d }d|_
|j }d |v sJ t j d
5 }|j d |j d d
dgggid | j d|j dt j |j ddid d }d|_
|j }d d d d |v sJ y # 1 sw Y xY w)NTr5 r r r r r rK errorsr ro r6 kebabeatr )r r9 r ru r[ r: r rQ rR r& rS rO rP rv r4 )rY r re rD rc r r= s r) #test_interactive_annotating_failingr sm $//557J $J OO # # ) ) +D
:==/!89
ZZ :
( A AM
VVXF v
- v*..!1 :'eWI;AWehi z}}o%< = GG #Sz ,
!
" v # s
BEE c |j j }d|_ |j t t
j j }|j j }t j j ||d i d t j j ||d i d t j d 5 }|j d|j dd d
d
igid | j d
|j dt# j$ |j& ddid d }d |_ |j# }d|v sJ d
|d v sJ |d d
d
k( sJ |j* D cg c] }d|j, v s| c}d } | j. sJ t# j0 | j. }
t3 |
d d d dk( sJ ddd yc c}w # 1 sw Y yxY w)zy
Test interactive annotating with drafts
:param business_client:
:param configured_project:
:return:
Tr r )rD rL r lead_timerm r ro r6 rJ r rK r r5 r r r r r rC rq r[ draftsN)r r9 r ru rw r
r7 rU r[ r rN rO rP rv r4 r: r rQ rR r& rS rx pathr loadsrX ) rY r re usersrD r= rc r reqhistoryjss r) 'test_interactive_annotating_with_draftsr s $//557J $J OO !!# $E # # ) ) +D""58BRS"T""58BRS"T - 2 v*..!1 :)sTWj\AZhkl z}}o%< = GG #Sz ,
!
fVn$$$f~c"c)))"#"3"3M3yCHH7L3MaP|||
ZZ
%2g;q>(+,11152 2* N+2 2s& B-G5G0G0A G50G55G>c
ddl m}m}m} |j j }|j j ||j ddddddgidgid d
}|j j ||j d d
}|j j | }|j j |j } | j J | j J |j j | }|j j |j } | j J | j J t j t" 5 |j j || d d d t j t" 5 |j j d d d y # 1 sw Y FxY w# 1 sw Y y xY w)Nr )FailedPredictionr PredictionMetar r r rl r r r rA )rD r1 r r rF error)rD r1 messagerF )r r% )failed_prediction)r r ) tasks.modelsr r r r[ r9 r7 rN r1 rV r& inference_timer
total_costr pytestraises Exception)
rY r r r r rD r r pmetas
r) test_predictions_metar 7 s II # # ) ) +D ##**
*vy\ehqgr[st
+
J )0077
8 %%% markskipif parametrize django_dbrh r r r r r r/ r+ r) r s
( # F F -1
. * 2 2! *! %.K!AF
*! %.K!AF
) %42 )++4hiv
7 ) *6'-$,1: aPQ%R "&
*6'-$,1: aPQ%R "&
2
2 I%
P
7 ) *6'-$,1: aPQ%R "&
*6'-$,1: aPQ%R "&
2
2 I%
OMPb )K cP jf)KX )++4hi'
R )
h
&(!3 4
4L
',y\ehqgr[st "%0 ',y\ehqgr[st "%0
"
Y-
`
- )
h
&(!3 4
4L
',y\ehqgr[st "%, ',y\ehqgr[st "%,
"
Y-
`
g %
h
&(!3 4
4L
',y\ehqgr[st "%, ',y\ehqgr[st "%0
"
Y-
`
e )
h
&(!3 4
4L
',y\ehqgr[st "%, ',y\ehqgr[st "%0
"
Y-
`
+ %
h
&(!3 4
4L
4L
;
B
C %
h
&(!3 4
4L
',y\ehqgr[st "%,
M'
T
g %
h
&(!3 4 ',y\ehqgr[st %)
',y\ehqgr[st "%,
]/
d
A %
h
&(!3 4
4L
',y\ehqgr[st "%0
M'
T
d )
h
&(!3 4
4L
',y\ehqgr[st "%0
M'
T
X )
h
&(!3fh5G H
',y\ehqgr[st "%0 ',y\ehqgr[st "%,
$
[.
A o tj
<o k
t jn
<o~ * *< + + \ ,2 ,2^ -( -(r+