\j; UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZmZm Z m!Z!m"Z"m#Z#ddl#m$Z$m%Z%ddl&m'Z'm(Z( ddl)m"Z* ddl,m"Z-ddl.m/Z/d d d d d Z0d diZ1ee2eee3ge$ffe4d <e1jkdeee3ge$fe#jle#jne#jpfD ddl9Z9dee2deejxdefdZ=de2de>e2dzde>e2fdZ?dejdeeAe2e2e3ffdZBdejddfdZCdejddfdZDde2d e2d!e3d"edee2geff d#ZEd$ee2gdfd%ee2de3fd&ZFGd'd(ZGdejddfd)ZHd*ZId+e jde d%ee2d,e jddf d-ZLdejddfd.ZMd/ZNejd0d12ZPePjd3ejd 4ZReRjd5ejd6d78d9:ePjd;<ZVeVjd=d>eRgeNd?@ZXeXjdAdBejdCdD:GdEdFejZZeXjdGdHdIdJgeCdKeZdLMeXjdNdOdPgdQdRje1dSTeVjdUdVeRgeNWZ\e\jeH3e\jdXdYe3gdZd[\eVjd]d^eRgeNWZ]e]jeM3e]jd_d`e3ejxsdaePjZ`e`jre`je`yePjy#e+$rYr+>s*  ^^U#s&(cgSr%)_1_2s r*r0Ks") compare_tocyr%r-r-r1r*r0r0Nsr1ctSr%)snapshotr-r1r*r0r0Oshr1)start take_snapshotparsersregexesreturnc|rXtstjdt|}d|vr,tdtj|j d|St }|S)Nz/yaml loading unavailable, please install pyyamlr!z@The legacy parser is incompatible with custom regexes, ignoring.)file)rsysexitprintstderrremover)r8r9ruless r* get_rulesrCSs\ HHF G'" w  RZZ  NN8 $ L Lr1itemallc|dk(r|sJ|S|jdr'|jdsJ|ddjdS|gS)N*{}r,) startswithendswithsplit)rDrEs r* parse_itemrOesQ s{ s  }}S!!!Abz$$v r1argsc#Kt}|jD]N}|jd\}fdt|gdDD]}||vs|j ||Pyw)N:c 3K|]a}t|rtttndgD]7}|dk7rt t tdndgD]}||dk(rdn||f9cyw)r"Nr) CACHEABLErOlistr#mapint)r(ppccsscss r*r+z#rules_to_parsers..ts 6?mz!T&\2& 8:f s3 1d 341#  27B / / / sA'A*r)setselectorrNrOadd)rPseenr^ptripletr[r\s @@r*rules_to_parsersrcpsq 5DMM ..%1a  $GH  G d"!   s AA-A-c tttj|j}t |}t t |}t|jjd|d|d||z ddtt|}tg|Dchc]\}}}| c}}|j}td|D}|D]\}} } djtttd|| dk7xr| | f} t| |d dd t!|| | |} t#| |} | d z }| d z t |z }t|dd|ddycc}}w)N: z lines, z unique (z.0%)c3K|]O\}}}tjdt|zt|z|xrtj|zQyw)N)mathceillenlog10)r(rar[r\s r*r+zrun_stdout..sG CJ1a !c!f*s1v%) sAA-r"TendflushgeAz>5.2fzs (>4.0fzus/line))rUrVr=internr<rkr]r?namercrCr9maxjoinstrfilter get_parserrun)rPlinescountuniquesr8ra_rBwr[nruparsertsecstpls r* run_stdoutrs` SZZ+ ,E JE#e*oG TYY^^ BugXgYi%PS?TTU VW#D)*G 3'2wq!Q23T\\ BE  NU  A 51axxCq!v+2C!Q.G!HIJ qcT T2Aq!U+  3w$hU# e CE{(34 5 3s E# c @tttj|j}t |dz}tt |}|stjdtg|Dchc]\}}}| c}}|j}ddi}|jd|Dtjtjt|dtj}|j||j!d t#j$|d } i} |d d d k(r>t'| \}} | D]+\} } }t)| | d |}t+|||z| | d | <-t-d|Ds$t-d|Dst-d|Drd | d<|j| y| D]S\}} t/| |}| D]+\} } }t)| | ||}t+|||z|| d | <-|j|Uycc}}w)NrrzNo parser selectedsizernc3NK|]\}}}|d||dk(r|n|d|fyw)rmr"Nr-)r(rar[rs r*r+zrun_csv..s<;B1aA3as!v+QaS!:6s#%unix)dialectquotingc |dSNr-rs r*r0zrun_csv..s qtr1)keyc |dSrr-rs r*r0zrun_csv..s qtr1rrrmc3.K|] \}}}|dk(yw)r!Nr-)r(rars r*r+zrun_csv..s 1gaAAM 1c3.K|] \}}}|dk(yw)r"Nr-)r(rr[s r*r+zrun_csv..s2wq!QqF{2rc3.K|] \}}}|dk(yw)rNr-)r(rr\s r*r+zrun_csv..s-'!QqAv-r)r)rUrVr=rtr<rkrcr>rCr9updatecsv DictWriterstdout QUOTE_MINIMALwriterowsort itertoolsgroupbynextrzr{rEdict)rPr|LENr8rarrBcolumnsrgroupedzeroespsrr) cachesizerows r*run_csvrs SZZ+ ,E e*t C#D)*G  %& 3'2wq!Q23T\\ BErlG NNFM   W !!  A JJw LL^L$^FfXQug& ' ? 1 11 2'2 2 -W- -v 6  26 * " < FE165)U;A'*1e}';C6(!E7# $ < 3 O3s2 H rr)rrBc|dk(rtS|dk(r t|}n:|dk(r t|}n)|dk(r t|}nt j d||t vrt j d|t j|}|tjStt||jS)Nr!rrr zunknown parser zunknown cache algorithm ) rr Re2Resolver RegexResolverr=r>r#getrparser)rr)rrBrr[s r*rzrzs 7  %  5   7  %  ?6*-. F +E956 5Ayay /!Qy\2 3 9 99r1rr|cvtj}|D] }|| tj|z Sr%)timeperf_counter_ns)rr|rlines r*r{r{s< A d     !A %%r1cLeZdZdedeefdZdedeefdZ dededdfd Z y) Beladymaxsizedatac:||_i|_tj|_i|_t |D]0\}}|j j|gj|2|j jD]}|jyr%) rr) collectionsdequequeue distances enumerate setdefaultappendvaluesreverse)selfrriefreqss r*__init__zBelady.__init__s /1 -8->->-@ /1dO 7DAq NN % %a , 3 3A 6 7^^**, E MMO r1rr:cH|j|j|jj|x}rg|jj d|k(sJ|j|x}r't j|j|d|f|S|j|=|S)NrrJ)rpopr)rrpopleftbisectinsort)rrr[dss r* __getitem__zBelady.__getitem__s s! s# #1 #::%%'*c1 11^^C((r( djj2b63-8 JJsOr1entryNcX|j|}|sy|d}t|j|jk\r@||jddkDry|jj \}}|j|=||j|<t j|j||fy)NrJr)rrkr)rrrrr)rrrr next_distancerks r* __setitem__zBelady.__setitem__s ^^C  2 tzz?dll *tzz"~a00::>>#DAq 1  3 djj=#"67r1) r& __module__ __qualname__rWr rxrr rrrr-r1r*rrsK49sx '> 8s8=8T8r1rc ttjddddGfdd}tt t j |jt}tt}t|d|dtttjt|jdz}dtd t ffd }t#j$t'j(t'j*|gt-dt.j1|jD]'\}}|}t3j4t#j6} t9t;|||} D]} | j=| t3j4t#j6} ||k(rd j?d } n7tAd | jC| dD}dj?|||z } t|jDjGdd||dd||jHz |z dzdd| ~~ *y)Nrn)domainsstring user_agentosdevicec.eZdZddZdededeffd Zy)run_hitrates..Counterr:Ncd|_y)Nrr})rs r*rz&run_hitrates..Counter.__init__-s DJr1uarc2|xjdz c_S)Nrr)rrrrs r*__call__z&run_hitrates..Counter.__call__0s JJ!OJHr1)r:N)r&rrrrxrrr)rsr*Counterr,s"  s V = r1rr|r~rrr:ct|Sr%)r)rr|s r*beladyzrun_hitrates..belady;sgu%%r1z{0:>14} {0:>12}rmc34K|]}|jywr%) size_diff)r(r\s r*r+zrun_hitrates..NsU11;;Usfilenamez{:8} bytes ({:3.0f}b/entry)8(z): dz2.0fz % hit rate )%rrALLrUrVr=rtr<rkr]r?rWrirlrv cachesizesr tracemallocr6rproductchainryr#rgccollectr7rrrformatsumr2r&r'r})rPrtotalr~rrr) cache_sizemissesbeforerrafterdiffoverheadr|rs @@r* run_hitratesr#s     A SZZ+ ,E JE#e*oG %'9- G DJJs4??+ ,q 01A&&&&..&v}}"?@ z **,j0ABC D LL   ))+ F?$++C0DU0@0@0TUUH077:%D ~~##%a(*aSD)9efll>RV[=[^a=abf.os7;;y;Qr1rr 11rernTro*r)targetrPdaemonrrrszus/line)rq) rUrVr=rtr<rrrrrrrrrr?randomRandom threadingEventBarrierthreadsrrangeThreadrsamplerkr6rrr]r)rPr|r resolversruresolverrr6rprrsttotlinesrs r* run_threadedrjs SZZ+ ,E -/ *E w{{9/EFG oeU3Q-RST  MO,- - 01 -I $:h b n"D1 MM" ! q 01!t||$ A   VQXXeSZ%@#F eg   ! ! #   u: ,  ! ! #b (D 0 Xe$G,D97:r1zFor good results the sample `file` should be an actual non-sorted non-deduplicated sample of user agent strings from traffic on a comparable (or the actual) site or application targeted for classification. ua_parserepi)progepilog)func)add_helpr<rzutf-8)encodingzeSample user agent file, the file must contain a single user agent string per line, use `-` for stdin.)typehelpcommands)titlebenchz7benchmark various parser configurations on sample filesa]Different sites and applications can have different traffic pattenrs, and thus want different setups and tradeoffs. This subcommand allows testing ua-parser's different base resolvers, caches, anc cache sizes in order to customise the parser to the application's requirements. It's also useful to bench the library itself though.)rparentsr descriptionz-Rz --regexesrbaiCustom regexes.yaml file, if ommitted the benchmark will use the embedded regexes file rom uap-core. Custom regexes files can allow evaluating the performance impact of new rules or cut-down reference files (if legacy rules are nor relevant to your needs). Because YAML is (mostly) a superset of JSON, JSON regexes files will also work fine.c leZdZ ddejdej deeeedfde eddf dZ y) ToFuncNr namespacer option_stringr:c|dk(rt||jty|dk(rt||jtyt d|)Nrrzinvalid output )setattrdestrr ValueError)rrr&rr's r*rzToFunc.__call__sE X  Ityy* 5 u_ Ityy' 2vj9: :r1r%) r&rrargparseArgumentParser Namespacer rxr r rr-r1r*r%r%s\ (, ;'' ;%% ;c8C=$./ ;  } ;  ;r1r%z-Oz--outputrrraBy default (`stdout`) the result of each configuration / combination is printed to stdout with the combination name followed by the total parse time for the file and the per-entry average. `csv` will instead output a valid CSV table to stdout, with a parser combination per column and a cache size per row. Combinations without cache will have the same value on every row. If no combination uses a cache, the output will have a single row with a first cell of value 0.)choicesdefaultr*actionrr^rGz)*:*:{10,20,50,100,200,500,1000,2000,5000}zjA generative selector expression, composed of 3 parts: 1. the parser (base), 2. the cache implementation (z, z) and 3. the cache size. For parser and cache `*` is an alias for stands in for "every value", a bracketed expression for an enumeration, and the selector can be repeated to explicitly list each configuration )nargsr0rhitratesz=measure hitrates of cache configurations against sample files)rr!rz --cachesizes+) 2rirriizHList of cache sizes to test hitrates for, for each cache algorithm. )r2rr0rr zPestimate impact of concurrency and contention on different parser configurationsz-nz --threads)rr0)cr,rrrriorrirrr=r rtypestypingrrrrrrr r r r r rrnrrrrrrrrrrloadersrrrr ImportErrorr ruser_agent_parserrrTr#rxrW__annotations__rrS3FifoSieverSimpleNamespacer5IOBaserCrUrOr.tuplercrrrzr{rrrr r rrEPILOGr-r set_defaultsfp add_argumentFileTypeadd_subparserssub add_parserr Actionr%rwr3threaded cpu_count parse_argsrPr print_helpr-r1r*rRs;           "- , 0%     8>tnS(8SE5L12 23D  XseUl #W[[1  tCy8BII+>8$StCy4/DI 8-- (5c3;O2P 5X''5D544($$44n: ::(+:4<: seSj:2& SE4K & C=& &-8-8`3x))3d3l    ??    C=       $:x))$:d$:N  ! k% @Xe,   3 1 * *-  B D (     4  "   ;X__ ;  u     %  8 911561B0CDD G   >> H D     <( 9   >> [ D     <( BLLN a 99IIdO y    0$u$$$H(%''&K s6;N5O3O 5N>=N>O  O  /P?P