dEd9e2d:e d;e?de d?e2d@e dAee d ee ddf fdBZ@dS )F N)IncrementalDecoder)aliases) lru_cache)findall) GeneratorListOptionalSetTupleUnion)MultibyteIncrementalDecoder )ENCODING_MARKSIANA_SUPPORTED_SIMILAR RE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED UNICODE_SECONDARY_RANGE_KEYWORDUTF8_MAXIMAL_ALLOCATION)maxsize characterreturnc t j | }n# t $ r Y dS w xY wd|v pd|v pd|v p d|v pd|v pd|v S )NFz
WITH GRAVEz
WITH ACUTEz WITH CEDILLAzWITH DIAERESISzWITH CIRCUMFLEXz
WITH TILDE unicodedataname
ValueErrorr descriptions O/opt/cloudlinux/venv/lib64/python3.11/site-packages/charset_normalizer/utils.pyis_accentuatedr s &+I66 uu # ' ; & ' [ ( ' { * ' + '
; &
%%c t j | }|s| S | d }t t |d d S )N r )r
decompositionsplitchrint)r
decomposedcodess r
remove_accentr+ , sO !/ ::J !'',,E s58R ! !! c f t | }t j D ]
\ }}||v r|c S dS )zK
Retrieve the Unicode range official name from a single character.
N)ordr items)r
character_ord
range_name ord_ranges r
unicode_ranger3 7 sO
Y M!8!>!@!@
I I % % & 4r, c X t j | }n# t $ r Y dS w xY wd|v S )NFLATINr r s r is_latinr6 E sF &+I66 uu k !!r! c V | d n# t $ r Y dS w xY wdS )NasciiFT)encodeUnicodeEncodeErrorr s r is_asciir< N sE !!!! uu 4s
&&c d t j | }d|v rdS t | }|dS d|v S )NPTF Punctuationr categoryr3 r character_categorycharacter_ranges r is_punctuationrE W sG ) 29 = =
t%29%=%=O u O ++r, c l t j | }d|v sd|v rdS t | }|dS d|v S )NSNTFFormsr@ rB s r is_symbolrJ f sR ) 29 = =
C+=$=$=t%29%=%=O u o %%r, c 0 t | }|dS d|v S )NF Emoticons)r3 )r rD s r is_emoticonrM u s% %29%=%=O u / ))r, c f | s| dv rdS t j | }d|v S )N> ｜+,;<>TZ)isspacer rA r rC s r is_separatorrX sC i+K K Kt ) 29 = = $ $$r, c V | | k S N)islowerisupperr; s r is_case_variabler] s% )"3"3"5"5 55r, c 6 t j | }|dk S )NCo)r rA rW s r is_private_use_onlyr` s ) 29 = = %%r, c X t j | }n# t $ r Y dS w xY wd|v S )NFCJKr r character_names r is_cjkre sH $))44 uu N ""r! c X t j | }n# t $ r Y dS w xY wd|v S )NFHIRAGANAr rc s r is_hiraganarh H $))44 uu ''r! c X t j | }n# t $ r Y dS w xY wd|v S )NFKATAKANAr rc s r is_katakanarl ri r! c X t j | }n# t $ r Y dS w xY wd|v S )NFHANGULr rc s r is_hangulro sH $))44 uu ~ %%r! c X t j | }n# t $ r Y dS w xY wd|v S )NFTHAIr rc s r is_thairr sH $))44 uu ^ ##r! r1 c D t fdt D S )Nc 3 K | ]}|v V d S rZ ).0keywordr1 s r z-is_unicode_range_secondary.. s( TTw*$TTTTTTr, )anyr )r1 s `r is_unicode_range_secondaryrz s' TTTT4STTT T TTr, c r | du o!| du o | dk o| dk S )NFu )rV isprintabler; s r is_unprintabler~ sR u$ " ! ! # #u , " "
! r, sequence search_zonec t | t st t | }t t
| dt
|| dd }t | dk rdS |D ][}| dd }t j D ] \ }}||k r|c c S ||k r|c c S \dS )zW
Extract using ASCII-only decoder any specified encoding in the first n-bytes.
Nr8 ignoreerrorsr -_)
isinstancebytes TypeErrorlenr r mindecodelowerreplacer r/ )r r seq_lenresultsspecified_encodingencoding_alias
encoding_ianas r any_specified_encodingr s h & & x==G ',3w ,,,-44WX4NN G
7||qt %
%
% / 5 5 7 7 ? ?S I I
.5]__ % % )NM !333$$$$$$ 222$$$$$$ 3 % 4r, r c | dv p>t t j d | j t
S )zQ
Verify is a specific encoding is a multi byte one based on it IANA name
> utf_7utf_8utf_16utf_32 utf_16_be utf_16_le utf_32_be utf_32_le utf_8_sig encodings.{})
issubclass importlib
import_moduleformatr r
)r s r is_multi_byte_encodingr sL
5 5d ; ;<<O#
r, c t D ]I}t | }t |t r|g}|D ] }| | r||fc c S JdS )z9
Identify and extract SIG/BOM in given sequence.
)Nr, )r r r
startswith)r
iana_encodingmarksmarks r identify_sig_or_bomr s
( + +
+9-+H eU # # GE + +D""4((
+$d******
+ + 9r, r c
| dvS )N> r r ru )r s r should_strip_sig_or_bomr s 4 44r, Tcp_namestrictc | dd } t j D ]\ }}| ||fv r|c S |r"t d | | S )Nr r z Unable to retrieve IANA for '{}')r r r r/ r r )r r r r s r iana_namer ! s mmoo%%c3//G
*1 ! !%
~}5 5 5 6 M;BB7KKLLL Nr, decoded_sequencec t }| D ])}t | }|| | *t | S rZ )setr3 addlist)r rangesr rD s r
range_scanr 1 sQ uuF% $ $ )6y)A)A "
?#### <<r, iana_name_a iana_name_bc t | st | rdS t j d | j }t j d | j } |d } |d }d}t d D ]C}t
|g }| | | | k r|dz
}D|dz S ) Ng r r r r r )r r r r r ranger r ) r r decoder_a decoder_bid_aid_bcharacter_match_counti
to_be_decodeds r
cp_similarityr ? s k** .D[.Q.Q s' k** ' k** )y 9 9 9D (y 9 9 9D!"
3ZZ ' ' $aSzz
;;} % %])C)C C C !Q & ! 3 &&r, c 2 | t v o|t | v S )z
Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
the function cp_similarity.
)r )r r s r
is_cp_similarr X s% -- ? 1+> >r, charset_normalizerz)%(asctime)s | %(levelname)s | %(message)slevel
format_stringc t j | }| | t j }| t j | | | d S rZ )logging getLoggersetLevel
StreamHandler setFormatter Formatter
addHandler)r r r loggerhandlers r set_logging_handlerr c sm
t
$
$F
OOE#%%G *=99:::
g r, sequencesr offsets
chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloadc # K |r!|du r |D ]} || | |z }
|
s d S |
V d S |D ]} | |z } | t | dz k r | | | |z } |r |du r|| z } | ||rdnd }
|r| dk r{| | dk rot |d }
|r]|
d |
|vrQt | | d z
d
D ]<}| || } |r |du r|| z } | |d }
|
d |
|v r n=|
V d S ) NF r r r r r r$ )r r r r )r r r r r r r r r r chunk chunk_end cut_sequencechunk_partial_size_chkjs r cut_sequence_chunksr q s +0E99 A#AJ$67E
KKKK $ $ AJI3y>>A---$QZ %78L#
:(8E(A(A *\ 9 '' #8Fxxh ( E %
"Q9Q<43G3G.1*b.A.A& $
" 5 5 56oMM "1a!eR 0 0 " "'09'= / F4D4M4M+6 +EL , 3 3M( 3 S S !8"8!89_LL !E M KKKKI$ $ r, )r )TrZ )A unicodedata2r ImportErrorr r codecsr encodings.aliasesr functoolsr rer typingr r r r
r r _multibytecodecr
constantr r r r r r strboolr r+ r3 r6 r< rE rJ rM rX r] r` re rh rl ro rr r rz r~ r r( r r r r r r floatr r INFOr r r ru r, r r s '&&&& % % % % % % % % % % % % ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 7 7 7 7 7 7 *+++ c d ,+ *+++"S "S " " " ,+" *+++
S
Xc]
,+
*+++" " " " " ,+" *+++ ,+ *+++ ,c ,d , , , ,+ , *+++ & & & & & ,+ & *+++*3 *4 * * * ,+* *+++%C %D % % % ,+% *+++6 6 6 6 6 ,+6&3 &4 & & & & *+++#c #d # # # ,+# *+++(3 (4 ( ( ( ,+( *+++(3 (4 ( ( ( ,+( *+++& & & & & ,+& *+++$s $t $ $ $ ,+$ 33.//000U3 U4 U U U 10U *+++c d ,+ U QT
@ 3 (% E(3-2F,G $53 54 5 5 5 5
s
D
C
c 's ' ' ' ' ' '2s % D
. &*7 777 7 7
7
7 7 7 c]7 sD$ 7 7 7 7 7 7s