U
Qab @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ddl m
Z
ddlmZmZmZmZ ddlZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m Z m!Z! ddlm"Z" ddl#m$Z$ dd l%m&Z& dd
l'm(Z( dd l)m*Z* dd l+m,Z, dd
l-m.Z. e/Z0e1dZ2e1dej3Z4e1dZ5e1dej3j6Z7d8 Z9ddddgZ:dZ;dZ edZ?dd Z@d d ZAd d ZBdGd dZCdHd!d"ZDdId#d$ZEdedfd%dZFdJd&d'ZGd(d) ZHe1d*ej3ZIeHd+d, ZJG d-d. d.ZKG d/d0 d0eKZLG d1d deZMe1d2jNZOd3d4 ZPd5d6 ZQdKd7d8ZRd9d: ZSG d;d< d<ZTG d=d> d>ejUZVejWjXfd?d@ZYdAdB ZZeRe;eYZYdCdD Z[dEdF Z\dS )Lz#PyPI and direct package downloading Nwraps)six)urllib http_client configparsermap)
CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST) ssl_support)log)DistutilsError) translate)get_all_headers)unescape)Wheelz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z .tar.gz .tar.bz2 .tar .zip .tgz PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_name z rP )filenamerG r& r& r' distros_for_filename s
rT c
c s| | d}|s,tdd |dd D r,dS tdt|d D ]8}t| |d|d| d||d |||dV q>dS )zGenerate alternative interpretations of a source distro name
Note: if `location` is a filesystem filename, you should call
``pkg_resources.normalize_path()`` on it before passing it to this
routine!
rJ c s s | ]}t d |V qdS )z py\d\.\d$N)rerE ).0pr& r& r' s z(interpret_distro_name.. Nr6 )
py_versionrB rQ )r9 anyrangerO r
join)rK rP rG rZ rB rQ r; rW r& r& r' r s
c c sf t }|j}|dkr:tj|j| D ]}|| |V q$n(| D ]"}||}||kr>|| |V q>dS )zHList unique elements, preserving order. Remember all elements ever seen.N)setaddr Zmoves filterfalse __contains__)iterablekeyseenZseen_addZelementkr& r& r' unique_everseen s
rf c s t fdd}|S )zs
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | |S N)rf )argskwargsfuncr& r' wrapper s z unique_values..wrapperr )rk rl r& rj r'
unique_values s rm z(<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c c s t |D ]d}| \}}tttj| d}d|ksDd|kr
t |D ] }t
j | t
|dV qNq
dD ]@}||}|dkrtt ||}|rtt
j | t
|dV qtdS )zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager3 r6 )z
| Home Pagez | Download URLr2 N)RELfinditergroupsr^ r strstripr+ r9 HREFr r# urljoin
htmldecoderF findsearch)r: pagerE tagZrelZrelsposr& r& r' find_external_links s
r| c @ s( e Zd ZdZdd Zdd Zdd ZdS ) ContentCheckerzP
A null content checker that defines the interface for checking content
c C s dS )z3
Feed a block of data to the hash.
Nr& selfblockr& r& r' feed s zContentChecker.feedc C s dS )zC
Check the hash. Return False if validation fails.
Tr& r r& r& r' is_valid s zContentChecker.is_validc C s dS )zu
Call reporter with information about the checker (hash name)
substituted into the template.
Nr& )r reportertemplater& r& r' report s zContentChecker.reportN)__name__
__module__ __qualname____doc__r r r r& r& r& r' r} s r} c @ sB e Zd ZedZdd Zedd Zdd Z dd Z
d
d Z d S )
HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s || _ t|| _|| _d S rg ) hash_namehashlibnewhashexpected)r r r r& r& r' __init__ s zHashChecker.__init__c C s> t j|d }|st S | j|}|s0t S | f | S )z5Construct a (possibly null) ContentChecker from a URLr2 )r r# r7 r} patternrx groupdict)clsr: r@ rE r& r& r' from_url s zHashChecker.from_urlc C s | j | d S rg )r updater~ r& r& r' r s zHashChecker.feedc C s | j | jkS rg )r Z hexdigestr r r& r& r' r " s zHashChecker.is_validc C s || j }||S rg )r )r r r msgr& r& r' r % s
zHashChecker.reportN) r r r rU compiler r classmethodr r r r r& r& r& r' r s
r c s< e Zd ZdZdJddZdKd d
ZdLd d ZdMd
dZdd Zdd Z dd Z
dd Z dNddZ dd Z
dO fdd Zd d Zd d Zd!d" Zd#d$ Zd%d& ZdPd'd(ZdQd)d*Zd+d, Zd-Zd.d/ Zd0d1 ZdRd2d3Zd4d5 Zd6d7 Z d8d9 Z d:d; Z dd?Z!d@dA Z"dBdC Z#dDdE Z$dFdG Z%dHdI Z& Z'S )Tr z;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTc O s t j| f|| |dd |d | _i | _i | _i | _td t
t |j | _
g | _|ortjor|prt }|rt|| _n
tjj| _d S )Nr1 |)r r r, index_url scanned_urls fetched_urls
package_pagesrU r r] r r rE allowsto_scanr Z is_availableZfind_ca_bundleZ
opener_foropenerr requesturlopen)r r ZhostsZ ca_bundleZ
verify_sslrh kwZuse_sslr& r& r' r - s
zPackageIndex.__init__Fc C s || j kr|sdS d| j |< t|s2| | dS tt|}|r\| |sPdS | d| |sn|rn|| jkrtt| j | dS | |sd| j|< dS |
d| d| j|< d}| ||| }|dkrdS d| j|j < d|j
dd kr | dS |j }| }t|tsNt|tjjr0d }n|j
d
p@d }||d }| t|D ](} tj |t | d }
| |
q`| | j!rt"|d
ddkr| #||}dS )z existswarnisdirrealpathlistdirr r] rT r r r r_ )r fnnestedr> itemr r& r& r' r u s z PackageIndex.process_filenamec C sb t |}|o |d dk}|s8| tj|d r entryr& r& r' rX s
z.PackageIndex.scan_egg_links..)filterrR r> r r itertoolsstarmap
scan_egg_link)r Z search_pathdirsZ egg_linksr& r& r' scan_egg_links s
zPackageIndex.scan_egg_linksc C s t tj|| }ttd ttj|}W 5 Q R X t |dkrDd S |\}}t
tj||D ](}tjj|f| |_ t |_
| | q^d S )NrY )openrR r> r] r r r rr rs rO r rK r
rB r_ )r r> r Z raw_lineslinesZegg_pathZ
setup_pathrH r& r& r' r s zPackageIndex.scan_egg_linkc
s fdd}t |D ]:}z |tj|t|d W q tk
rN Y qX q||\}}|rt||D ]H}t |\}} |
dr| s|r|d||f 7 }n
| | qlt
dd |S dS d S )
z#Process the contents of a PyPI pagec s | jrtttjj| t jd d}t|dkrd|d krt |d }t
|d }d j |
i | < t|t|fS dS )Nr1 rY r5 r6 r T)NN)r- r r r r r# r8 rO r9 r r r
setdefaultr+ r )r r; pkgverr r& r' scan s z(PackageIndex.process_index..scanr6 .pyz
#egg=%s-%sc S s d| ddd S )Nz%sr6 rY )rF )mr& r& r' z,PackageIndex.process_index..r N)rt rp r r# ru rv rF r$ r| rA r, need_version_infoscan_urlPYPI_MD5sub)
r r: ry r rE r r new_urlr/ fragr& r r' r s(
zPackageIndex.process_indexc C s | d| d S )NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allr r: r& r& r' r s z PackageIndex.need_version_infoc G s: | j | jkr*|r | j|f| | d | | j d S )Nz6Scanning index of all packages (this may take a while))r r r r r r r rh r& r& r' r s zPackageIndex.scan_allc C sz | | j|j d | j|js:| | j|j d | j|jsR| | t| j|jdD ]}| | qfd S )Nr1 r& ) r r unsafe_namer r rc rL not_found_in_indexr )r requirementr: r& r& r'
find_packages s
zPackageIndex.find_packagesc sR | | | | |j D ]"}||kr0| S | d|| q tt| ||S )Nz%s does not match %s)prescanr rc r superr obtain)r r Z installerrH __class__r& r' r s
zPackageIndex.obtainc C sL | | jd| | sH| t| td|jjtj
|f dS )z-
checker is a ContentChecker
z Validating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) r r r r rR unlinkr r r. r> rP )r checkerrS tfpr& r& r'
check_hash s
zPackageIndex.check_hashc C sN |D ]D}| j dks0t|r0|ds0tt|r<| | q| j | qdS )z;Add `urls` to the list that will be prescanned for searchesNfile:)r r r- r r r append)r Zurlsr: r& r& r' add_find_links s
zPackageIndex.add_find_linksc C s" | j rtt| j| j d| _ dS )z7Scan urls scheduled for prescanning (e.g. --find-links)N)r r r r r r& r& r' r s zPackageIndex.prescanc C s< | |j r| jd }}n | jd }}|||j | d S )Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rc r r r r )r r Zmethr r& r& r' r # s
z PackageIndex.not_found_in_indexc C s~ t |tsjt|}|rR| |d||}t|\}}|drN| |||}|S tj
|rb|S t |}t |
||ddS )a Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
r6 r rK N)r r r
_download_urlrF rA r, gen_setuprR r> r r( r fetch_distribution)r r% tmpdirr< foundr/ r@ r& r& r' r3 - s
zPackageIndex.downloadc s d| i d}d
fdd }|rH | ||}|s^|dk r^|||}|dkrjdk rx ||}|dkr|s| ||}|dkrdrdpd| n d| |j|jd S dS ) a| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
zSearching for %sNc s |d kr }|| j D ]v}|jtkrF sF|krd| d|< q|| ko\|jtkp\ }|r|j}||_tj
|jr| S qd S )Nz&Skipping development or system egg: %sr6 ) rc rB r r r
r3 rK download_locationrR r> r )ZreqenvrH ZtestZloc
develop_okr Zskippedsourcer r& r' rw g s& z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of r zBest match: %s)rK )N)r r r r r Zcloner ) r r r
force_scanr r Z local_indexrH rw r& r r' r O s2
z PackageIndex.fetch_distributionc C s" | ||||}|dk r |jS dS )a3 Obtain a file suitable for fulfilling `requirement`
DEPRECATED; use the ``fetch_distribution()`` method now instead. For
backward compatibility, this routine is identical but returns the
``location`` of the downloaded distribution instead of a distribution
object.
N)r rK )r r r r r rH r& r& r' fetch s zPackageIndex.fetchc
C s t |}|r*dd t||dd D p,g }t|dkrtj|}tj||krtj ||}ddl
m } |||st
|| |}ttj |dd2} | d|d j|d jtj|d f W 5 Q R X |S |rtd ||f ntd
d S ) Nc S s g | ]}|j r|qS r& )rM )rV dr& r& r'
s z*PackageIndex.gen_setup..r6 r )samefilezsetup.pywzIfrom setuptools import setup
setup(name=%r, version=%r, py_modules=[%r])
zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rD rE r rF rO rR r> rP dirnamer] Z setuptools.command.easy_installr shutilZcopy2r writerL rM splitextr )
r rS r@ r rE r rP Zdstr r r& r& r' r sB
zPackageIndex.gen_setupi c C s | d| d }zt|}| |}t|tjjrJt d||j
|j f | }d}| j }d}d|krt
|d} ttt| }| ||||| t|dV}
||} | r|| |
| |d7 }| ||||| qqq| |||
W 5 Q R X |W S |r| X d S ) NzDownloading %szCan't download %s: %s %sr r2 zcontent-lengthzContent-Lengthwbr6 )r r r r r r r r r r r r dl_blocksizer maxr int
reporthookr r r r
r ) r r: rS fpr r blocknumZbssizeZsizesr r r& r& r' _download_to s:
zPackageIndex._download_toc C s d S rg r& )r r: rS r Zblksizer r& r& r' r s zPackageIndex.reporthookc
C s | drt|S zt|| jW S ttjfk
r } z.z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r-
local_openopen_with_authr r$ r
InvalidURLr] rh r r r r r ZURLErrorreasonZ
BadStatusLinelineZ
HTTPExceptionsocket)r r: Zwarningvr r& r& r' r s>
"zPackageIndex.open_urlc C s t |\}}|r0d|kr4|dddd}qnd}|drJ|d d }tj||}|dksj|d rv| ||S |d
ks|d r| ||S |d r| ||S |d
krt
j t
j
|d S | |d | ||S d S )Nz...\_Z__downloaded__rI r* Zsvnzsvn+Zgitzgit+zhg+r rY T)rA replacer, rR r> r] r-
_download_svn
_download_git _download_hgr r url2pathnamer# r7 r _attempt_download)r r< r: r r. r@ rS r& r& r' r s$
zPackageIndex._download_urlc C s | |d d S )NT)r r r& r& r' r : s zPackageIndex.scan_urlc C s6 | ||}d|dd kr.| |||S |S d S )Nr r r )r r r+ _download_html)r r: rS r r& r& r' r) = s z PackageIndex._attempt_downloadc C sn t |}|D ]>}| r td|rF| t| | || S qLq | t| td| d S )Nz ([^- ]+ - )?Revision \d+:z Unexpected HTML page found at ) r rs rU rx r rR r r% r )r r: r rS r r r& r& r' r* D s
zPackageIndex._download_htmlc C s t dt |ddd }d}| drd|krtj|\}}}}}} |s|drd |d
d kr|d
d d d\}}t |\}
} |
rd |
kr|
d d\} }
d | |
f }nd
|
}| }|||||| f}tj
|}| d|| t
d|||f |S )Nz"SVN download support is deprecatedr5 r6 r r zsvn:@z//r1 rY :z --username=%s --password=%sz --username=z'Doing subversion checkout from %s to %szsvn checkout%s -q %s %s)warningsr UserWarningr9 r+ r- r r# r7
_splituser
urlunparser rR system)r r: rS Zcredsr< netlocr> rW qr authhostuserZpwr; r& r& r' r% S s& zPackageIndex._download_svnc C sp t j| \}}}}}|ddd }|ddd }d }d|krR|dd\}}t j||||df} | |fS )N+r6 r2 r5 r r+ r )r r# Zurlsplitr9 rsplitZ
urlunsplit)r:
pop_prefixr< r2 r> r? r revr& r& r' _vcs_split_rev_from_urli s z$PackageIndex._vcs_split_rev_from_urlc C sl | ddd }| j|dd\}}| d|| td||f |d k rh| d| td ||f |S )
Nr5 r6 r Tr9 z Doing git clone from %s to %szgit clone --quiet %s %szChecking out %sz git -C %s checkout --quiet %sr9 r; r rR r1 r r: rS r: r& r& r' r& { s zPackageIndex._download_gitc C sl | ddd }| j|dd\}}| d|| td||f |d k rh| d| td ||f |S )
Nr5 r6 r Tr< z Doing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qr= r> r& r& r' r' s zPackageIndex._download_hgc G s t j|f| d S rg )r r r r& r& r' r s zPackageIndex.debugc G s t j|f| d S rg )r r r r& r& r' r s zPackageIndex.infoc G s t j|f| d S rg )r r r r& r& r' r s zPackageIndex.warn)r r NT)F)F)F)N)N)FFFN)FF)N)F)(r r r r r r r r r r r r r r r r r r r r3 r r r r r r r r r r) r* r% staticmethodr; r& r' r r r
__classcell__r& r& r r' r * sX
3
+
#
L
)$
# z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s | d}t|S )Nr )rF r )rE Zwhatr& r& r'
decode_entity s
rA c C s
t t| S )a
Decode HTML entities in the given text.
>>> htmldecode(
... 'https://../package_name-0.1.2.tar.gz'
... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz')
'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
)
entity_subrA )textr& r& r' rv s rv c s fdd}|S )Nc s fdd}|S )Nc s2 t }t z | |W S t | X d S rg )r ZgetdefaulttimeoutZsetdefaulttimeout)rh ri Z old_timeout)rk timeoutr& r' _socket_timeout s
z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr& )rk rE rD rj r' rE s z'socket_timeout.<locals>._socket_timeoutr& )rD rE r& rF r' socket_timeout s rG c C s2 t j| }| }t|}| }|ddS )aq
A function compatible with Python 2.3-3.3 that will encode
auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
r )r r# r8 encodebase64Z b64encoder r$ )r4 Zauth_sZ
auth_bytesZ
encoded_bytesZencodedr& r& r' _encode_auth s
rK c @ s( e Zd ZdZdd Zdd Zdd ZdS )
Credentialz:
A username/password pair. Use like a namedtuple.
c C s || _ || _d S rg usernamepassword)r rN rO r& r& r' r s zCredential.__init__c c s | j V | jV d S rg rM r r& r& r' __iter__ s zCredential.__iter__c C s dt | S )Nz%(username)s:%(password)s)varsr r& r& r' __str__ s zCredential.__str__N)r r r r r rP rR r& r& r& r' rL s rL c @ s0 e Zd Zdd Zedd Zdd Zdd Zd S )
PyPIConfigc C sP t dddgd}tj| | tjtjdd}tj |rL|
| dS )z%
Load from ~/.pypirc
rN rO
repositoryr ~z.pypircN) dictfromkeysr RawConfigParserr rR r> r]
expanduserr r )r defaultsZrcr& r& r' r s
zPyPIConfig.__init__c s& fdd D }tt j|S )Nc s g | ]} |d r|qS )rT )r rs )rV sectionr r& r' r s z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)ZsectionsrV r _get_repo_cred)r Zsections_with_repositoriesr& r r' creds_by_repository s
z PyPIConfig.creds_by_repositoryc C s6 | |d }|t| |d | |d fS )NrT rN rO )r rs rL )r r[ Zrepor& r& r' r\ s
zPyPIConfig._get_repo_credc C s* | j D ]\}}||r
| S q
dS )z
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N)r] itemsr- )r r: rT credr& r& r' find_credential
s
zPyPIConfig.find_credentialN)r r r r propertyr] r\ r` r& r& r& r' rS s
rS c C s: t j| }|\}}}}}}|dr0td|dkrFt|\} }
nd} | s~t | } | r~t | } | j
| f} t j d | | rdt
| } ||
||||f}
t j|
}t j|}|d| n t j| }|dt ||}| r6t j|j\}}}}}}||kr6||
kr6||||||f}
t j|
|_|S )
z4Open a urllib2 request, handling HTTP authenticationr, znonnumeric port: '')ZhttpZhttpsN*Authenticating as %s for %s (from .pypirc)zBasic Z
Authorizationz
User-Agent)rb )r r# r7 r, r r r/ rS r` rr rN r r rK r0 r ZRequestZ
add_header
user_agentr: )r: r Zparsedr< r2 r> Zparamsr? r r4 Zaddressr_ r r; r r r s2Zh2Zpath2Zparam2Zquery2Zfrag2r& r& r' r s8
r c C s | d\}}} |r|nd| fS )zJsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.r+ N)
rpartition)r5 r6 Zdelimr& r& r' r/ F s r/ c C s | S rg r& )r: r& r& r'
fix_sf_urlP s rf c
C s t j| \}}}}}}t j|}tj|r<t j| S | drtj
|rg }t |D ]d} tj || }
| dkrt
|
d} | } W 5 Q R X qntj
|
r| d7 } |dj| d q`d}
|
j| d |d} d \}}n
d
\}}} d d i}t| }t j| ||||S )
z7Read a local path, with special support for directoriesr1 z
index.htmlrz<a href="{name}">{name}</a>)r. zB<html><head><title>{url}{files} |