Revert "Dammit."

This reverts commit acf8c73de7e478804b5273fbdd2e7225f7976af9.

1 files changed, 686 insertions, 0 deletions

diff --git a/lib/python2.7/site-packages/pip-0.8.1-py2.7.egg/pip/index.py b/lib/python2.7/site-packages/pip-0.8.1-py2.7.egg/pip/index.py
new file mode 100755
index 00000000..e42d8c86
--- /dev/null
+++ b/lib/python2.7/site-packages/pip-0.8.1-py2.7.egg/pip/index.py
@@ -0,0 +1,686 @@
+"""Routines related to PyPI, indexes"""
+
+import sys
+import os
+import re
+import mimetypes
+import threading
+import posixpath
+import pkg_resources
+import urllib
+import urllib2
+import urlparse
+import httplib
+import random
+import socket
+import string
+from Queue import Queue
+from Queue import Empty as QueueEmpty
+from pip.log import logger
+from pip.util import Inf
+from pip.util import normalize_name, splitext
+from pip.exceptions import DistributionNotFound
+from pip.backwardcompat import WindowsError, product
+from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest
+
+__all__ = ['PackageFinder']
+
+
+DEFAULT_MIRROR_URL = "last.pypi.python.org"
+
+
+class PackageFinder(object):
+    """This finds packages.
+
+    This is meant to match easy_install's technique for looking for
+    packages, by reading pages and looking for appropriate links
+    """
+
+    def __init__(self, find_links, index_urls,
+            use_mirrors=False, mirrors=None, main_mirror_url=None):
+        self.find_links = find_links
+        self.index_urls = index_urls
+        self.dependency_links = []
+        self.cache = PageCache()
+        # These are boring links that have already been logged somehow:
+        self.logged_links = set()
+        if use_mirrors:
+            self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url)
+            logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls))
+        else:
+            self.mirror_urls = []
+
+    def add_dependency_links(self, links):
+        ## FIXME: this shouldn't be global list this, it should only
+        ## apply to requirements of the package that specifies the
+        ## dependency_links value
+        ## FIXME: also, we should track comes_from (i.e., use Link)
+        self.dependency_links.extend(links)
+
+    @staticmethod
+    def _sort_locations(locations):
+        """
+        Sort locations into "files" (archives) and "urls", and return
+        a pair of lists (files,urls)
+        """
+        files = []
+        urls = []
+
+        # puts the url for the given file path into the appropriate
+        # list
+        def sort_path(path):
+            url = path_to_url2(path)
+            if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
+                urls.append(url)
+            else:
+                files.append(url)
+
+        for url in locations:
+            if url.startswith('file:'):
+                path = url_to_path(url)
+                if os.path.isdir(path):
+                    path = os.path.realpath(path)
+                    for item in os.listdir(path):
+                        sort_path(os.path.join(path, item))
+                elif os.path.isfile(path):
+                    sort_path(path)
+            else:
+                urls.append(url)
+        return files, urls
+
+    def find_requirement(self, req, upgrade):
+        url_name = req.url_name
+        # Only check main index if index URL is given:
+        main_index_url = None
+        if self.index_urls:
+            # Check that we have the url_name correctly spelled:
+            main_index_url = Link(posixpath.join(self.index_urls[0], url_name))
+            # This will also cache the page, so it's okay that we get it again later:
+            page = self._get_page(main_index_url, req)
+            if page is None:
+                url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name
+
+        # Combine index URLs with mirror URLs here to allow
+        # adding more index URLs from requirements files
+        all_index_urls = self.index_urls + self.mirror_urls
+
+        def mkurl_pypi_url(url):
+            loc = posixpath.join(url, url_name)
+            # For maximum compatibility with easy_install, ensure the path
+            # ends in a trailing slash.  Although this isn't in the spec
+            # (and PyPI can handle it without the slash) some other index
+            # implementations might break if they relied on easy_install's behavior.
+            if not loc.endswith('/'):
+                loc = loc + '/'
+            return loc
+        if url_name is not None:
+            locations = [
+                mkurl_pypi_url(url)
+                for url in all_index_urls] + self.find_links
+        else:
+            locations = list(self.find_links)
+        locations.extend(self.dependency_links)
+        for version in req.absolute_versions:
+            if url_name is not None and main_index_url is not None:
+                locations = [
+                    posixpath.join(main_index_url.url, version)] + locations
+
+        file_locations, url_locations = self._sort_locations(locations)
+
+        locations = [Link(url) for url in url_locations]
+        logger.debug('URLs to search for versions for %s:' % req)
+        for location in locations:
+            logger.debug('* %s' % location)
+        found_versions = []
+        found_versions.extend(
+            self._package_versions(
+                [Link(url, '-f') for url in self.find_links], req.name.lower()))
+        page_versions = []
+        for page in self._get_pages(locations, req):
+            logger.debug('Analyzing links from page %s' % page.url)
+            logger.indent += 2
+            try:
+                page_versions.extend(self._package_versions(page.links, req.name.lower()))
+            finally:
+                logger.indent -= 2
+        dependency_versions = list(self._package_versions(
+            [Link(url) for url in self.dependency_links], req.name.lower()))
+        if dependency_versions:
+            logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions]))
+        file_versions = list(self._package_versions(
+                [Link(url) for url in file_locations], req.name.lower()))
+        if not found_versions and not page_versions and not dependency_versions and not file_versions:
+            logger.fatal('Could not find any downloads that satisfy the requirement %s' % req)
+            raise DistributionNotFound('No distributions at all found for %s' % req)
+        if req.satisfied_by is not None:
+            found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version))
+        if file_versions:
+            file_versions.sort(reverse=True)
+            logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions]))
+            found_versions = file_versions + found_versions
+        all_versions = found_versions + page_versions + dependency_versions
+        applicable_versions = []
+        for (parsed_version, link, version) in all_versions:
+            if version not in req.req:
+                logger.info("Ignoring link %s, version %s doesn't match %s"
+                            % (link, version, ','.join([''.join(s) for s in req.req.specs])))
+                continue
+            applicable_versions.append((link, version))
+        applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True)
+        existing_applicable = bool([link for link, version in applicable_versions if link is Inf])
+        if not upgrade and existing_applicable:
+            if applicable_versions[0][1] is Inf:
+                logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement'
+                            % req.satisfied_by.version)
+            else:
+                logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)'
+                            % (req.satisfied_by.version, applicable_versions[0][1]))
+            return None
+        if not applicable_versions:
+            logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)'
+                         % (req, ', '.join([version for parsed_version, link, version in found_versions])))
+            raise DistributionNotFound('No distributions matching the version for %s' % req)
+        if applicable_versions[0][0] is Inf:
+            # We have an existing version, and its the best version
+            logger.info('Installed version (%s) is most up-to-date (past versions: %s)'
+                        % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none'))
+            return None
+        if len(applicable_versions) > 1:
+            logger.info('Using version %s (newest of versions: %s)' %
+                        (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions])))
+        return applicable_versions[0][0]
+
+    def _find_url_name(self, index_url, url_name, req):
+        """Finds the true URL name of a package, when the given name isn't quite correct.
+        This is usually used to implement case-insensitivity."""
+        if not index_url.url.endswith('/'):
+            # Vaguely part of the PyPI API... weird but true.
+            ## FIXME: bad to modify this?
+            index_url.url += '/'
+        page = self._get_page(index_url, req)
+        if page is None:
+            logger.fatal('Cannot fetch index base URL %s' % index_url)
+            return
+        norm_name = normalize_name(req.url_name)
+        for link in page.links:
+            base = posixpath.basename(link.path.rstrip('/'))
+            if norm_name == normalize_name(base):
+                logger.notify('Real name of requirement %s is %s' % (url_name, base))
+                return base
+        return None
+
+    def _get_pages(self, locations, req):
+        """Yields (page, page_url) from the given locations, skipping
+        locations that have errors, and adding download/homepage links"""
+        pending_queue = Queue()
+        for location in locations:
+            pending_queue.put(location)
+        done = []
+        seen = set()
+        threads = []
+        for i in range(min(10, len(locations))):
+            t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen))
+            t.setDaemon(True)
+            threads.append(t)
+            t.start()
+        for t in threads:
+            t.join()
+        return done
+
+    _log_lock = threading.Lock()
+
+    def _get_queued_page(self, req, pending_queue, done, seen):
+        while 1:
+            try:
+                location = pending_queue.get(False)
+            except QueueEmpty:
+                return
+            if location in seen:
+                continue
+            seen.add(location)
+            page = self._get_page(location, req)
+            if page is None:
+                continue
+            done.append(page)
+            for link in page.rel_links():
+                pending_queue.put(link)
+
+    _egg_fragment_re = re.compile(r'#egg=([^&]*)')
+    _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
+    _py_version_re = re.compile(r'-py([123]\.[0-9])$')
+
+    def _sort_links(self, links):
+        "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates"
+        eggs, no_eggs = [], []
+        seen = set()
+        for link in links:
+            if link not in seen:
+                seen.add(link)
+                if link.egg_fragment:
+                    eggs.append(link)
+                else:
+                    no_eggs.append(link)
+        return no_eggs + eggs
+
+    def _package_versions(self, links, search_name):
+        for link in self._sort_links(links):
+            for v in self._link_package_versions(link, search_name):
+                yield v
+
+    def _link_package_versions(self, link, search_name):
+        """
+        Return an iterable of triples (pkg_resources_version_key,
+        link, python_version) that can be extracted from the given
+        link.
+
+        Meant to be overridden by subclasses, not called by clients.
+        """
+        if link.egg_fragment:
+            egg_info = link.egg_fragment
+        else:
+            egg_info, ext = link.splitext()
+            if not ext:
+                if link not in self.logged_links:
+                    logger.debug('Skipping link %s; not a file' % link)
+                    self.logged_links.add(link)
+                return []
+            if egg_info.endswith('.tar'):
+                # Special double-extension case:
+                egg_info = egg_info[:-4]
+                ext = '.tar' + ext
+            if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
+                if link not in self.logged_links:
+                    logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
+                    self.logged_links.add(link)
+                return []
+        version = self._egg_info_matches(egg_info, search_name, link)
+        if version is None:
+            logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
+            return []
+        match = self._py_version_re.search(version)
+        if match:
+            version = version[:match.start()]
+            py_version = match.group(1)
+            if py_version != sys.version[:3]:
+                logger.debug('Skipping %s because Python version is incorrect' % link)
+                return []
+        logger.debug('Found link %s, version: %s' % (link, version))
+        return [(pkg_resources.parse_version(version),
+               link,
+               version)]
+
+    def _egg_info_matches(self, egg_info, search_name, link):
+        match = self._egg_info_re.search(egg_info)
+        if not match:
+            logger.debug('Could not parse version from link: %s' % link)
+            return None
+        name = match.group(0).lower()
+        # To match the "safe" name that pkg_resources creates:
+        name = name.replace('_', '-')
+        if name.startswith(search_name.lower()):
+            return match.group(0)[len(search_name):].lstrip('-')
+        else:
+            return None
+
+    def _get_page(self, link, req):
+        return HTMLPage.get_page(link, req, cache=self.cache)
+
+    def _get_mirror_urls(self, mirrors=None, main_mirror_url=None):
+        """Retrieves a list of URLs from the main mirror DNS entry
+        unless a list of mirror URLs are passed.
+        """
+        if not mirrors:
+            mirrors = get_mirrors(main_mirror_url)
+            # Should this be made "less random"? E.g. netselect like?
+            random.shuffle(mirrors)
+
+        mirror_urls = set()
+        for mirror_url in mirrors:
+            # Make sure we have a valid URL
+            if not ("http://" or "https://" or "file://") in mirror_url:
+                mirror_url = "http://%s" % mirror_url
+            if not mirror_url.endswith("/simple"):
+                mirror_url = "%s/simple/" % mirror_url
+            mirror_urls.add(mirror_url)
+
+        return list(mirror_urls)
+
+
+class PageCache(object):
+    """Cache of HTML pages"""
+
+    failure_limit = 3
+
+    def __init__(self):
+        self._failures = {}
+        self._pages = {}
+        self._archives = {}
+
+    def too_many_failures(self, url):
+        return self._failures.get(url, 0) >= self.failure_limit
+
+    def get_page(self, url):
+        return self._pages.get(url)
+
+    def is_archive(self, url):
+        return self._archives.get(url, False)
+
+    def set_is_archive(self, url, value=True):
+        self._archives[url] = value
+
+    def add_page_failure(self, url, level):
+        self._failures[url] = self._failures.get(url, 0)+level
+
+    def add_page(self, urls, page):
+        for url in urls:
+            self._pages[url] = page
+
+
+class HTMLPage(object):
+    """Represents one page, along with its URL"""
+
+    ## FIXME: these regexes are horrible hacks:
+    _homepage_re = re.compile(r'<th>\s*home\s*page', re.I)
+    _download_re = re.compile(r'<th>\s*download\s+url', re.I)
+    ## These aren't so aweful:
+    _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I)
+    _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S)
+    _base_re = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I)
+
+    def __init__(self, content, url, headers=None):
+        self.content = content
+        self.url = url
+        self.headers = headers
+
+    def __str__(self):
+        return self.url
+
+    @classmethod
+    def get_page(cls, link, req, cache=None, skip_archives=True):
+        url = link.url
+        url = url.split('#', 1)[0]
+        if cache.too_many_failures(url):
+            return None
+
+        # Check for VCS schemes that do not support lookup as web pages.
+        from pip.vcs import VcsSupport
+        for scheme in VcsSupport.schemes:
+            if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
+                logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals())
+                return None
+
+        if cache is not None:
+            inst = cache.get_page(url)
+            if inst is not None:
+                return inst
+        try:
+            if skip_archives:
+                if cache is not None:
+                    if cache.is_archive(url):
+                        return None
+                filename = link.filename
+                for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
+                    if filename.endswith(bad_ext):
+                        content_type = cls._get_content_type(url)
+                        if content_type.lower().startswith('text/html'):
+                            break
+                        else:
+                            logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type))
+                            if cache is not None:
+                                cache.set_is_archive(url)
+                            return None
+            logger.debug('Getting page %s' % url)
+
+            # Tack index.html onto file:// URLs that point to directories
+            (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
+            if scheme == 'file' and os.path.isdir(urllib.url2pathname(path)):
+                # add trailing slash if not present so urljoin doesn't trim final segment
+                if not url.endswith('/'):
+                    url += '/'
+                url = urlparse.urljoin(url, 'index.html')
+                logger.debug(' file: URL is directory, getting %s' % url)
+
+            resp = urlopen(url)
+
+            real_url = geturl(resp)
+            headers = resp.info()
+            inst = cls(resp.read(), real_url, headers)
+        except (urllib2.HTTPError, urllib2.URLError, socket.timeout, socket.error, OSError, WindowsError), e:
+            desc = str(e)
+            if isinstance(e, socket.timeout):
+                log_meth = logger.info
+                level =1
+                desc = 'timed out'
+            elif isinstance(e, urllib2.URLError):
+                log_meth = logger.info
+                if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
+                    desc = 'timed out'
+                    level = 1
+                else:
+                    level = 2
+            elif isinstance(e, urllib2.HTTPError) and e.code == 404:
+                ## FIXME: notify?
+                log_meth = logger.info
+                level = 2
+            else:
+                log_meth = logger.info
+                level = 1
+            log_meth('Could not fetch URL %s: %s' % (link, desc))
+            log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req))
+            if cache is not None:
+                cache.add_page_failure(url, level)
+            return None
+        if cache is not None:
+            cache.add_page([url, real_url], inst)
+        return inst
+
+    @staticmethod
+    def _get_content_type(url):
+        """Get the Content-Type of the given url, using a HEAD request"""
+        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
+        if not scheme in ('http', 'https', 'ftp', 'ftps'):
+            ## FIXME: some warning or something?
+            ## assertion error?
+            return ''
+        req = Urllib2HeadRequest(url, headers={'Host': netloc})
+        resp = urlopen(req)
+        try:
+            if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'):
+                ## FIXME: doesn't handle redirects
+                return ''
+            return resp.info().get('content-type', '')
+        finally:
+            resp.close()
+
+    @property
+    def base_url(self):
+        if not hasattr(self, "_base_url"):
+            match = self._base_re.search(self.content)
+            if match:
+                self._base_url = match.group(1)
+            else:
+                self._base_url = self.url
+        return self._base_url
+
+    @property
+    def links(self):
+        """Yields all links in the page"""
+        for match in self._href_re.finditer(self.content):
+            url = match.group(1) or match.group(2) or match.group(3)
+            url = self.clean_link(urlparse.urljoin(self.base_url, url))
+            yield Link(url, self)
+
+    def rel_links(self):
+        for url in self.explicit_rel_links():
+            yield url
+        for url in self.scraped_rel_links():
+            yield url
+
+    def explicit_rel_links(self, rels=('homepage', 'download')):
+        """Yields all links with the given relations"""
+        for match in self._rel_re.finditer(self.content):
+            found_rels = match.group(1).lower().split()
+            for rel in rels:
+                if rel in found_rels:
+                    break
+            else:
+                continue
+            match = self._href_re.search(match.group(0))
+            if not match:
+                continue
+            url = match.group(1) or match.group(2) or match.group(3)
+            url = self.clean_link(urlparse.urljoin(self.base_url, url))
+            yield Link(url, self)
+
+    def scraped_rel_links(self):
+        for regex in (self._homepage_re, self._download_re):
+            match = regex.search(self.content)
+            if not match:
+                continue
+            href_match = self._href_re.search(self.content, pos=match.end())
+            if not href_match:
+                continue
+            url = match.group(1) or match.group(2) or match.group(3)
+            if not url:
+                continue
+            url = self.clean_link(urlparse.urljoin(self.base_url, url))
+            yield Link(url, self)
+
+    _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
+
+    def clean_link(self, url):
+        """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
+        the link, it will be rewritten to %20 (while not over-quoting
+        % or other characters)."""
+        return self._clean_re.sub(
+            lambda match: '%%%2x' % ord(match.group(0)), url)
+
+
+class Link(object):
+
+    def __init__(self, url, comes_from=None):
+        self.url = url
+        self.comes_from = comes_from
+
+    def __str__(self):
+        if self.comes_from:
+            return '%s (from %s)' % (self.url, self.comes_from)
+        else:
+            return self.url
+
+    def __repr__(self):
+        return '<Link %s>' % self
+
+    def __eq__(self, other):
+        return self.url == other.url
+
+    def __hash__(self):
+        return hash(self.url)
+
+    @property
+    def filename(self):
+        url = self.url
+        url = url.split('#', 1)[0]
+        url = url.split('?', 1)[0]
+        url = url.rstrip('/')
+        name = posixpath.basename(url)
+        assert name, (
+            'URL %r produced no filename' % url)
+        return name
+
+    @property
+    def scheme(self):
+        return urlparse.urlsplit(self.url)[0]
+
+    @property
+    def path(self):
+        return urlparse.urlsplit(self.url)[2]
+
+    def splitext(self):
+        return splitext(posixpath.basename(self.path.rstrip('/')))
+
+    _egg_fragment_re = re.compile(r'#egg=([^&]*)')
+
+    @property
+    def egg_fragment(self):
+        match = self._egg_fragment_re.search(self.url)
+        if not match:
+            return None
+        return match.group(1)
+
+    _md5_re = re.compile(r'md5=([a-f0-9]+)')
+
+    @property
+    def md5_hash(self):
+        match = self._md5_re.search(self.url)
+        if match:
+            return match.group(1)
+        return None
+
+    @property
+    def show_url(self):
+        return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
+
+
+def get_requirement_from_url(url):
+    """Get a requirement from the URL, if possible.  This looks for #egg
+    in the URL"""
+    link = Link(url)
+    egg_info = link.egg_fragment
+    if not egg_info:
+        egg_info = splitext(link.filename)[0]
+    return package_to_requirement(egg_info)
+
+
+def package_to_requirement(package_name):
+    """Translate a name like Foo-1.2 to Foo==1.3"""
+    match = re.search(r'^(.*?)(-dev|-\d.*)', package_name)
+    if match:
+        name = match.group(1)
+        version = match.group(2)
+    else:
+        name = package_name
+        version = ''
+    if version:
+        return '%s==%s' % (name, version)
+    else:
+        return name
+
+
+def get_mirrors(hostname=None):
+    """Return the list of mirrors from the last record found on the DNS
+    entry::
+
+    >>> from pip.index import get_mirrors
+    >>> get_mirrors()
+    ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org',
+    'd.pypi.python.org']
+
+    Originally written for the distutils2 project by Alexis Metaireau.
+    """
+    if hostname is None:
+        hostname = DEFAULT_MIRROR_URL
+
+    # return the last mirror registered on PyPI.
+    try:
+        hostname = socket.gethostbyname_ex(hostname)[0]
+    except socket.gaierror:
+        return []
+    end_letter = hostname.split(".", 1)
+
+    # determine the list from the last one.
+    return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])]
+
+
+def string_range(last):
+    """Compute the range of string between "a" and last.
+
+    This works for simple "a to z" lists, but also for "a to zz" lists.
+    """
+    for k in range(len(last)):
+        for x in product(string.ascii_lowercase, repeat=k+1):
+            result = ''.join(x)
+            yield result
+            if result == last:
+                return
+