627 lines
25 KiB
Python
627 lines
25 KiB
Python
|
# ex:ts=4:sw=4:sts=4:et
|
||
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
||
|
|
"""
|
||
|
|
BitBake 'Fetch' implementations
|
||
|
|
|
||
|
|
Classes for obtaining upstream sources for the
|
||
|
|
BitBake build tools.
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Copyright (C) 2003, 2004 Chris Larson
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License version 2 as
|
||
|
|
# published by the Free Software Foundation.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License along
|
||
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
|
#
|
||
|
|
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
|
||
|
|
|
||
|
|
import re
|
||
|
|
import tempfile
|
||
|
|
import subprocess
|
||
|
|
import os
|
||
|
|
import logging
|
||
|
|
import errno
|
||
|
|
import bb
|
||
|
|
import bb.progress
|
||
|
|
import urllib.request, urllib.parse, urllib.error
|
||
|
|
from bb.fetch2 import FetchMethod
|
||
|
|
from bb.fetch2 import FetchError
|
||
|
|
from bb.fetch2 import logger
|
||
|
|
from bb.fetch2 import runfetchcmd
|
||
|
|
from bb.utils import export_proxies
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from bs4 import SoupStrainer
|
||
|
|
|
||
|
|
class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
|
||
|
|
"""
|
||
|
|
Extract progress information from wget output.
|
||
|
|
Note: relies on --progress=dot (with -v or without -q/-nv) being
|
||
|
|
specified on the wget command line.
|
||
|
|
"""
|
||
|
|
def __init__(self, d):
|
||
|
|
super(WgetProgressHandler, self).__init__(d)
|
||
|
|
# Send an initial progress event so the bar gets shown
|
||
|
|
self._fire_progress(0)
|
||
|
|
|
||
|
|
def writeline(self, line):
|
||
|
|
percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
|
||
|
|
if percs:
|
||
|
|
progress = int(percs[-1][0])
|
||
|
|
rate = percs[-1][1] + '/s'
|
||
|
|
self.update(progress, rate)
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
class Wget(FetchMethod):
|
||
|
|
"""Class to fetch urls via 'wget'"""
|
||
|
|
def supports(self, ud, d):
|
||
|
|
"""
|
||
|
|
Check to see if a given url can be fetched with wget.
|
||
|
|
"""
|
||
|
|
return ud.type in ['http', 'https', 'ftp']
|
||
|
|
|
||
|
|
def recommends_checksum(self, urldata):
|
||
|
|
return True
|
||
|
|
|
||
|
|
def urldata_init(self, ud, d):
|
||
|
|
if 'protocol' in ud.parm:
|
||
|
|
if ud.parm['protocol'] == 'git':
|
||
|
|
raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url)
|
||
|
|
|
||
|
|
if 'downloadfilename' in ud.parm:
|
||
|
|
ud.basename = ud.parm['downloadfilename']
|
||
|
|
else:
|
||
|
|
ud.basename = os.path.basename(ud.path)
|
||
|
|
|
||
|
|
ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
|
||
|
|
if not ud.localfile:
|
||
|
|
ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
|
||
|
|
|
||
|
|
self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
|
||
|
|
|
||
|
|
def _runwget(self, ud, d, command, quiet, workdir=None):
|
||
|
|
|
||
|
|
progresshandler = WgetProgressHandler(d)
|
||
|
|
|
||
|
|
logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
|
||
|
|
bb.fetch2.check_network_access(d, command, ud.url)
|
||
|
|
runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
|
||
|
|
|
||
|
|
def download(self, ud, d):
|
||
|
|
"""Fetch urls"""
|
||
|
|
|
||
|
|
fetchcmd = self.basecmd
|
||
|
|
|
||
|
|
if 'downloadfilename' in ud.parm:
|
||
|
|
dldir = d.getVar("DL_DIR")
|
||
|
|
bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
|
||
|
|
fetchcmd += " -O " + dldir + os.sep + ud.localfile
|
||
|
|
|
||
|
|
if ud.user and ud.pswd:
|
||
|
|
fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
|
||
|
|
|
||
|
|
uri = ud.url.split(";")[0]
|
||
|
|
if os.path.exists(ud.localpath):
|
||
|
|
# file exists, but we didnt complete it.. trying again..
|
||
|
|
fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
|
||
|
|
else:
|
||
|
|
fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
|
||
|
|
|
||
|
|
self._runwget(ud, d, fetchcmd, False)
|
||
|
|
|
||
|
|
# Sanity check since wget can pretend it succeed when it didn't
|
||
|
|
# Also, this used to happen if sourceforge sent us to the mirror page
|
||
|
|
if not os.path.exists(ud.localpath):
|
||
|
|
raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
|
||
|
|
|
||
|
|
if os.path.getsize(ud.localpath) == 0:
|
||
|
|
os.remove(ud.localpath)
|
||
|
|
raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def checkstatus(self, fetch, ud, d, try_again=True):
|
||
|
|
import urllib.request, urllib.error, urllib.parse, socket, http.client
|
||
|
|
from urllib.response import addinfourl
|
||
|
|
from bb.fetch2 import FetchConnectionCache
|
||
|
|
|
||
|
|
class HTTPConnectionCache(http.client.HTTPConnection):
|
||
|
|
if fetch.connection_cache:
|
||
|
|
def connect(self):
|
||
|
|
"""Connect to the host and port specified in __init__."""
|
||
|
|
|
||
|
|
sock = fetch.connection_cache.get_connection(self.host, self.port)
|
||
|
|
if sock:
|
||
|
|
self.sock = sock
|
||
|
|
else:
|
||
|
|
self.sock = socket.create_connection((self.host, self.port),
|
||
|
|
self.timeout, self.source_address)
|
||
|
|
fetch.connection_cache.add_connection(self.host, self.port, self.sock)
|
||
|
|
|
||
|
|
if self._tunnel_host:
|
||
|
|
self._tunnel()
|
||
|
|
|
||
|
|
class CacheHTTPHandler(urllib.request.HTTPHandler):
|
||
|
|
def http_open(self, req):
|
||
|
|
return self.do_open(HTTPConnectionCache, req)
|
||
|
|
|
||
|
|
def do_open(self, http_class, req):
|
||
|
|
"""Return an addinfourl object for the request, using http_class.
|
||
|
|
|
||
|
|
http_class must implement the HTTPConnection API from httplib.
|
||
|
|
The addinfourl return value is a file-like object. It also
|
||
|
|
has methods and attributes including:
|
||
|
|
- info(): return a mimetools.Message object for the headers
|
||
|
|
- geturl(): return the original request URL
|
||
|
|
- code: HTTP status code
|
||
|
|
"""
|
||
|
|
host = req.host
|
||
|
|
if not host:
|
||
|
|
raise urlllib2.URLError('no host given')
|
||
|
|
|
||
|
|
h = http_class(host, timeout=req.timeout) # will parse host:port
|
||
|
|
h.set_debuglevel(self._debuglevel)
|
||
|
|
|
||
|
|
headers = dict(req.unredirected_hdrs)
|
||
|
|
headers.update(dict((k, v) for k, v in list(req.headers.items())
|
||
|
|
if k not in headers))
|
||
|
|
|
||
|
|
# We want to make an HTTP/1.1 request, but the addinfourl
|
||
|
|
# class isn't prepared to deal with a persistent connection.
|
||
|
|
# It will try to read all remaining data from the socket,
|
||
|
|
# which will block while the server waits for the next request.
|
||
|
|
# So make sure the connection gets closed after the (only)
|
||
|
|
# request.
|
||
|
|
|
||
|
|
# Don't close connection when connection_cache is enabled,
|
||
|
|
if fetch.connection_cache is None:
|
||
|
|
headers["Connection"] = "close"
|
||
|
|
else:
|
||
|
|
headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
|
||
|
|
|
||
|
|
headers = dict(
|
||
|
|
(name.title(), val) for name, val in list(headers.items()))
|
||
|
|
|
||
|
|
if req._tunnel_host:
|
||
|
|
tunnel_headers = {}
|
||
|
|
proxy_auth_hdr = "Proxy-Authorization"
|
||
|
|
if proxy_auth_hdr in headers:
|
||
|
|
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
|
||
|
|
# Proxy-Authorization should not be sent to origin
|
||
|
|
# server.
|
||
|
|
del headers[proxy_auth_hdr]
|
||
|
|
h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
|
||
|
|
|
||
|
|
try:
|
||
|
|
h.request(req.get_method(), req.selector, req.data, headers)
|
||
|
|
except socket.error as err: # XXX what error?
|
||
|
|
# Don't close connection when cache is enabled.
|
||
|
|
# Instead, try to detect connections that are no longer
|
||
|
|
# usable (for example, closed unexpectedly) and remove
|
||
|
|
# them from the cache.
|
||
|
|
if fetch.connection_cache is None:
|
||
|
|
h.close()
|
||
|
|
elif isinstance(err, OSError) and err.errno == errno.EBADF:
|
||
|
|
# This happens when the server closes the connection despite the Keep-Alive.
|
||
|
|
# Apparently urllib then uses the file descriptor, expecting it to be
|
||
|
|
# connected, when in reality the connection is already gone.
|
||
|
|
# We let the request fail and expect it to be
|
||
|
|
# tried once more ("try_again" in check_status()),
|
||
|
|
# with the dead connection removed from the cache.
|
||
|
|
# If it still fails, we give up, which can happend for bad
|
||
|
|
# HTTP proxy settings.
|
||
|
|
fetch.connection_cache.remove_connection(h.host, h.port)
|
||
|
|
raise urllib.error.URLError(err)
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
r = h.getresponse(buffering=True)
|
||
|
|
except TypeError: # buffering kw not supported
|
||
|
|
r = h.getresponse()
|
||
|
|
|
||
|
|
# Pick apart the HTTPResponse object to get the addinfourl
|
||
|
|
# object initialized properly.
|
||
|
|
|
||
|
|
# Wrap the HTTPResponse object in socket's file object adapter
|
||
|
|
# for Windows. That adapter calls recv(), so delegate recv()
|
||
|
|
# to read(). This weird wrapping allows the returned object to
|
||
|
|
# have readline() and readlines() methods.
|
||
|
|
|
||
|
|
# XXX It might be better to extract the read buffering code
|
||
|
|
# out of socket._fileobject() and into a base class.
|
||
|
|
r.recv = r.read
|
||
|
|
|
||
|
|
# no data, just have to read
|
||
|
|
r.read()
|
||
|
|
class fp_dummy(object):
|
||
|
|
def read(self):
|
||
|
|
return ""
|
||
|
|
def readline(self):
|
||
|
|
return ""
|
||
|
|
def close(self):
|
||
|
|
pass
|
||
|
|
closed = False
|
||
|
|
|
||
|
|
resp = addinfourl(fp_dummy(), r.msg, req.get_full_url())
|
||
|
|
resp.code = r.status
|
||
|
|
resp.msg = r.reason
|
||
|
|
|
||
|
|
# Close connection when server request it.
|
||
|
|
if fetch.connection_cache is not None:
|
||
|
|
if 'Connection' in r.msg and r.msg['Connection'] == 'close':
|
||
|
|
fetch.connection_cache.remove_connection(h.host, h.port)
|
||
|
|
|
||
|
|
return resp
|
||
|
|
|
||
|
|
class HTTPMethodFallback(urllib.request.BaseHandler):
|
||
|
|
"""
|
||
|
|
Fallback to GET if HEAD is not allowed (405 HTTP error)
|
||
|
|
"""
|
||
|
|
def http_error_405(self, req, fp, code, msg, headers):
|
||
|
|
fp.read()
|
||
|
|
fp.close()
|
||
|
|
|
||
|
|
newheaders = dict((k,v) for k,v in list(req.headers.items())
|
||
|
|
if k.lower() not in ("content-length", "content-type"))
|
||
|
|
return self.parent.open(urllib.request.Request(req.get_full_url(),
|
||
|
|
headers=newheaders,
|
||
|
|
origin_req_host=req.origin_req_host,
|
||
|
|
unverifiable=True))
|
||
|
|
|
||
|
|
"""
|
||
|
|
Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
|
||
|
|
Forbidden when they actually mean 405 Method Not Allowed.
|
||
|
|
"""
|
||
|
|
http_error_403 = http_error_405
|
||
|
|
|
||
|
|
|
||
|
|
class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||
|
|
"""
|
||
|
|
urllib2.HTTPRedirectHandler resets the method to GET on redirect,
|
||
|
|
when we want to follow redirects using the original method.
|
||
|
|
"""
|
||
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||
|
|
newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
|
||
|
|
newreq.get_method = lambda: req.get_method()
|
||
|
|
return newreq
|
||
|
|
exported_proxies = export_proxies(d)
|
||
|
|
|
||
|
|
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
|
||
|
|
if export_proxies:
|
||
|
|
handlers.append(urllib.request.ProxyHandler())
|
||
|
|
handlers.append(CacheHTTPHandler())
|
||
|
|
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
|
||
|
|
# see PEP-0476, this causes verification errors on some https servers
|
||
|
|
# so disable by default.
|
||
|
|
import ssl
|
||
|
|
if hasattr(ssl, '_create_unverified_context'):
|
||
|
|
handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
|
||
|
|
opener = urllib.request.build_opener(*handlers)
|
||
|
|
|
||
|
|
try:
|
||
|
|
uri = ud.url.split(";")[0]
|
||
|
|
r = urllib.request.Request(uri)
|
||
|
|
r.get_method = lambda: "HEAD"
|
||
|
|
# Some servers (FusionForge, as used on Alioth) require that the
|
||
|
|
# optional Accept header is set.
|
||
|
|
r.add_header("Accept", "*/*")
|
||
|
|
def add_basic_auth(login_str, request):
|
||
|
|
'''Adds Basic auth to http request, pass in login:password as string'''
|
||
|
|
import base64
|
||
|
|
encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
|
||
|
|
authheader = "Basic %s" % encodeuser
|
||
|
|
r.add_header("Authorization", authheader)
|
||
|
|
|
||
|
|
if ud.user:
|
||
|
|
add_basic_auth(ud.user, r)
|
||
|
|
|
||
|
|
try:
|
||
|
|
import netrc, urllib.parse
|
||
|
|
n = netrc.netrc()
|
||
|
|
login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
|
||
|
|
add_basic_auth("%s:%s" % (login, password), r)
|
||
|
|
except (TypeError, ImportError, IOError, netrc.NetrcParseError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
with opener.open(r) as response:
|
||
|
|
pass
|
||
|
|
except urllib.error.URLError as e:
|
||
|
|
if try_again:
|
||
|
|
logger.debug(2, "checkstatus: trying again")
|
||
|
|
return self.checkstatus(fetch, ud, d, False)
|
||
|
|
else:
|
||
|
|
# debug for now to avoid spamming the logs in e.g. remote sstate searches
|
||
|
|
logger.debug(2, "checkstatus() urlopen failed: %s" % e)
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
def _parse_path(self, regex, s):
|
||
|
|
"""
|
||
|
|
Find and group name, version and archive type in the given string s
|
||
|
|
"""
|
||
|
|
|
||
|
|
m = regex.search(s)
|
||
|
|
if m:
|
||
|
|
pname = ''
|
||
|
|
pver = ''
|
||
|
|
ptype = ''
|
||
|
|
|
||
|
|
mdict = m.groupdict()
|
||
|
|
if 'name' in mdict.keys():
|
||
|
|
pname = mdict['name']
|
||
|
|
if 'pver' in mdict.keys():
|
||
|
|
pver = mdict['pver']
|
||
|
|
if 'type' in mdict.keys():
|
||
|
|
ptype = mdict['type']
|
||
|
|
|
||
|
|
bb.debug(3, "_parse_path: %s, %s, %s" % (pname, pver, ptype))
|
||
|
|
|
||
|
|
return (pname, pver, ptype)
|
||
|
|
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _modelate_version(self, version):
|
||
|
|
if version[0] in ['.', '-']:
|
||
|
|
if version[1].isdigit():
|
||
|
|
version = version[1] + version[0] + version[2:len(version)]
|
||
|
|
else:
|
||
|
|
version = version[1:len(version)]
|
||
|
|
|
||
|
|
version = re.sub('-', '.', version)
|
||
|
|
version = re.sub('_', '.', version)
|
||
|
|
version = re.sub('(rc)+', '.1000.', version)
|
||
|
|
version = re.sub('(beta)+', '.100.', version)
|
||
|
|
version = re.sub('(alpha)+', '.10.', version)
|
||
|
|
if version[0] == 'v':
|
||
|
|
version = version[1:len(version)]
|
||
|
|
return version
|
||
|
|
|
||
|
|
def _vercmp(self, old, new):
|
||
|
|
"""
|
||
|
|
Check whether 'new' is newer than 'old' version. We use existing vercmp() for the
|
||
|
|
purpose. PE is cleared in comparison as it's not for build, and PR is cleared too
|
||
|
|
for simplicity as it's somehow difficult to get from various upstream format
|
||
|
|
"""
|
||
|
|
|
||
|
|
(oldpn, oldpv, oldsuffix) = old
|
||
|
|
(newpn, newpv, newsuffix) = new
|
||
|
|
|
||
|
|
"""
|
||
|
|
Check for a new suffix type that we have never heard of before
|
||
|
|
"""
|
||
|
|
if (newsuffix):
|
||
|
|
m = self.suffix_regex_comp.search(newsuffix)
|
||
|
|
if not m:
|
||
|
|
bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix))
|
||
|
|
return False
|
||
|
|
|
||
|
|
"""
|
||
|
|
Not our package so ignore it
|
||
|
|
"""
|
||
|
|
if oldpn != newpn:
|
||
|
|
return False
|
||
|
|
|
||
|
|
oldpv = self._modelate_version(oldpv)
|
||
|
|
newpv = self._modelate_version(newpv)
|
||
|
|
|
||
|
|
return bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, ""))
|
||
|
|
|
||
|
|
def _fetch_index(self, uri, ud, d):
|
||
|
|
"""
|
||
|
|
Run fetch checkstatus to get directory information
|
||
|
|
"""
|
||
|
|
f = tempfile.NamedTemporaryFile()
|
||
|
|
with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
|
||
|
|
agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
|
||
|
|
fetchcmd = self.basecmd
|
||
|
|
fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
|
||
|
|
try:
|
||
|
|
self._runwget(ud, d, fetchcmd, True, workdir=workdir)
|
||
|
|
fetchresult = f.read()
|
||
|
|
except bb.fetch2.BBFetchException:
|
||
|
|
fetchresult = ""
|
||
|
|
|
||
|
|
return fetchresult
|
||
|
|
|
||
|
|
def _check_latest_version(self, url, package, package_regex, current_version, ud, d):
|
||
|
|
"""
|
||
|
|
Return the latest version of a package inside a given directory path
|
||
|
|
If error or no version, return ""
|
||
|
|
"""
|
||
|
|
valid = 0
|
||
|
|
version = ['', '', '']
|
||
|
|
|
||
|
|
bb.debug(3, "VersionURL: %s" % (url))
|
||
|
|
soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a"))
|
||
|
|
if not soup:
|
||
|
|
bb.debug(3, "*** %s NO SOUP" % (url))
|
||
|
|
return ""
|
||
|
|
|
||
|
|
for line in soup.find_all('a', href=True):
|
||
|
|
bb.debug(3, "line['href'] = '%s'" % (line['href']))
|
||
|
|
bb.debug(3, "line = '%s'" % (str(line)))
|
||
|
|
|
||
|
|
newver = self._parse_path(package_regex, line['href'])
|
||
|
|
if not newver:
|
||
|
|
newver = self._parse_path(package_regex, str(line))
|
||
|
|
|
||
|
|
if newver:
|
||
|
|
bb.debug(3, "Upstream version found: %s" % newver[1])
|
||
|
|
if valid == 0:
|
||
|
|
version = newver
|
||
|
|
valid = 1
|
||
|
|
elif self._vercmp(version, newver) < 0:
|
||
|
|
version = newver
|
||
|
|
|
||
|
|
pupver = re.sub('_', '.', version[1])
|
||
|
|
|
||
|
|
bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" %
|
||
|
|
(package, pupver or "N/A", current_version[1]))
|
||
|
|
|
||
|
|
if valid:
|
||
|
|
return pupver
|
||
|
|
|
||
|
|
return ""
|
||
|
|
|
||
|
|
def _check_latest_version_by_dir(self, dirver, package, package_regex,
|
||
|
|
current_version, ud, d):
|
||
|
|
"""
|
||
|
|
Scan every directory in order to get upstream version.
|
||
|
|
"""
|
||
|
|
version_dir = ['', '', '']
|
||
|
|
version = ['', '', '']
|
||
|
|
|
||
|
|
dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
|
||
|
|
s = dirver_regex.search(dirver)
|
||
|
|
if s:
|
||
|
|
version_dir[1] = s.group('ver')
|
||
|
|
else:
|
||
|
|
version_dir[1] = dirver
|
||
|
|
|
||
|
|
dirs_uri = bb.fetch.encodeurl([ud.type, ud.host,
|
||
|
|
ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
|
||
|
|
bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
|
||
|
|
|
||
|
|
soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a"))
|
||
|
|
if not soup:
|
||
|
|
return version[1]
|
||
|
|
|
||
|
|
for line in soup.find_all('a', href=True):
|
||
|
|
s = dirver_regex.search(line['href'].strip("/"))
|
||
|
|
if s:
|
||
|
|
sver = s.group('ver')
|
||
|
|
|
||
|
|
# When prefix is part of the version directory it need to
|
||
|
|
# ensure that only version directory is used so remove previous
|
||
|
|
# directories if exists.
|
||
|
|
#
|
||
|
|
# Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
|
||
|
|
# result is v2.5.
|
||
|
|
spfx = s.group('pfx').split('/')[-1]
|
||
|
|
|
||
|
|
version_dir_new = ['', sver, '']
|
||
|
|
if self._vercmp(version_dir, version_dir_new) <= 0:
|
||
|
|
dirver_new = spfx + sver
|
||
|
|
path = ud.path.replace(dirver, dirver_new, True) \
|
||
|
|
.split(package)[0]
|
||
|
|
uri = bb.fetch.encodeurl([ud.type, ud.host, path,
|
||
|
|
ud.user, ud.pswd, {}])
|
||
|
|
|
||
|
|
pupver = self._check_latest_version(uri,
|
||
|
|
package, package_regex, current_version, ud, d)
|
||
|
|
if pupver:
|
||
|
|
version[1] = pupver
|
||
|
|
|
||
|
|
version_dir = version_dir_new
|
||
|
|
|
||
|
|
return version[1]
|
||
|
|
|
||
|
|
def _init_regexes(self, package, ud, d):
|
||
|
|
"""
|
||
|
|
Match as many patterns as possible such as:
|
||
|
|
gnome-common-2.20.0.tar.gz (most common format)
|
||
|
|
gtk+-2.90.1.tar.gz
|
||
|
|
xf86-input-synaptics-12.6.9.tar.gz
|
||
|
|
dri2proto-2.3.tar.gz
|
||
|
|
blktool_4.orig.tar.gz
|
||
|
|
libid3tag-0.15.1b.tar.gz
|
||
|
|
unzip552.tar.gz
|
||
|
|
icu4c-3_6-src.tgz
|
||
|
|
genext2fs_1.3.orig.tar.gz
|
||
|
|
gst-fluendo-mp3
|
||
|
|
"""
|
||
|
|
# match most patterns which uses "-" as separator to version digits
|
||
|
|
pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]"
|
||
|
|
# a loose pattern such as for unzip552.tar.gz
|
||
|
|
pn_prefix2 = "[a-zA-Z]+"
|
||
|
|
# a loose pattern such as for 80325-quicky-0.4.tar.gz
|
||
|
|
pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+"
|
||
|
|
# Save the Package Name (pn) Regex for use later
|
||
|
|
pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
|
||
|
|
|
||
|
|
# match version
|
||
|
|
pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
|
||
|
|
|
||
|
|
# match arch
|
||
|
|
parch_regex = "-source|_all_"
|
||
|
|
|
||
|
|
# src.rpm extension was added only for rpm package. Can be removed if the rpm
|
||
|
|
# packaged will always be considered as having to be manually upgraded
|
||
|
|
psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
|
||
|
|
|
||
|
|
# match name, version and archive type of a package
|
||
|
|
package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
|
||
|
|
% (pn_regex, pver_regex, parch_regex, psuffix_regex))
|
||
|
|
self.suffix_regex_comp = re.compile(psuffix_regex)
|
||
|
|
|
||
|
|
# compile regex, can be specific by package or generic regex
|
||
|
|
pn_regex = d.getVar('UPSTREAM_CHECK_REGEX')
|
||
|
|
if pn_regex:
|
||
|
|
package_custom_regex_comp = re.compile(pn_regex)
|
||
|
|
else:
|
||
|
|
version = self._parse_path(package_regex_comp, package)
|
||
|
|
if version:
|
||
|
|
package_custom_regex_comp = re.compile(
|
||
|
|
"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" %
|
||
|
|
(re.escape(version[0]), pver_regex, parch_regex, psuffix_regex))
|
||
|
|
else:
|
||
|
|
package_custom_regex_comp = None
|
||
|
|
|
||
|
|
return package_custom_regex_comp
|
||
|
|
|
||
|
|
def latest_versionstring(self, ud, d):
|
||
|
|
"""
|
||
|
|
Manipulate the URL and try to obtain the latest package version
|
||
|
|
|
||
|
|
sanity check to ensure same name and type.
|
||
|
|
"""
|
||
|
|
package = ud.path.split("/")[-1]
|
||
|
|
current_version = ['', d.getVar('PV'), '']
|
||
|
|
|
||
|
|
"""possible to have no version in pkg name, such as spectrum-fw"""
|
||
|
|
if not re.search("\d+", package):
|
||
|
|
current_version[1] = re.sub('_', '.', current_version[1])
|
||
|
|
current_version[1] = re.sub('-', '.', current_version[1])
|
||
|
|
return (current_version[1], '')
|
||
|
|
|
||
|
|
package_regex = self._init_regexes(package, ud, d)
|
||
|
|
if package_regex is None:
|
||
|
|
bb.warn("latest_versionstring: package %s don't match pattern" % (package))
|
||
|
|
return ('', '')
|
||
|
|
bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
|
||
|
|
|
||
|
|
uri = ""
|
||
|
|
regex_uri = d.getVar("UPSTREAM_CHECK_URI")
|
||
|
|
if not regex_uri:
|
||
|
|
path = ud.path.split(package)[0]
|
||
|
|
|
||
|
|
# search for version matches on folders inside the path, like:
|
||
|
|
# "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
|
||
|
|
dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
|
||
|
|
m = dirver_regex.search(path)
|
||
|
|
if m:
|
||
|
|
pn = d.getVar('PN')
|
||
|
|
dirver = m.group('dirver')
|
||
|
|
|
||
|
|
dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
|
||
|
|
if not dirver_pn_regex.search(dirver):
|
||
|
|
return (self._check_latest_version_by_dir(dirver,
|
||
|
|
package, package_regex, current_version, ud, d), '')
|
||
|
|
|
||
|
|
uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
|
||
|
|
else:
|
||
|
|
uri = regex_uri
|
||
|
|
|
||
|
|
return (self._check_latest_version(uri, package, package_regex,
|
||
|
|
current_version, ud, d), '')
|