rpi/domo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bump buildroot to 2019.02

Browse Source
This commit is contained in:
NADAL Jean-Baptiste
2019-03-28 22:49:48 +01:00
parent 5598b1b762
commit 920d307141
5121 changed files with 78550 additions and 46132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
bsp/buildroot/utils/scanpypi
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
"""
Utility for building Buildroot packages for existing PyPI packages
@@ -7,13 +7,12 @@ Any package built by scanpypi should be manually checked for
errors.
"""
from __future__ import print_function
from __future__ import absolute_import
import argparse
import json
import urllib2
import sys
import os
import shutil
import StringIO
import tarfile
import zipfile
import errno
@@ -23,6 +22,16 @@ import textwrap
import tempfile
import imp
from functools import wraps
import six.moves.urllib.request
import six.moves.urllib.error
import six.moves.urllib.parse
from six.moves import map
from six.moves import zip
from six.moves import input
if six.PY2:
import StringIO
else:
import io
BUF_SIZE = 65536
@@ -88,6 +97,7 @@ def pkg_buildroot_name(pkg_name):
pkg_name -- String to rename
"""
name = re.sub('[^\w-]', '', pkg_name.lower())
name = name.replace('_', '-')
prefix = 'python-'
pattern = re.compile('^(?!' + prefix + ')(.+?)$')
name = pattern.sub(r'python-\1', name)
@@ -147,15 +157,15 @@ class BuildrootPackage():
self.metadata_url = 'https://pypi.org/pypi/{pkg}/json'.format(
pkg=self.real_name)
try:
pkg_json = urllib2.urlopen(self.metadata_url).read().decode()
except urllib2.HTTPError as error:
pkg_json = six.moves.urllib.request.urlopen(self.metadata_url).read().decode()
except six.moves.urllib.error.HTTPError as error:
print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
print('ERROR: Could not find package {pkg}.\n'
'Check syntax inside the python package index:\n'
'https://pypi.python.org/pypi/ '
.format(pkg=self.real_name))
raise
except urllib2.URLError:
except six.moves.urllib.error.URLError:
print('ERROR: Could not find package {pkg}.\n'
'Check syntax inside the python package index:\n'
'https://pypi.python.org/pypi/ '
@@ -169,6 +179,7 @@ class BuildrootPackage():
"""
Download a package using metadata from pypi
"""
download = None
try:
self.metadata['urls'][0]['filename']
except IndexError:
@@ -181,7 +192,7 @@ class BuildrootPackage():
'digests': None}]
# In this case, we can't get the name of the downloaded file
# from the pypi api, so we need to find it, this should work
urlpath = urllib2.urlparse.urlparse(
urlpath = six.moves.urllib.parse.urlparse(
self.metadata['info']['download_url']).path
# urlparse().path give something like
# /path/to/file-version.tar.gz
@@ -192,9 +203,9 @@ class BuildrootPackage():
continue
try:
print('Downloading package {pkg} from {url}...'.format(
pkg=self.real_name, url=download_url['url']))
download = urllib2.urlopen(download_url['url'])
except urllib2.HTTPError as http_error:
pkg=self.real_name, url=download_url['url']))
download = six.moves.urllib.request.urlopen(download_url['url'])
except six.moves.urllib.error.HTTPError as http_error:
download = http_error
else:
self.used_url = download_url
@@ -204,14 +215,33 @@ class BuildrootPackage():
self.md5_sum = hashlib.md5(self.as_string).hexdigest()
if self.md5_sum == download_url['digests']['md5']:
break
else:
if download.__class__ == urllib2.HTTPError:
raise download
raise DownloadFailed('Failed to download package {pkg}'
if download is None:
raise DownloadFailed('Failed to download package {pkg}: '
'No source archive available'
.format(pkg=self.real_name))
elif download.__class__ == six.moves.urllib.error.HTTPError:
raise download
self.filename = self.used_url['filename']
self.url = self.used_url['url']
def check_archive(self, members):
"""
Check archive content before extracting
Keyword arguments:
members -- list of archive members
"""
# Protect against https://github.com/snyk/zip-slip-vulnerability
# Older python versions do not validate that the extracted files are
# inside the target directory. Detect and error out on evil paths
evil = [e for e in members if os.path.relpath(e).startswith(('/', '..'))]
if evil:
print('ERROR: Refusing to extract {} with suspicious members {}'.format(
self.filename, evil))
sys.exit(1)
def extract_package(self, tmp_path):
"""
Extract the package contents into a directrory
@@ -219,7 +249,10 @@ class BuildrootPackage():
Keyword arguments:
tmp_path -- directory where you want the package to be extracted
"""
as_file = StringIO.StringIO(self.as_string)
if six.PY2:
as_file = StringIO.StringIO(self.as_string)
else:
as_file = io.BytesIO(self.as_string)
if self.filename[-3:] == 'zip':
with zipfile.ZipFile(as_file) as as_zipfile:
tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
@@ -233,6 +266,7 @@ class BuildrootPackage():
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
self.check_archive(as_zipfile.namelist())
as_zipfile.extractall(tmp_pkg)
pkg_filename = self.filename.split(".zip")[0]
else:
@@ -248,6 +282,7 @@ class BuildrootPackage():
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
self.check_archive(as_tarfile.getnames())
as_tarfile.extractall(tmp_pkg)
pkg_filename = self.filename.split(".tar")[0]
@@ -265,6 +300,12 @@ class BuildrootPackage():
sys.path.append(self.tmp_extract)
s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
setup = imp.load_module('setup', s_file, s_path, s_desc)
if self.metadata_name in self.setup_args:
pass
elif self.metadata_name.replace('_', '-') in self.setup_args:
self.metadata_name = self.metadata_name.replace('_', '-')
elif self.metadata_name.replace('-', '_') in self.setup_args:
self.metadata_name = self.metadata_name.replace('-', '_')
try:
self.setup_metadata = self.setup_args[self.metadata_name]
except KeyError:
@@ -304,8 +345,8 @@ class BuildrootPackage():
if len(item) > 0 and item[0] != '#']
req_not_found = self.pkg_req
self.pkg_req = map(pkg_buildroot_name, self.pkg_req)
pkg_tuples = zip(req_not_found, self.pkg_req)
self.pkg_req = list(map(pkg_buildroot_name, self.pkg_req))
pkg_tuples = list(zip(req_not_found, self.pkg_req))
# pkg_tuples is a list of tuples that looks like
# ('werkzeug','python-werkzeug') because I need both when checking if
# dependencies already exist or are already in the download list
@@ -338,13 +379,14 @@ class BuildrootPackage():
version=self.version)
lines.append(version_line)
targz = self.filename.replace(
self.version,
'$({name}_VERSION)'.format(name=self.mk_name))
targz_line = '{name}_SOURCE = {filename}\n'.format(
name=self.mk_name,
filename=targz)
lines.append(targz_line)
if self.buildroot_name != self.real_name:
targz = self.filename.replace(
self.version,
'$({name}_VERSION)'.format(name=self.mk_name))
targz_line = '{name}_SOURCE = {filename}\n'.format(
name=self.mk_name,
filename=targz)
lines.append(targz_line)
if self.filename not in self.url:
# Sometimes the filename is in the url, sometimes it's not
@@ -413,8 +455,7 @@ class BuildrootPackage():
classifiers_licenses = [regexp.sub(r"\1", lic)
for lic in self.metadata['info']['classifiers']
if regexp.match(lic)]
licenses = map(lambda x: license_dict[x] if x in license_dict else x,
classifiers_licenses)
licenses = [license_dict[x] if x in license_dict else x for x in classifiers_licenses]
if not len(licenses):
print('WARNING: License has been set to "{license}". It is most'
' likely wrong, please change it if need be'.format(
@@ -428,8 +469,10 @@ class BuildrootPackage():
for license_file in license_files:
with open(license_file) as lic_file:
match = liclookup.match(lic_file.read())
if match.confidence >= 90.0:
if match is not None and match.confidence >= 90.0:
license_names.append(match.license.id)
else:
license_names.append("FIXME: license id couldn't be detected")
if len(license_names) > 0:
license_line = ('{name}_LICENSE ='
@@ -547,7 +590,7 @@ class BuildrootPackage():
hash_line = '{method}\t{digest} {filename}\n'.format(
method='sha256',
digest=sha256.hexdigest(),
filename=os.path.basename(license_file))
filename=license_file.replace(self.tmp_extract, '')[1:])
lines.append(hash_line)
with open(path_to_hash, 'w') as hash_file:
@@ -574,7 +617,7 @@ class BuildrootPackage():
lines.append('\thelp\n')
help_lines = textwrap.wrap(self.metadata['info']['summary'],
help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
initial_indent='\t ',
subsequent_indent='\t ')
@@ -585,7 +628,7 @@ class BuildrootPackage():
# \t + two spaces is 3 char long
help_lines.append('')
help_lines.append('\t ' + self.metadata['info']['home_page'])
help_lines = map(lambda x: x + '\n', help_lines)
help_lines = [x + '\n' for x in help_lines]
lines += help_lines
with open(path_to_config, 'w') as config_file:
@@ -626,7 +669,7 @@ def main():
print('Fetching package', package.real_name)
try:
package.fetch_package_info()
except (urllib2.URLError, urllib2.HTTPError):
except (six.moves.urllib.error.URLError, six.moves.urllib.error.HTTPError):
continue
if package.metadata_name.lower() == 'setuptools':
# setuptools imports itself, that does not work very well
@@ -636,7 +679,7 @@ def main():
try:
package.download_package()
except urllib2.HTTPError as error:
except six.moves.urllib.error.HTTPError as error:
print('Error: {code} {reason}'.format(code=error.code,
reason=error.reason))
print('Error downloading package :', package.buildroot_name)
@@ -684,7 +727,7 @@ def main():
continue
print('Error: Package {name} already exists'
.format(name=package.pkg_dir))
del_pkg = raw_input(
del_pkg = input(
'Do you want to delete existing package ? [y/N]')
if del_pkg.lower() == 'y':
shutil.rmtree(package.pkg_dir)