330 lines
14 KiB
Python
330 lines
14 KiB
Python
|
# Recipe creation tool - node.js NPM module support plugin
|
||
|
|
#
|
||
|
|
# Copyright (C) 2016 Intel Corporation
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License version 2 as
|
||
|
|
# published by the Free Software Foundation.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License along
|
||
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
|
|
||
|
|
import os
|
||
|
|
import logging
|
||
|
|
import subprocess
|
||
|
|
import tempfile
|
||
|
|
import shutil
|
||
|
|
import json
|
||
|
|
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
|
||
|
|
|
||
|
|
logger = logging.getLogger('recipetool')
|
||
|
|
|
||
|
|
|
||
|
|
tinfoil = None
|
||
|
|
|
||
|
|
def tinfoil_init(instance):
|
||
|
|
global tinfoil
|
||
|
|
tinfoil = instance
|
||
|
|
|
||
|
|
|
||
|
|
class NpmRecipeHandler(RecipeHandler):
|
||
|
|
lockdownpath = None
|
||
|
|
|
||
|
|
def _ensure_npm(self, fixed_setup=False):
|
||
|
|
if not tinfoil.recipes_parsed:
|
||
|
|
tinfoil.parse_recipes()
|
||
|
|
try:
|
||
|
|
rd = tinfoil.parse_recipe('nodejs-native')
|
||
|
|
except bb.providers.NoProvider:
|
||
|
|
if fixed_setup:
|
||
|
|
msg = 'nodejs-native is required for npm but is not available within this SDK'
|
||
|
|
else:
|
||
|
|
msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
|
||
|
|
logger.error(msg)
|
||
|
|
return None
|
||
|
|
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
|
||
|
|
npmpath = os.path.join(bindir, 'npm')
|
||
|
|
if not os.path.exists(npmpath):
|
||
|
|
tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
|
||
|
|
if not os.path.exists(npmpath):
|
||
|
|
logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
|
||
|
|
return None
|
||
|
|
return bindir
|
||
|
|
|
||
|
|
def _handle_license(self, data):
|
||
|
|
'''
|
||
|
|
Handle the license value from an npm package.json file
|
||
|
|
'''
|
||
|
|
license = None
|
||
|
|
if 'license' in data:
|
||
|
|
license = data['license']
|
||
|
|
if isinstance(license, dict):
|
||
|
|
license = license.get('type', None)
|
||
|
|
if license:
|
||
|
|
if 'OR' in license:
|
||
|
|
license = license.replace('OR', '|')
|
||
|
|
license = license.replace('AND', '&')
|
||
|
|
license = license.replace(' ', '_')
|
||
|
|
if not license[0] == '(':
|
||
|
|
license = '(' + license + ')'
|
||
|
|
else:
|
||
|
|
license = license.replace('AND', '&')
|
||
|
|
if license[0] == '(':
|
||
|
|
license = license[1:]
|
||
|
|
if license[-1] == ')':
|
||
|
|
license = license[:-1]
|
||
|
|
license = license.replace('MIT/X11', 'MIT')
|
||
|
|
license = license.replace('Public Domain', 'PD')
|
||
|
|
license = license.replace('SEE LICENSE IN EULA',
|
||
|
|
'SEE-LICENSE-IN-EULA')
|
||
|
|
return license
|
||
|
|
|
||
|
|
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
|
||
|
|
try:
|
||
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||
|
|
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||
|
|
except bb.process.ExecutionError as e:
|
||
|
|
logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
|
||
|
|
return
|
||
|
|
|
||
|
|
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
|
||
|
|
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
|
||
|
|
extravalues.setdefault('extrafiles', {})
|
||
|
|
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
|
||
|
|
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
|
||
|
|
|
||
|
|
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
|
||
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||
|
|
if not NpmRecipeHandler.lockdownpath:
|
||
|
|
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
|
||
|
|
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
|
||
|
|
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||
|
|
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
|
||
|
|
if not os.path.exists(relockbin):
|
||
|
|
logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||
|
|
except bb.process.ExecutionError as e:
|
||
|
|
logger.warn('lockdown-relock failed:\n%s' % e.stdout)
|
||
|
|
return
|
||
|
|
|
||
|
|
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
|
||
|
|
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
|
||
|
|
extravalues.setdefault('extrafiles', {})
|
||
|
|
extravalues['extrafiles']['lockdown.json'] = tmpfile
|
||
|
|
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
|
||
|
|
|
||
|
|
def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
|
||
|
|
import scriptutils
|
||
|
|
# If this isn't a single module we need to get the dependencies
|
||
|
|
# and add them to SRC_URI
|
||
|
|
def varfunc(varname, origvalue, op, newlines):
|
||
|
|
if varname == 'SRC_URI':
|
||
|
|
if not origvalue.startswith('npm://'):
|
||
|
|
src_uri = origvalue.split()
|
||
|
|
deplist = {}
|
||
|
|
for dep, depver in optdeps.items():
|
||
|
|
depdata = self.get_npm_data(dep, depver, d)
|
||
|
|
if self.check_npm_optional_dependency(depdata):
|
||
|
|
deplist[dep] = depdata
|
||
|
|
for dep, depver in devdeps.items():
|
||
|
|
depdata = self.get_npm_data(dep, depver, d)
|
||
|
|
if self.check_npm_optional_dependency(depdata):
|
||
|
|
deplist[dep] = depdata
|
||
|
|
for dep, depver in deps.items():
|
||
|
|
depdata = self.get_npm_data(dep, depver, d)
|
||
|
|
deplist[dep] = depdata
|
||
|
|
|
||
|
|
extra_urls = []
|
||
|
|
for dep, depdata in deplist.items():
|
||
|
|
version = depdata.get('version', None)
|
||
|
|
if version:
|
||
|
|
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
|
||
|
|
extra_urls.append(url)
|
||
|
|
if extra_urls:
|
||
|
|
scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
|
||
|
|
src_uri.extend(extra_urls)
|
||
|
|
return src_uri, None, -1, True
|
||
|
|
return origvalue, None, 0, True
|
||
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
|
||
|
|
if updated:
|
||
|
|
del lines_before[:]
|
||
|
|
for line in newlines:
|
||
|
|
# Hack to avoid newlines that edit_metadata inserts
|
||
|
|
if line.endswith('\n'):
|
||
|
|
line = line[:-1]
|
||
|
|
lines_before.append(line)
|
||
|
|
return updated
|
||
|
|
|
||
|
|
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
||
|
|
import bb.utils
|
||
|
|
import oe.package
|
||
|
|
from collections import OrderedDict
|
||
|
|
|
||
|
|
if 'buildsystem' in handled:
|
||
|
|
return False
|
||
|
|
|
||
|
|
def read_package_json(fn):
|
||
|
|
with open(fn, 'r', errors='surrogateescape') as f:
|
||
|
|
return json.loads(f.read())
|
||
|
|
|
||
|
|
files = RecipeHandler.checkfiles(srctree, ['package.json'])
|
||
|
|
if files:
|
||
|
|
d = bb.data.createCopy(tinfoil.config_data)
|
||
|
|
npm_bindir = self._ensure_npm()
|
||
|
|
if not npm_bindir:
|
||
|
|
sys.exit(14)
|
||
|
|
d.prependVar('PATH', '%s:' % npm_bindir)
|
||
|
|
|
||
|
|
data = read_package_json(files[0])
|
||
|
|
if 'name' in data and 'version' in data:
|
||
|
|
extravalues['PN'] = data['name']
|
||
|
|
extravalues['PV'] = data['version']
|
||
|
|
classes.append('npm')
|
||
|
|
handled.append('buildsystem')
|
||
|
|
if 'description' in data:
|
||
|
|
extravalues['SUMMARY'] = data['description']
|
||
|
|
if 'homepage' in data:
|
||
|
|
extravalues['HOMEPAGE'] = data['homepage']
|
||
|
|
|
||
|
|
fetchdev = extravalues['fetchdev'] or None
|
||
|
|
deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
|
||
|
|
self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
|
||
|
|
|
||
|
|
# Shrinkwrap
|
||
|
|
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
|
||
|
|
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
|
||
|
|
|
||
|
|
# Lockdown
|
||
|
|
self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
|
||
|
|
|
||
|
|
# Split each npm module out to is own package
|
||
|
|
npmpackages = oe.package.npm_split_package_dirs(srctree)
|
||
|
|
licvalues = None
|
||
|
|
for item in handled:
|
||
|
|
if isinstance(item, tuple):
|
||
|
|
if item[0] == 'license':
|
||
|
|
licvalues = item[1]
|
||
|
|
break
|
||
|
|
if not licvalues:
|
||
|
|
licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
|
||
|
|
if licvalues:
|
||
|
|
# Augment the license list with information we have in the packages
|
||
|
|
licenses = {}
|
||
|
|
license = self._handle_license(data)
|
||
|
|
if license:
|
||
|
|
licenses['${PN}'] = license
|
||
|
|
for pkgname, pkgitem in npmpackages.items():
|
||
|
|
_, pdata = pkgitem
|
||
|
|
license = self._handle_license(pdata)
|
||
|
|
if license:
|
||
|
|
licenses[pkgname] = license
|
||
|
|
# Now write out the package-specific license values
|
||
|
|
# We need to strip out the json data dicts for this since split_pkg_licenses
|
||
|
|
# isn't expecting it
|
||
|
|
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
|
||
|
|
packages['${PN}'] = ''
|
||
|
|
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
|
||
|
|
all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
|
||
|
|
if '&' in all_licenses:
|
||
|
|
all_licenses.remove('&')
|
||
|
|
extravalues['LICENSE'] = ' & '.join(all_licenses)
|
||
|
|
|
||
|
|
# Need to move S setting after inherit npm
|
||
|
|
for i, line in enumerate(lines_before):
|
||
|
|
if line.startswith('S ='):
|
||
|
|
lines_before.pop(i)
|
||
|
|
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
|
||
|
|
lines_after.insert(1, line)
|
||
|
|
break
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
return False
|
||
|
|
|
||
|
|
# FIXME this is duplicated from lib/bb/fetch2/npm.py
|
||
|
|
def _parse_view(self, output):
|
||
|
|
'''
|
||
|
|
Parse the output of npm view --json; the last JSON result
|
||
|
|
is assumed to be the one that we're interested in.
|
||
|
|
'''
|
||
|
|
pdata = None
|
||
|
|
outdeps = {}
|
||
|
|
datalines = []
|
||
|
|
bracelevel = 0
|
||
|
|
for line in output.splitlines():
|
||
|
|
if bracelevel:
|
||
|
|
datalines.append(line)
|
||
|
|
elif '{' in line:
|
||
|
|
datalines = []
|
||
|
|
datalines.append(line)
|
||
|
|
bracelevel = bracelevel + line.count('{') - line.count('}')
|
||
|
|
if datalines:
|
||
|
|
pdata = json.loads('\n'.join(datalines))
|
||
|
|
return pdata
|
||
|
|
|
||
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||
|
|
# (split out from _getdependencies())
|
||
|
|
def get_npm_data(self, pkg, version, d):
|
||
|
|
import bb.fetch2
|
||
|
|
pkgfullname = pkg
|
||
|
|
if version != '*' and not '/' in version:
|
||
|
|
pkgfullname += "@'%s'" % version
|
||
|
|
logger.debug(2, "Calling getdeps on %s" % pkg)
|
||
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||
|
|
fetchcmd = "npm view %s --json" % pkgfullname
|
||
|
|
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||
|
|
data = self._parse_view(output)
|
||
|
|
return data
|
||
|
|
|
||
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||
|
|
# (split out from _getdependencies())
|
||
|
|
def get_npm_package_dependencies(self, pdata, fetchdev):
|
||
|
|
dependencies = pdata.get('dependencies', {})
|
||
|
|
optionalDependencies = pdata.get('optionalDependencies', {})
|
||
|
|
dependencies.update(optionalDependencies)
|
||
|
|
if fetchdev:
|
||
|
|
devDependencies = pdata.get('devDependencies', {})
|
||
|
|
dependencies.update(devDependencies)
|
||
|
|
else:
|
||
|
|
devDependencies = {}
|
||
|
|
depsfound = {}
|
||
|
|
optdepsfound = {}
|
||
|
|
devdepsfound = {}
|
||
|
|
for dep in dependencies:
|
||
|
|
if dep in optionalDependencies:
|
||
|
|
optdepsfound[dep] = dependencies[dep]
|
||
|
|
elif dep in devDependencies:
|
||
|
|
devdepsfound[dep] = dependencies[dep]
|
||
|
|
else:
|
||
|
|
depsfound[dep] = dependencies[dep]
|
||
|
|
return depsfound, optdepsfound, devdepsfound
|
||
|
|
|
||
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||
|
|
# (split out from _getdependencies())
|
||
|
|
def check_npm_optional_dependency(self, pdata):
|
||
|
|
pkg_os = pdata.get('os', None)
|
||
|
|
if pkg_os:
|
||
|
|
if not isinstance(pkg_os, list):
|
||
|
|
pkg_os = [pkg_os]
|
||
|
|
blacklist = False
|
||
|
|
for item in pkg_os:
|
||
|
|
if item.startswith('!'):
|
||
|
|
blacklist = True
|
||
|
|
break
|
||
|
|
if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
|
||
|
|
logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
def register_recipe_handlers(handlers):
|
||
|
|
handlers.append((NpmRecipeHandler(), 60))
|