forked from brl/citadel
901 lines
39 KiB
Python
901 lines
39 KiB
Python
|
# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
|
||
|
|
#
|
||
|
|
# Copyright (C) 2012-2017 Intel Corporation
|
||
|
|
# Copyright (C) 2011 Mentor Graphics Corporation
|
||
|
|
# Copyright (C) 2006-2012 Richard Purdie
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License version 2 as
|
||
|
|
# published by the Free Software Foundation.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License along
|
||
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import atexit
|
||
|
|
import re
|
||
|
|
from collections import OrderedDict, defaultdict
|
||
|
|
|
||
|
|
import bb.cache
|
||
|
|
import bb.cooker
|
||
|
|
import bb.providers
|
||
|
|
import bb.taskdata
|
||
|
|
import bb.utils
|
||
|
|
import bb.command
|
||
|
|
import bb.remotedata
|
||
|
|
from bb.cookerdata import CookerConfiguration, ConfigParameters
|
||
|
|
from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
|
||
|
|
import bb.fetch2
|
||
|
|
|
||
|
|
|
||
|
|
# We need this in order to shut down the connection to the bitbake server,
|
||
|
|
# otherwise the process will never properly exit
|
||
|
|
_server_connections = []
|
||
|
|
def _terminate_connections():
|
||
|
|
for connection in _server_connections:
|
||
|
|
connection.terminate()
|
||
|
|
atexit.register(_terminate_connections)
|
||
|
|
|
||
|
|
class TinfoilUIException(Exception):
|
||
|
|
"""Exception raised when the UI returns non-zero from its main function"""
|
||
|
|
def __init__(self, returncode):
|
||
|
|
self.returncode = returncode
|
||
|
|
def __repr__(self):
|
||
|
|
return 'UI module main returned %d' % self.returncode
|
||
|
|
|
||
|
|
class TinfoilCommandFailed(Exception):
|
||
|
|
"""Exception raised when run_command fails"""
|
||
|
|
|
||
|
|
class TinfoilDataStoreConnector:
|
||
|
|
"""Connector object used to enable access to datastore objects via tinfoil"""
|
||
|
|
|
||
|
|
def __init__(self, tinfoil, dsindex):
|
||
|
|
self.tinfoil = tinfoil
|
||
|
|
self.dsindex = dsindex
|
||
|
|
def getVar(self, name):
|
||
|
|
value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
|
||
|
|
overrides = None
|
||
|
|
if isinstance(value, dict):
|
||
|
|
if '_connector_origtype' in value:
|
||
|
|
value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
|
||
|
|
del value['_connector_origtype']
|
||
|
|
if '_connector_overrides' in value:
|
||
|
|
overrides = value['_connector_overrides']
|
||
|
|
del value['_connector_overrides']
|
||
|
|
return value, overrides
|
||
|
|
def getKeys(self):
|
||
|
|
return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
|
||
|
|
def getVarHistory(self, name):
|
||
|
|
return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
|
||
|
|
def expandPythonRef(self, varname, expr, d):
|
||
|
|
ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
|
||
|
|
ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
|
||
|
|
return ret
|
||
|
|
def setVar(self, varname, value):
|
||
|
|
if self.dsindex is None:
|
||
|
|
self.tinfoil.run_command('setVariable', varname, value)
|
||
|
|
else:
|
||
|
|
# Not currently implemented - indicate that setting should
|
||
|
|
# be redirected to local side
|
||
|
|
return True
|
||
|
|
def setVarFlag(self, varname, flagname, value):
|
||
|
|
if self.dsindex is None:
|
||
|
|
self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
|
||
|
|
else:
|
||
|
|
# Not currently implemented - indicate that setting should
|
||
|
|
# be redirected to local side
|
||
|
|
return True
|
||
|
|
def delVar(self, varname):
|
||
|
|
if self.dsindex is None:
|
||
|
|
self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
|
||
|
|
else:
|
||
|
|
# Not currently implemented - indicate that setting should
|
||
|
|
# be redirected to local side
|
||
|
|
return True
|
||
|
|
def delVarFlag(self, varname, flagname):
|
||
|
|
if self.dsindex is None:
|
||
|
|
self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
|
||
|
|
else:
|
||
|
|
# Not currently implemented - indicate that setting should
|
||
|
|
# be redirected to local side
|
||
|
|
return True
|
||
|
|
def renameVar(self, name, newname):
|
||
|
|
if self.dsindex is None:
|
||
|
|
self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
|
||
|
|
else:
|
||
|
|
# Not currently implemented - indicate that setting should
|
||
|
|
# be redirected to local side
|
||
|
|
return True
|
||
|
|
|
||
|
|
class TinfoilCookerAdapter:
|
||
|
|
"""
|
||
|
|
Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
|
||
|
|
since now Tinfoil is on the client side it no longer has direct access.
|
||
|
|
"""
|
||
|
|
|
||
|
|
class TinfoilCookerCollectionAdapter:
|
||
|
|
""" cooker.collection adapter """
|
||
|
|
def __init__(self, tinfoil):
|
||
|
|
self.tinfoil = tinfoil
|
||
|
|
def get_file_appends(self, fn):
|
||
|
|
return self.tinfoil.get_file_appends(fn)
|
||
|
|
def __getattr__(self, name):
|
||
|
|
if name == 'overlayed':
|
||
|
|
return self.tinfoil.get_overlayed_recipes()
|
||
|
|
elif name == 'bbappends':
|
||
|
|
return self.tinfoil.run_command('getAllAppends')
|
||
|
|
else:
|
||
|
|
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||
|
|
|
||
|
|
class TinfoilRecipeCacheAdapter:
|
||
|
|
""" cooker.recipecache adapter """
|
||
|
|
def __init__(self, tinfoil):
|
||
|
|
self.tinfoil = tinfoil
|
||
|
|
self._cache = {}
|
||
|
|
|
||
|
|
def get_pkg_pn_fn(self):
|
||
|
|
pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
|
||
|
|
pkg_fn = {}
|
||
|
|
for pn, fnlist in pkg_pn.items():
|
||
|
|
for fn in fnlist:
|
||
|
|
pkg_fn[fn] = pn
|
||
|
|
self._cache['pkg_pn'] = pkg_pn
|
||
|
|
self._cache['pkg_fn'] = pkg_fn
|
||
|
|
|
||
|
|
def __getattr__(self, name):
|
||
|
|
# Grab these only when they are requested since they aren't always used
|
||
|
|
if name in self._cache:
|
||
|
|
return self._cache[name]
|
||
|
|
elif name == 'pkg_pn':
|
||
|
|
self.get_pkg_pn_fn()
|
||
|
|
return self._cache[name]
|
||
|
|
elif name == 'pkg_fn':
|
||
|
|
self.get_pkg_pn_fn()
|
||
|
|
return self._cache[name]
|
||
|
|
elif name == 'deps':
|
||
|
|
attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
|
||
|
|
elif name == 'rundeps':
|
||
|
|
attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
|
||
|
|
elif name == 'runrecs':
|
||
|
|
attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
|
||
|
|
elif name == 'pkg_pepvpr':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
|
||
|
|
elif name == 'inherits':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
|
||
|
|
elif name == 'bbfile_priority':
|
||
|
|
attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
|
||
|
|
elif name == 'pkg_dp':
|
||
|
|
attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
|
||
|
|
elif name == 'fn_provides':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRecipeProvides') or {}
|
||
|
|
elif name == 'packages':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRecipePackages') or {}
|
||
|
|
elif name == 'packages_dynamic':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic') or {}
|
||
|
|
elif name == 'rproviders':
|
||
|
|
attrvalue = self.tinfoil.run_command('getRProviders') or {}
|
||
|
|
else:
|
||
|
|
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||
|
|
|
||
|
|
self._cache[name] = attrvalue
|
||
|
|
return attrvalue
|
||
|
|
|
||
|
|
def __init__(self, tinfoil):
|
||
|
|
self.tinfoil = tinfoil
|
||
|
|
self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
|
||
|
|
self.recipecaches = {}
|
||
|
|
# FIXME all machines
|
||
|
|
self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
|
||
|
|
self._cache = {}
|
||
|
|
def __getattr__(self, name):
|
||
|
|
# Grab these only when they are requested since they aren't always used
|
||
|
|
if name in self._cache:
|
||
|
|
return self._cache[name]
|
||
|
|
elif name == 'skiplist':
|
||
|
|
attrvalue = self.tinfoil.get_skipped_recipes()
|
||
|
|
elif name == 'bbfile_config_priorities':
|
||
|
|
ret = self.tinfoil.run_command('getLayerPriorities')
|
||
|
|
bbfile_config_priorities = []
|
||
|
|
for collection, pattern, regex, pri in ret:
|
||
|
|
bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
|
||
|
|
|
||
|
|
attrvalue = bbfile_config_priorities
|
||
|
|
else:
|
||
|
|
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||
|
|
|
||
|
|
self._cache[name] = attrvalue
|
||
|
|
return attrvalue
|
||
|
|
|
||
|
|
def findBestProvider(self, pn):
|
||
|
|
return self.tinfoil.find_best_provider(pn)
|
||
|
|
|
||
|
|
|
||
|
|
class TinfoilRecipeInfo:
|
||
|
|
"""
|
||
|
|
Provides a convenient representation of the cached information for a single recipe.
|
||
|
|
Some attributes are set on construction, others are read on-demand (which internally
|
||
|
|
may result in a remote procedure call to the bitbake server the first time).
|
||
|
|
Note that only information which is cached is available through this object - if
|
||
|
|
you need other variable values you will need to parse the recipe using
|
||
|
|
Tinfoil.parse_recipe().
|
||
|
|
"""
|
||
|
|
def __init__(self, recipecache, d, pn, fn, fns):
|
||
|
|
self._recipecache = recipecache
|
||
|
|
self._d = d
|
||
|
|
self.pn = pn
|
||
|
|
self.fn = fn
|
||
|
|
self.fns = fns
|
||
|
|
self.inherit_files = recipecache.inherits[fn]
|
||
|
|
self.depends = recipecache.deps[fn]
|
||
|
|
(self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
|
||
|
|
self._cached_packages = None
|
||
|
|
self._cached_rprovides = None
|
||
|
|
self._cached_packages_dynamic = None
|
||
|
|
|
||
|
|
def __getattr__(self, name):
|
||
|
|
if name == 'alternates':
|
||
|
|
return [x for x in self.fns if x != self.fn]
|
||
|
|
elif name == 'rdepends':
|
||
|
|
return self._recipecache.rundeps[self.fn]
|
||
|
|
elif name == 'rrecommends':
|
||
|
|
return self._recipecache.runrecs[self.fn]
|
||
|
|
elif name == 'provides':
|
||
|
|
return self._recipecache.fn_provides[self.fn]
|
||
|
|
elif name == 'packages':
|
||
|
|
if self._cached_packages is None:
|
||
|
|
self._cached_packages = []
|
||
|
|
for pkg, fns in self._recipecache.packages.items():
|
||
|
|
if self.fn in fns:
|
||
|
|
self._cached_packages.append(pkg)
|
||
|
|
return self._cached_packages
|
||
|
|
elif name == 'packages_dynamic':
|
||
|
|
if self._cached_packages_dynamic is None:
|
||
|
|
self._cached_packages_dynamic = []
|
||
|
|
for pkg, fns in self._recipecache.packages_dynamic.items():
|
||
|
|
if self.fn in fns:
|
||
|
|
self._cached_packages_dynamic.append(pkg)
|
||
|
|
return self._cached_packages_dynamic
|
||
|
|
elif name == 'rprovides':
|
||
|
|
if self._cached_rprovides is None:
|
||
|
|
self._cached_rprovides = []
|
||
|
|
for pkg, fns in self._recipecache.rproviders.items():
|
||
|
|
if self.fn in fns:
|
||
|
|
self._cached_rprovides.append(pkg)
|
||
|
|
return self._cached_rprovides
|
||
|
|
else:
|
||
|
|
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||
|
|
def inherits(self, only_recipe=False):
|
||
|
|
"""
|
||
|
|
Get the inherited classes for a recipe. Returns the class names only.
|
||
|
|
Parameters:
|
||
|
|
only_recipe: True to return only the classes inherited by the recipe
|
||
|
|
itself, False to return all classes inherited within
|
||
|
|
the context for the recipe (which includes globally
|
||
|
|
inherited classes).
|
||
|
|
"""
|
||
|
|
if only_recipe:
|
||
|
|
global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
|
||
|
|
else:
|
||
|
|
global_inherit = []
|
||
|
|
for clsfile in self.inherit_files:
|
||
|
|
if only_recipe and clsfile in global_inherit:
|
||
|
|
continue
|
||
|
|
clsname = os.path.splitext(os.path.basename(clsfile))[0]
|
||
|
|
yield clsname
|
||
|
|
def __str__(self):
|
||
|
|
return '%s' % self.pn
|
||
|
|
|
||
|
|
|
||
|
|
class Tinfoil:
|
||
|
|
"""
|
||
|
|
Tinfoil - an API for scripts and utilities to query
|
||
|
|
BitBake internals and perform build operations.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
|
||
|
|
"""
|
||
|
|
Create a new tinfoil object.
|
||
|
|
Parameters:
|
||
|
|
output: specifies where console output should be sent. Defaults
|
||
|
|
to sys.stdout.
|
||
|
|
tracking: True to enable variable history tracking, False to
|
||
|
|
disable it (default). Enabling this has a minor
|
||
|
|
performance impact so typically it isn't enabled
|
||
|
|
unless you need to query variable history.
|
||
|
|
setup_logging: True to setup a logger so that things like
|
||
|
|
bb.warn() will work immediately and timeout warnings
|
||
|
|
are visible; False to let BitBake do this itself.
|
||
|
|
"""
|
||
|
|
self.logger = logging.getLogger('BitBake')
|
||
|
|
self.config_data = None
|
||
|
|
self.cooker = None
|
||
|
|
self.tracking = tracking
|
||
|
|
self.ui_module = None
|
||
|
|
self.server_connection = None
|
||
|
|
self.recipes_parsed = False
|
||
|
|
self.quiet = 0
|
||
|
|
self.oldhandlers = self.logger.handlers[:]
|
||
|
|
if setup_logging:
|
||
|
|
# This is the *client-side* logger, nothing to do with
|
||
|
|
# logging messages from the server
|
||
|
|
bb.msg.logger_create('BitBake', output)
|
||
|
|
self.localhandlers = []
|
||
|
|
for handler in self.logger.handlers:
|
||
|
|
if handler not in self.oldhandlers:
|
||
|
|
self.localhandlers.append(handler)
|
||
|
|
|
||
|
|
def __enter__(self):
|
||
|
|
return self
|
||
|
|
|
||
|
|
def __exit__(self, type, value, traceback):
|
||
|
|
self.shutdown()
|
||
|
|
|
||
|
|
def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
|
||
|
|
"""
|
||
|
|
Prepares the underlying BitBake system to be used via tinfoil.
|
||
|
|
This function must be called prior to calling any of the other
|
||
|
|
functions in the API.
|
||
|
|
NOTE: if you call prepare() you must absolutely call shutdown()
|
||
|
|
before your code terminates. You can use a "with" block to ensure
|
||
|
|
this happens e.g.
|
||
|
|
|
||
|
|
with bb.tinfoil.Tinfoil() as tinfoil:
|
||
|
|
tinfoil.prepare()
|
||
|
|
...
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
config_only: True to read only the configuration and not load
|
||
|
|
the cache / parse recipes. This is useful if you just
|
||
|
|
want to query the value of a variable at the global
|
||
|
|
level or you want to do anything else that doesn't
|
||
|
|
involve knowing anything about the recipes in the
|
||
|
|
current configuration. False loads the cache / parses
|
||
|
|
recipes.
|
||
|
|
config_params: optionally specify your own configuration
|
||
|
|
parameters. If not specified an instance of
|
||
|
|
TinfoilConfigParameters will be created internally.
|
||
|
|
quiet: quiet level controlling console output - equivalent
|
||
|
|
to bitbake's -q/--quiet option. Default of 0 gives
|
||
|
|
the same output level as normal bitbake execution.
|
||
|
|
extra_features: extra features to be added to the feature
|
||
|
|
set requested from the server. See
|
||
|
|
CookerFeatures._feature_list for possible
|
||
|
|
features.
|
||
|
|
"""
|
||
|
|
self.quiet = quiet
|
||
|
|
|
||
|
|
if self.tracking:
|
||
|
|
extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
|
||
|
|
else:
|
||
|
|
extrafeatures = []
|
||
|
|
|
||
|
|
if extra_features:
|
||
|
|
extrafeatures += extra_features
|
||
|
|
|
||
|
|
if not config_params:
|
||
|
|
config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
|
||
|
|
|
||
|
|
cookerconfig = CookerConfiguration()
|
||
|
|
cookerconfig.setConfigParameters(config_params)
|
||
|
|
|
||
|
|
if not config_only:
|
||
|
|
# Disable local loggers because the UI module is going to set up its own
|
||
|
|
for handler in self.localhandlers:
|
||
|
|
self.logger.handlers.remove(handler)
|
||
|
|
self.localhandlers = []
|
||
|
|
|
||
|
|
self.server_connection, ui_module = setup_bitbake(config_params,
|
||
|
|
cookerconfig,
|
||
|
|
extrafeatures)
|
||
|
|
|
||
|
|
self.ui_module = ui_module
|
||
|
|
|
||
|
|
# Ensure the path to bitbake's bin directory is in PATH so that things like
|
||
|
|
# bitbake-worker can be run (usually this is the case, but it doesn't have to be)
|
||
|
|
path = os.getenv('PATH').split(':')
|
||
|
|
bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
|
||
|
|
for entry in path:
|
||
|
|
if entry.endswith(os.sep):
|
||
|
|
entry = entry[:-1]
|
||
|
|
if os.path.abspath(entry) == bitbakebinpath:
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
path.insert(0, bitbakebinpath)
|
||
|
|
os.environ['PATH'] = ':'.join(path)
|
||
|
|
|
||
|
|
if self.server_connection:
|
||
|
|
_server_connections.append(self.server_connection)
|
||
|
|
if config_only:
|
||
|
|
config_params.updateToServer(self.server_connection.connection, os.environ.copy())
|
||
|
|
self.run_command('parseConfiguration')
|
||
|
|
else:
|
||
|
|
self.run_actions(config_params)
|
||
|
|
self.recipes_parsed = True
|
||
|
|
|
||
|
|
self.config_data = bb.data.init()
|
||
|
|
connector = TinfoilDataStoreConnector(self, None)
|
||
|
|
self.config_data.setVar('_remote_data', connector)
|
||
|
|
self.cooker = TinfoilCookerAdapter(self)
|
||
|
|
self.cooker_data = self.cooker.recipecaches['']
|
||
|
|
else:
|
||
|
|
raise Exception('Failed to start bitbake server')
|
||
|
|
|
||
|
|
def run_actions(self, config_params):
|
||
|
|
"""
|
||
|
|
Run the actions specified in config_params through the UI.
|
||
|
|
"""
|
||
|
|
ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
|
||
|
|
if ret:
|
||
|
|
raise TinfoilUIException(ret)
|
||
|
|
|
||
|
|
def parseRecipes(self):
|
||
|
|
"""
|
||
|
|
Legacy function - use parse_recipes() instead.
|
||
|
|
"""
|
||
|
|
self.parse_recipes()
|
||
|
|
|
||
|
|
def parse_recipes(self):
|
||
|
|
"""
|
||
|
|
Load information on all recipes. Normally you should specify
|
||
|
|
config_only=False when calling prepare() instead of using this
|
||
|
|
function; this function is designed for situations where you need
|
||
|
|
to initialise Tinfoil and use it with config_only=True first and
|
||
|
|
then conditionally call this function to parse recipes later.
|
||
|
|
"""
|
||
|
|
config_params = TinfoilConfigParameters(config_only=False)
|
||
|
|
self.run_actions(config_params)
|
||
|
|
self.recipes_parsed = True
|
||
|
|
|
||
|
|
def run_command(self, command, *params):
|
||
|
|
"""
|
||
|
|
Run a command on the server (as implemented in bb.command).
|
||
|
|
Note that there are two types of command - synchronous and
|
||
|
|
asynchronous; in order to receive the results of asynchronous
|
||
|
|
commands you will need to set an appropriate event mask
|
||
|
|
using set_event_mask() and listen for the result using
|
||
|
|
wait_event() - with the correct event mask you'll at least get
|
||
|
|
bb.command.CommandCompleted and possibly other events before
|
||
|
|
that depending on the command.
|
||
|
|
"""
|
||
|
|
if not self.server_connection:
|
||
|
|
raise Exception('Not connected to server (did you call .prepare()?)')
|
||
|
|
|
||
|
|
commandline = [command]
|
||
|
|
if params:
|
||
|
|
commandline.extend(params)
|
||
|
|
result = self.server_connection.connection.runCommand(commandline)
|
||
|
|
if result[1]:
|
||
|
|
raise TinfoilCommandFailed(result[1])
|
||
|
|
return result[0]
|
||
|
|
|
||
|
|
def set_event_mask(self, eventlist):
|
||
|
|
"""Set the event mask which will be applied within wait_event()"""
|
||
|
|
if not self.server_connection:
|
||
|
|
raise Exception('Not connected to server (did you call .prepare()?)')
|
||
|
|
llevel, debug_domains = bb.msg.constructLogOptions()
|
||
|
|
ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
|
||
|
|
if not ret:
|
||
|
|
raise Exception('setEventMask failed')
|
||
|
|
|
||
|
|
def wait_event(self, timeout=0):
|
||
|
|
"""
|
||
|
|
Wait for an event from the server for the specified time.
|
||
|
|
A timeout of 0 means don't wait if there are no events in the queue.
|
||
|
|
Returns the next event in the queue or None if the timeout was
|
||
|
|
reached. Note that in order to recieve any events you will
|
||
|
|
first need to set the internal event mask using set_event_mask()
|
||
|
|
(otherwise whatever event mask the UI set up will be in effect).
|
||
|
|
"""
|
||
|
|
if not self.server_connection:
|
||
|
|
raise Exception('Not connected to server (did you call .prepare()?)')
|
||
|
|
return self.server_connection.events.waitEvent(timeout)
|
||
|
|
|
||
|
|
def get_overlayed_recipes(self):
|
||
|
|
"""
|
||
|
|
Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
|
||
|
|
"""
|
||
|
|
return defaultdict(list, self.run_command('getOverlayedRecipes'))
|
||
|
|
|
||
|
|
def get_skipped_recipes(self):
|
||
|
|
"""
|
||
|
|
Find recipes which were skipped (i.e. SkipRecipe was raised
|
||
|
|
during parsing).
|
||
|
|
"""
|
||
|
|
return OrderedDict(self.run_command('getSkippedRecipes'))
|
||
|
|
|
||
|
|
def get_all_providers(self):
|
||
|
|
return defaultdict(list, self.run_command('allProviders'))
|
||
|
|
|
||
|
|
def find_providers(self):
|
||
|
|
return self.run_command('findProviders')
|
||
|
|
|
||
|
|
def find_best_provider(self, pn):
|
||
|
|
return self.run_command('findBestProvider', pn)
|
||
|
|
|
||
|
|
def get_runtime_providers(self, rdep):
|
||
|
|
return self.run_command('getRuntimeProviders', rdep)
|
||
|
|
|
||
|
|
def get_recipe_file(self, pn):
|
||
|
|
"""
|
||
|
|
Get the file name for the specified recipe/target. Raises
|
||
|
|
bb.providers.NoProvider if there is no match or the recipe was
|
||
|
|
skipped.
|
||
|
|
"""
|
||
|
|
best = self.find_best_provider(pn)
|
||
|
|
if not best or (len(best) > 3 and not best[3]):
|
||
|
|
skiplist = self.get_skipped_recipes()
|
||
|
|
taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
|
||
|
|
skipreasons = taskdata.get_reasons(pn)
|
||
|
|
if skipreasons:
|
||
|
|
raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons)))
|
||
|
|
else:
|
||
|
|
raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn)
|
||
|
|
return best[3]
|
||
|
|
|
||
|
|
def get_file_appends(self, fn):
|
||
|
|
"""
|
||
|
|
Find the bbappends for a recipe file
|
||
|
|
"""
|
||
|
|
return self.run_command('getFileAppends', fn)
|
||
|
|
|
||
|
|
def all_recipes(self, mc='', sort=True):
|
||
|
|
"""
|
||
|
|
Enable iterating over all recipes in the current configuration.
|
||
|
|
Returns an iterator over TinfoilRecipeInfo objects created on demand.
|
||
|
|
Parameters:
|
||
|
|
mc: The multiconfig, default of '' uses the main configuration.
|
||
|
|
sort: True to sort recipes alphabetically (default), False otherwise
|
||
|
|
"""
|
||
|
|
recipecache = self.cooker.recipecaches[mc]
|
||
|
|
if sort:
|
||
|
|
recipes = sorted(recipecache.pkg_pn.items())
|
||
|
|
else:
|
||
|
|
recipes = recipecache.pkg_pn.items()
|
||
|
|
for pn, fns in recipes:
|
||
|
|
prov = self.find_best_provider(pn)
|
||
|
|
recipe = TinfoilRecipeInfo(recipecache,
|
||
|
|
self.config_data,
|
||
|
|
pn=pn,
|
||
|
|
fn=prov[3],
|
||
|
|
fns=fns)
|
||
|
|
yield recipe
|
||
|
|
|
||
|
|
def all_recipe_files(self, mc='', variants=True, preferred_only=False):
|
||
|
|
"""
|
||
|
|
Enable iterating over all recipe files in the current configuration.
|
||
|
|
Returns an iterator over file paths.
|
||
|
|
Parameters:
|
||
|
|
mc: The multiconfig, default of '' uses the main configuration.
|
||
|
|
variants: True to include variants of recipes created through
|
||
|
|
BBCLASSEXTEND (default) or False to exclude them
|
||
|
|
preferred_only: True to include only the preferred recipe where
|
||
|
|
multiple exist providing the same PN, False to list
|
||
|
|
all recipes
|
||
|
|
"""
|
||
|
|
recipecache = self.cooker.recipecaches[mc]
|
||
|
|
if preferred_only:
|
||
|
|
files = []
|
||
|
|
for pn in recipecache.pkg_pn.keys():
|
||
|
|
prov = self.find_best_provider(pn)
|
||
|
|
files.append(prov[3])
|
||
|
|
else:
|
||
|
|
files = recipecache.pkg_fn.keys()
|
||
|
|
for fn in sorted(files):
|
||
|
|
if not variants and fn.startswith('virtual:'):
|
||
|
|
continue
|
||
|
|
yield fn
|
||
|
|
|
||
|
|
|
||
|
|
def get_recipe_info(self, pn, mc=''):
|
||
|
|
"""
|
||
|
|
Get information on a specific recipe in the current configuration by name (PN).
|
||
|
|
Returns a TinfoilRecipeInfo object created on demand.
|
||
|
|
Parameters:
|
||
|
|
mc: The multiconfig, default of '' uses the main configuration.
|
||
|
|
"""
|
||
|
|
recipecache = self.cooker.recipecaches[mc]
|
||
|
|
prov = self.find_best_provider(pn)
|
||
|
|
fn = prov[3]
|
||
|
|
if fn:
|
||
|
|
actual_pn = recipecache.pkg_fn[fn]
|
||
|
|
recipe = TinfoilRecipeInfo(recipecache,
|
||
|
|
self.config_data,
|
||
|
|
pn=actual_pn,
|
||
|
|
fn=fn,
|
||
|
|
fns=recipecache.pkg_pn[actual_pn])
|
||
|
|
return recipe
|
||
|
|
else:
|
||
|
|
return None
|
||
|
|
|
||
|
|
def parse_recipe(self, pn):
|
||
|
|
"""
|
||
|
|
Parse the specified recipe and return a datastore object
|
||
|
|
representing the environment for the recipe.
|
||
|
|
"""
|
||
|
|
fn = self.get_recipe_file(pn)
|
||
|
|
return self.parse_recipe_file(fn)
|
||
|
|
|
||
|
|
def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
|
||
|
|
"""
|
||
|
|
Parse the specified recipe file (with or without bbappends)
|
||
|
|
and return a datastore object representing the environment
|
||
|
|
for the recipe.
|
||
|
|
Parameters:
|
||
|
|
fn: recipe file to parse - can be a file path or virtual
|
||
|
|
specification
|
||
|
|
appends: True to apply bbappends, False otherwise
|
||
|
|
appendlist: optional list of bbappend files to apply, if you
|
||
|
|
want to filter them
|
||
|
|
config_data: custom config datastore to use. NOTE: if you
|
||
|
|
specify config_data then you cannot use a virtual
|
||
|
|
specification for fn.
|
||
|
|
"""
|
||
|
|
if self.tracking:
|
||
|
|
# Enable history tracking just for the parse operation
|
||
|
|
self.run_command('enableDataTracking')
|
||
|
|
try:
|
||
|
|
if appends and appendlist == []:
|
||
|
|
appends = False
|
||
|
|
if config_data:
|
||
|
|
dctr = bb.remotedata.RemoteDatastores.transmit_datastore(config_data)
|
||
|
|
dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, dctr)
|
||
|
|
else:
|
||
|
|
dscon = self.run_command('parseRecipeFile', fn, appends, appendlist)
|
||
|
|
if dscon:
|
||
|
|
return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
|
||
|
|
else:
|
||
|
|
return None
|
||
|
|
finally:
|
||
|
|
if self.tracking:
|
||
|
|
self.run_command('disableDataTracking')
|
||
|
|
|
||
|
|
def build_file(self, buildfile, task, internal=True):
|
||
|
|
"""
|
||
|
|
Runs the specified task for just a single recipe (i.e. no dependencies).
|
||
|
|
This is equivalent to bitbake -b, except with the default internal=True
|
||
|
|
no warning about dependencies will be produced, normal info messages
|
||
|
|
from the runqueue will be silenced and BuildInit, BuildStarted and
|
||
|
|
BuildCompleted events will not be fired.
|
||
|
|
"""
|
||
|
|
return self.run_command('buildFile', buildfile, task, internal)
|
||
|
|
|
||
|
|
def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
|
||
|
|
"""
|
||
|
|
Builds the specified targets. This is equivalent to a normal invocation
|
||
|
|
of bitbake. Has built-in event handling which is enabled by default and
|
||
|
|
can be extended if needed.
|
||
|
|
Parameters:
|
||
|
|
targets:
|
||
|
|
One or more targets to build. Can be a list or a
|
||
|
|
space-separated string.
|
||
|
|
task:
|
||
|
|
The task to run; if None then the value of BB_DEFAULT_TASK
|
||
|
|
will be used. Default None.
|
||
|
|
handle_events:
|
||
|
|
True to handle events in a similar way to normal bitbake
|
||
|
|
invocation with knotty; False to return immediately (on the
|
||
|
|
assumption that the caller will handle the events instead).
|
||
|
|
Default True.
|
||
|
|
extra_events:
|
||
|
|
An optional list of events to add to the event mask (if
|
||
|
|
handle_events=True). If you add events here you also need
|
||
|
|
to specify a callback function in event_callback that will
|
||
|
|
handle the additional events. Default None.
|
||
|
|
event_callback:
|
||
|
|
An optional function taking a single parameter which
|
||
|
|
will be called first upon receiving any event (if
|
||
|
|
handle_events=True) so that the caller can override or
|
||
|
|
extend the event handling. Default None.
|
||
|
|
"""
|
||
|
|
if isinstance(targets, str):
|
||
|
|
targets = targets.split()
|
||
|
|
if not task:
|
||
|
|
task = self.config_data.getVar('BB_DEFAULT_TASK')
|
||
|
|
|
||
|
|
if handle_events:
|
||
|
|
# A reasonable set of default events matching up with those we handle below
|
||
|
|
eventmask = [
|
||
|
|
'bb.event.BuildStarted',
|
||
|
|
'bb.event.BuildCompleted',
|
||
|
|
'logging.LogRecord',
|
||
|
|
'bb.event.NoProvider',
|
||
|
|
'bb.command.CommandCompleted',
|
||
|
|
'bb.command.CommandFailed',
|
||
|
|
'bb.build.TaskStarted',
|
||
|
|
'bb.build.TaskFailed',
|
||
|
|
'bb.build.TaskSucceeded',
|
||
|
|
'bb.build.TaskFailedSilent',
|
||
|
|
'bb.build.TaskProgress',
|
||
|
|
'bb.runqueue.runQueueTaskStarted',
|
||
|
|
'bb.runqueue.sceneQueueTaskStarted',
|
||
|
|
'bb.event.ProcessStarted',
|
||
|
|
'bb.event.ProcessProgress',
|
||
|
|
'bb.event.ProcessFinished',
|
||
|
|
]
|
||
|
|
if extra_events:
|
||
|
|
eventmask.extend(extra_events)
|
||
|
|
ret = self.set_event_mask(eventmask)
|
||
|
|
|
||
|
|
includelogs = self.config_data.getVar('BBINCLUDELOGS')
|
||
|
|
loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
|
||
|
|
|
||
|
|
ret = self.run_command('buildTargets', targets, task)
|
||
|
|
if handle_events:
|
||
|
|
result = False
|
||
|
|
# Borrowed from knotty, instead somewhat hackily we use the helper
|
||
|
|
# as the object to store "shutdown" on
|
||
|
|
helper = bb.ui.uihelper.BBUIHelper()
|
||
|
|
# We set up logging optionally in the constructor so now we need to
|
||
|
|
# grab the handlers to pass to TerminalFilter
|
||
|
|
console = None
|
||
|
|
errconsole = None
|
||
|
|
for handler in self.logger.handlers:
|
||
|
|
if isinstance(handler, logging.StreamHandler):
|
||
|
|
if handler.stream == sys.stdout:
|
||
|
|
console = handler
|
||
|
|
elif handler.stream == sys.stderr:
|
||
|
|
errconsole = handler
|
||
|
|
format_str = "%(levelname)s: %(message)s"
|
||
|
|
format = bb.msg.BBLogFormatter(format_str)
|
||
|
|
helper.shutdown = 0
|
||
|
|
parseprogress = None
|
||
|
|
termfilter = bb.ui.knotty.TerminalFilter(helper, helper, console, errconsole, format, quiet=self.quiet)
|
||
|
|
try:
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
event = self.wait_event(0.25)
|
||
|
|
if event:
|
||
|
|
if event_callback and event_callback(event):
|
||
|
|
continue
|
||
|
|
if helper.eventHandler(event):
|
||
|
|
if isinstance(event, bb.build.TaskFailedSilent):
|
||
|
|
logger.warning("Logfile for failed setscene task is %s" % event.logfile)
|
||
|
|
elif isinstance(event, bb.build.TaskFailed):
|
||
|
|
bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
|
||
|
|
continue
|
||
|
|
if isinstance(event, bb.event.ProcessStarted):
|
||
|
|
if self.quiet > 1:
|
||
|
|
continue
|
||
|
|
parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
|
||
|
|
parseprogress.start(False)
|
||
|
|
continue
|
||
|
|
if isinstance(event, bb.event.ProcessProgress):
|
||
|
|
if self.quiet > 1:
|
||
|
|
continue
|
||
|
|
if parseprogress:
|
||
|
|
parseprogress.update(event.progress)
|
||
|
|
else:
|
||
|
|
bb.warn("Got ProcessProgress event for someting that never started?")
|
||
|
|
continue
|
||
|
|
if isinstance(event, bb.event.ProcessFinished):
|
||
|
|
if self.quiet > 1:
|
||
|
|
continue
|
||
|
|
if parseprogress:
|
||
|
|
parseprogress.finish()
|
||
|
|
parseprogress = None
|
||
|
|
continue
|
||
|
|
if isinstance(event, bb.command.CommandCompleted):
|
||
|
|
result = True
|
||
|
|
break
|
||
|
|
if isinstance(event, bb.command.CommandFailed):
|
||
|
|
self.logger.error(str(event))
|
||
|
|
result = False
|
||
|
|
break
|
||
|
|
if isinstance(event, logging.LogRecord):
|
||
|
|
if event.taskpid == 0 or event.levelno > logging.INFO:
|
||
|
|
self.logger.handle(event)
|
||
|
|
continue
|
||
|
|
if isinstance(event, bb.event.NoProvider):
|
||
|
|
self.logger.error(str(event))
|
||
|
|
result = False
|
||
|
|
break
|
||
|
|
|
||
|
|
elif helper.shutdown > 1:
|
||
|
|
break
|
||
|
|
termfilter.updateFooter()
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
termfilter.clearFooter()
|
||
|
|
if helper.shutdown == 1:
|
||
|
|
print("\nSecond Keyboard Interrupt, stopping...\n")
|
||
|
|
ret = self.run_command("stateForceShutdown")
|
||
|
|
if ret and ret[2]:
|
||
|
|
self.logger.error("Unable to cleanly stop: %s" % ret[2])
|
||
|
|
elif helper.shutdown == 0:
|
||
|
|
print("\nKeyboard Interrupt, closing down...\n")
|
||
|
|
interrupted = True
|
||
|
|
ret = self.run_command("stateShutdown")
|
||
|
|
if ret and ret[2]:
|
||
|
|
self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
|
||
|
|
helper.shutdown = helper.shutdown + 1
|
||
|
|
termfilter.clearFooter()
|
||
|
|
finally:
|
||
|
|
termfilter.finish()
|
||
|
|
if helper.failed_tasks:
|
||
|
|
result = False
|
||
|
|
return result
|
||
|
|
else:
|
||
|
|
return ret
|
||
|
|
|
||
|
|
def shutdown(self):
|
||
|
|
"""
|
||
|
|
Shut down tinfoil. Disconnects from the server and gracefully
|
||
|
|
releases any associated resources. You must call this function if
|
||
|
|
prepare() has been called, or use a with... block when you create
|
||
|
|
the tinfoil object which will ensure that it gets called.
|
||
|
|
"""
|
||
|
|
if self.server_connection:
|
||
|
|
self.run_command('clientComplete')
|
||
|
|
_server_connections.remove(self.server_connection)
|
||
|
|
bb.event.ui_queue = []
|
||
|
|
self.server_connection.terminate()
|
||
|
|
self.server_connection = None
|
||
|
|
|
||
|
|
# Restore logging handlers to how it looked when we started
|
||
|
|
if self.oldhandlers:
|
||
|
|
for handler in self.logger.handlers:
|
||
|
|
if handler not in self.oldhandlers:
|
||
|
|
self.logger.handlers.remove(handler)
|
||
|
|
|
||
|
|
def _reconvert_type(self, obj, origtypename):
|
||
|
|
"""
|
||
|
|
Convert an object back to the right type, in the case
|
||
|
|
that marshalling has changed it (especially with xmlrpc)
|
||
|
|
"""
|
||
|
|
supported_types = {
|
||
|
|
'set': set,
|
||
|
|
'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
|
||
|
|
}
|
||
|
|
|
||
|
|
origtype = supported_types.get(origtypename, None)
|
||
|
|
if origtype is None:
|
||
|
|
raise Exception('Unsupported type "%s"' % origtypename)
|
||
|
|
if type(obj) == origtype:
|
||
|
|
newobj = obj
|
||
|
|
elif isinstance(obj, dict):
|
||
|
|
# New style class
|
||
|
|
newobj = origtype()
|
||
|
|
for k,v in obj.items():
|
||
|
|
setattr(newobj, k, v)
|
||
|
|
else:
|
||
|
|
# Assume we can coerce the type
|
||
|
|
newobj = origtype(obj)
|
||
|
|
|
||
|
|
if isinstance(newobj, bb.command.DataStoreConnectionHandle):
|
||
|
|
connector = TinfoilDataStoreConnector(self, newobj.dsindex)
|
||
|
|
newobj = bb.data.init()
|
||
|
|
newobj.setVar('_remote_data', connector)
|
||
|
|
|
||
|
|
return newobj
|
||
|
|
|
||
|
|
|
||
|
|
class TinfoilConfigParameters(BitBakeConfigParameters):
|
||
|
|
|
||
|
|
def __init__(self, config_only, **options):
|
||
|
|
self.initial_options = options
|
||
|
|
# Apply some sane defaults
|
||
|
|
if not 'parse_only' in options:
|
||
|
|
self.initial_options['parse_only'] = not config_only
|
||
|
|
#if not 'status_only' in options:
|
||
|
|
# self.initial_options['status_only'] = config_only
|
||
|
|
if not 'ui' in options:
|
||
|
|
self.initial_options['ui'] = 'knotty'
|
||
|
|
if not 'argv' in options:
|
||
|
|
self.initial_options['argv'] = []
|
||
|
|
|
||
|
|
super(TinfoilConfigParameters, self).__init__()
|
||
|
|
|
||
|
|
def parseCommandLine(self, argv=None):
|
||
|
|
# We don't want any parameters parsed from the command line
|
||
|
|
opts = super(TinfoilConfigParameters, self).parseCommandLine([])
|
||
|
|
for key, val in self.initial_options.items():
|
||
|
|
setattr(opts[0], key, val)
|
||
|
|
return opts
|