1833 lines
67 KiB
Python
1833 lines
67 KiB
Python
#
|
|
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
#
|
|
# BitBake Toaster Implementation
|
|
#
|
|
# Copyright (C) 2013 Intel Corporation
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
from django.db import models, IntegrityError, DataError
|
|
from django.db.models import F, Q, Sum, Count
|
|
from django.utils import timezone
|
|
from django.utils.encoding import force_bytes
|
|
|
|
from django.core.urlresolvers import reverse
|
|
|
|
from django.core import validators
|
|
from django.conf import settings
|
|
import django.db.models.signals
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import itertools
|
|
from signal import SIGUSR1
|
|
|
|
|
|
import logging
|
|
logger = logging.getLogger("toaster")
|
|
|
|
if 'sqlite' in settings.DATABASES['default']['ENGINE']:
|
|
from django.db import transaction, OperationalError
|
|
from time import sleep
|
|
|
|
_base_save = models.Model.save
|
|
def save(self, *args, **kwargs):
|
|
while True:
|
|
try:
|
|
with transaction.atomic():
|
|
return _base_save(self, *args, **kwargs)
|
|
except OperationalError as err:
|
|
if 'database is locked' in str(err):
|
|
logger.warning("%s, model: %s, args: %s, kwargs: %s",
|
|
err, self.__class__, args, kwargs)
|
|
sleep(0.5)
|
|
continue
|
|
raise
|
|
|
|
models.Model.save = save
|
|
|
|
# HACK: Monkey patch Django to fix 'database is locked' issue
|
|
|
|
from django.db.models.query import QuerySet
|
|
_base_insert = QuerySet._insert
|
|
def _insert(self, *args, **kwargs):
|
|
with transaction.atomic(using=self.db, savepoint=False):
|
|
return _base_insert(self, *args, **kwargs)
|
|
QuerySet._insert = _insert
|
|
|
|
from django.utils import six
|
|
def _create_object_from_params(self, lookup, params):
|
|
"""
|
|
Tries to create an object using passed params.
|
|
Used by get_or_create and update_or_create
|
|
"""
|
|
try:
|
|
obj = self.create(**params)
|
|
return obj, True
|
|
except (IntegrityError, DataError):
|
|
exc_info = sys.exc_info()
|
|
try:
|
|
return self.get(**lookup), False
|
|
except self.model.DoesNotExist:
|
|
pass
|
|
six.reraise(*exc_info)
|
|
|
|
QuerySet._create_object_from_params = _create_object_from_params
|
|
|
|
# end of HACK
|
|
|
|
class GitURLValidator(validators.URLValidator):
|
|
import re
|
|
regex = re.compile(
|
|
r'^(?:ssh|git|http|ftp)s?://' # http:// or https://
|
|
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
|
r'localhost|' # localhost...
|
|
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
|
|
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
|
|
r'(?::\d+)?' # optional port
|
|
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
|
|
|
def GitURLField(**kwargs):
|
|
r = models.URLField(**kwargs)
|
|
for i in range(len(r.validators)):
|
|
if isinstance(r.validators[i], validators.URLValidator):
|
|
r.validators[i] = GitURLValidator()
|
|
return r
|
|
|
|
|
|
class ToasterSetting(models.Model):
|
|
name = models.CharField(max_length=63)
|
|
helptext = models.TextField()
|
|
value = models.CharField(max_length=255)
|
|
|
|
def __unicode__(self):
|
|
return "Setting %s = %s" % (self.name, self.value)
|
|
|
|
|
|
class ProjectManager(models.Manager):
|
|
def create_project(self, name, release):
|
|
if release is not None:
|
|
prj = self.model(name=name,
|
|
bitbake_version=release.bitbake_version,
|
|
release=release)
|
|
else:
|
|
prj = self.model(name=name,
|
|
bitbake_version=None,
|
|
release=None)
|
|
|
|
prj.save()
|
|
|
|
for defaultconf in ToasterSetting.objects.filter(
|
|
name__startswith="DEFCONF_"):
|
|
name = defaultconf.name[8:]
|
|
ProjectVariable.objects.create(project=prj,
|
|
name=name,
|
|
value=defaultconf.value)
|
|
|
|
if release is None:
|
|
return prj
|
|
|
|
for rdl in release.releasedefaultlayer_set.all():
|
|
lv = Layer_Version.objects.filter(
|
|
layer__name=rdl.layer_name,
|
|
release=release).first()
|
|
|
|
if lv:
|
|
ProjectLayer.objects.create(project=prj,
|
|
layercommit=lv,
|
|
optional=False)
|
|
else:
|
|
logger.warning("Default project layer %s not found" %
|
|
rdl.layer_name)
|
|
|
|
return prj
|
|
|
|
# return single object with is_default = True
|
|
def get_or_create_default_project(self):
|
|
projects = super(ProjectManager, self).filter(is_default=True)
|
|
|
|
if len(projects) > 1:
|
|
raise Exception('Inconsistent project data: multiple ' +
|
|
'default projects (i.e. with is_default=True)')
|
|
elif len(projects) < 1:
|
|
options = {
|
|
'name': 'Command line builds',
|
|
'short_description':
|
|
'Project for builds started outside Toaster',
|
|
'is_default': True
|
|
}
|
|
project = Project.objects.create(**options)
|
|
project.save()
|
|
|
|
return project
|
|
else:
|
|
return projects[0]
|
|
|
|
|
|
class Project(models.Model):
|
|
search_allowed_fields = ['name', 'short_description', 'release__name',
|
|
'release__branch_name']
|
|
name = models.CharField(max_length=100)
|
|
short_description = models.CharField(max_length=50, blank=True)
|
|
bitbake_version = models.ForeignKey('BitbakeVersion', null=True)
|
|
release = models.ForeignKey("Release", null=True)
|
|
created = models.DateTimeField(auto_now_add=True)
|
|
updated = models.DateTimeField(auto_now=True)
|
|
# This is a horrible hack; since Toaster has no "User" model available when
|
|
# running in interactive mode, we can't reference the field here directly
|
|
# Instead, we keep a possible null reference to the User id,
|
|
# as not to force
|
|
# hard links to possibly missing models
|
|
user_id = models.IntegerField(null=True)
|
|
objects = ProjectManager()
|
|
|
|
# set to True for the project which is the default container
|
|
# for builds initiated by the command line etc.
|
|
is_default= models.BooleanField(default=False)
|
|
|
|
def __unicode__(self):
|
|
return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
|
|
|
|
def get_current_machine_name(self):
|
|
try:
|
|
return self.projectvariable_set.get(name="MACHINE").value
|
|
except (ProjectVariable.DoesNotExist,IndexError):
|
|
return None;
|
|
|
|
def get_number_of_builds(self):
|
|
"""Return the number of builds which have ended"""
|
|
|
|
return self.build_set.exclude(
|
|
Q(outcome=Build.IN_PROGRESS) |
|
|
Q(outcome=Build.CANCELLED)
|
|
).count()
|
|
|
|
def get_last_build_id(self):
|
|
try:
|
|
return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id
|
|
except (Build.DoesNotExist,IndexError):
|
|
return( -1 )
|
|
|
|
def get_last_outcome(self):
|
|
build_id = self.get_last_build_id()
|
|
if (-1 == build_id):
|
|
return( "" )
|
|
try:
|
|
return Build.objects.filter( id = build_id )[ 0 ].outcome
|
|
except (Build.DoesNotExist,IndexError):
|
|
return( "not_found" )
|
|
|
|
def get_last_target(self):
|
|
build_id = self.get_last_build_id()
|
|
if (-1 == build_id):
|
|
return( "" )
|
|
try:
|
|
return Target.objects.filter(build = build_id)[0].target
|
|
except (Target.DoesNotExist,IndexError):
|
|
return( "not_found" )
|
|
|
|
def get_last_errors(self):
|
|
build_id = self.get_last_build_id()
|
|
if (-1 == build_id):
|
|
return( 0 )
|
|
try:
|
|
return Build.objects.filter(id = build_id)[ 0 ].errors.count()
|
|
except (Build.DoesNotExist,IndexError):
|
|
return( "not_found" )
|
|
|
|
def get_last_warnings(self):
|
|
build_id = self.get_last_build_id()
|
|
if (-1 == build_id):
|
|
return( 0 )
|
|
try:
|
|
return Build.objects.filter(id = build_id)[ 0 ].warnings.count()
|
|
except (Build.DoesNotExist,IndexError):
|
|
return( "not_found" )
|
|
|
|
def get_last_build_extensions(self):
|
|
"""
|
|
Get list of file name extensions for images produced by the most
|
|
recent build
|
|
"""
|
|
last_build = Build.objects.get(pk = self.get_last_build_id())
|
|
return last_build.get_image_file_extensions()
|
|
|
|
def get_last_imgfiles(self):
|
|
build_id = self.get_last_build_id()
|
|
if (-1 == build_id):
|
|
return( "" )
|
|
try:
|
|
return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value
|
|
except (Variable.DoesNotExist,IndexError):
|
|
return( "not_found" )
|
|
|
|
def get_all_compatible_layer_versions(self):
|
|
""" Returns Queryset of all Layer_Versions which are compatible with
|
|
this project"""
|
|
queryset = None
|
|
|
|
# guard on release, as it can be null
|
|
if self.release:
|
|
queryset = Layer_Version.objects.filter(
|
|
(Q(release=self.release) &
|
|
Q(build=None) &
|
|
Q(project=None)) |
|
|
Q(project=self))
|
|
else:
|
|
queryset = Layer_Version.objects.none()
|
|
|
|
return queryset
|
|
|
|
def get_project_layer_versions(self, pk=False):
|
|
""" Returns the Layer_Versions currently added to this project """
|
|
layer_versions = self.projectlayer_set.all().values_list('layercommit',
|
|
flat=True)
|
|
|
|
if pk is False:
|
|
return Layer_Version.objects.filter(pk__in=layer_versions)
|
|
else:
|
|
return layer_versions
|
|
|
|
|
|
def get_available_machines(self):
|
|
""" Returns QuerySet of all Machines which are provided by the
|
|
Layers currently added to the Project """
|
|
queryset = Machine.objects.filter(
|
|
layer_version__in=self.get_project_layer_versions())
|
|
|
|
return queryset
|
|
|
|
def get_all_compatible_machines(self):
|
|
""" Returns QuerySet of all the compatible machines available to the
|
|
project including ones from Layers not currently added """
|
|
queryset = Machine.objects.filter(
|
|
layer_version__in=self.get_all_compatible_layer_versions())
|
|
|
|
return queryset
|
|
|
|
def get_available_distros(self):
|
|
""" Returns QuerySet of all Distros which are provided by the
|
|
Layers currently added to the Project """
|
|
queryset = Distro.objects.filter(
|
|
layer_version__in=self.get_project_layer_versions())
|
|
|
|
return queryset
|
|
|
|
def get_all_compatible_distros(self):
|
|
""" Returns QuerySet of all the compatible Wind River distros available to the
|
|
project including ones from Layers not currently added """
|
|
queryset = Distro.objects.filter(
|
|
layer_version__in=self.get_all_compatible_layer_versions())
|
|
|
|
return queryset
|
|
|
|
def get_available_recipes(self):
|
|
""" Returns QuerySet of all the recipes that are provided by layers
|
|
added to this project """
|
|
queryset = Recipe.objects.filter(
|
|
layer_version__in=self.get_project_layer_versions())
|
|
|
|
return queryset
|
|
|
|
def get_all_compatible_recipes(self):
|
|
""" Returns QuerySet of all the compatible Recipes available to the
|
|
project including ones from Layers not currently added """
|
|
queryset = Recipe.objects.filter(
|
|
layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='')
|
|
|
|
return queryset
|
|
|
|
def schedule_build(self):
|
|
|
|
from bldcontrol.models import BuildRequest, BRTarget, BRLayer
|
|
from bldcontrol.models import BRBitbake, BRVariable
|
|
|
|
try:
|
|
now = timezone.now()
|
|
build = Build.objects.create(project=self,
|
|
completed_on=now,
|
|
started_on=now)
|
|
|
|
br = BuildRequest.objects.create(project=self,
|
|
state=BuildRequest.REQ_QUEUED,
|
|
build=build)
|
|
BRBitbake.objects.create(req=br,
|
|
giturl=self.bitbake_version.giturl,
|
|
commit=self.bitbake_version.branch,
|
|
dirpath=self.bitbake_version.dirpath)
|
|
|
|
for t in self.projecttarget_set.all():
|
|
BRTarget.objects.create(req=br, target=t.target, task=t.task)
|
|
Target.objects.create(build=br.build, target=t.target,
|
|
task=t.task)
|
|
# If we're about to build a custom image recipe make sure
|
|
# that layer is currently in the project before we create the
|
|
# BRLayer objects
|
|
customrecipe = CustomImageRecipe.objects.filter(
|
|
name=t.target,
|
|
project=self).first()
|
|
if customrecipe:
|
|
ProjectLayer.objects.get_or_create(
|
|
project=self,
|
|
layercommit=customrecipe.layer_version,
|
|
optional=False)
|
|
|
|
for l in self.projectlayer_set.all().order_by("pk"):
|
|
commit = l.layercommit.get_vcs_reference()
|
|
logger.debug("Adding layer to build %s" %
|
|
l.layercommit.layer.name)
|
|
BRLayer.objects.create(
|
|
req=br,
|
|
name=l.layercommit.layer.name,
|
|
giturl=l.layercommit.layer.vcs_url,
|
|
commit=commit,
|
|
dirpath=l.layercommit.dirpath,
|
|
layer_version=l.layercommit,
|
|
local_source_dir=l.layercommit.layer.local_source_dir
|
|
)
|
|
|
|
for v in self.projectvariable_set.all():
|
|
BRVariable.objects.create(req=br, name=v.name, value=v.value)
|
|
|
|
try:
|
|
br.build.machine = self.projectvariable_set.get(
|
|
name='MACHINE').value
|
|
br.build.save()
|
|
except ProjectVariable.DoesNotExist:
|
|
pass
|
|
|
|
br.save()
|
|
signal_runbuilds()
|
|
|
|
except Exception:
|
|
# revert the build request creation since we're not done cleanly
|
|
br.delete()
|
|
raise
|
|
return br
|
|
|
|
class Build(models.Model):
|
|
SUCCEEDED = 0
|
|
FAILED = 1
|
|
IN_PROGRESS = 2
|
|
CANCELLED = 3
|
|
|
|
BUILD_OUTCOME = (
|
|
(SUCCEEDED, 'Succeeded'),
|
|
(FAILED, 'Failed'),
|
|
(IN_PROGRESS, 'In Progress'),
|
|
(CANCELLED, 'Cancelled'),
|
|
)
|
|
|
|
search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"]
|
|
|
|
project = models.ForeignKey(Project) # must have a project
|
|
machine = models.CharField(max_length=100)
|
|
distro = models.CharField(max_length=100)
|
|
distro_version = models.CharField(max_length=100)
|
|
started_on = models.DateTimeField()
|
|
completed_on = models.DateTimeField()
|
|
outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
|
|
cooker_log_path = models.CharField(max_length=500)
|
|
build_name = models.CharField(max_length=100, default='')
|
|
bitbake_version = models.CharField(max_length=50)
|
|
|
|
# number of recipes to parse for this build
|
|
recipes_to_parse = models.IntegerField(default=1)
|
|
|
|
# number of recipes parsed so far for this build
|
|
recipes_parsed = models.IntegerField(default=1)
|
|
|
|
# number of repos to clone for this build
|
|
repos_to_clone = models.IntegerField(default=1)
|
|
|
|
# number of repos cloned so far for this build (default off)
|
|
repos_cloned = models.IntegerField(default=1)
|
|
|
|
@staticmethod
|
|
def get_recent(project=None):
|
|
"""
|
|
Return recent builds as a list; if project is set, only return
|
|
builds for that project
|
|
"""
|
|
|
|
builds = Build.objects.all()
|
|
|
|
if project:
|
|
builds = builds.filter(project=project)
|
|
|
|
finished_criteria = \
|
|
Q(outcome=Build.SUCCEEDED) | \
|
|
Q(outcome=Build.FAILED) | \
|
|
Q(outcome=Build.CANCELLED)
|
|
|
|
recent_builds = list(itertools.chain(
|
|
builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"),
|
|
builds.filter(finished_criteria).order_by("-completed_on")[:3]
|
|
))
|
|
|
|
# add percentage done property to each build; this is used
|
|
# to show build progress in mrb_section.html
|
|
for build in recent_builds:
|
|
build.percentDone = build.completeper()
|
|
build.outcomeText = build.get_outcome_text()
|
|
|
|
return recent_builds
|
|
|
|
def started(self):
|
|
"""
|
|
As build variables are only added for a build when its BuildStarted event
|
|
is received, a build with no build variables is counted as
|
|
"in preparation" and not properly started yet. This method
|
|
will return False if a build has no build variables (it never properly
|
|
started), or True otherwise.
|
|
|
|
Note that this is a temporary workaround for the fact that we don't
|
|
have a fine-grained state variable on a build which would allow us
|
|
to record "in progress" (BuildStarted received) vs. "in preparation".
|
|
"""
|
|
variables = Variable.objects.filter(build=self)
|
|
return len(variables) > 0
|
|
|
|
def completeper(self):
|
|
tf = Task.objects.filter(build = self)
|
|
tfc = tf.count()
|
|
if tfc > 0:
|
|
completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
|
|
else:
|
|
completeper = 0
|
|
return completeper
|
|
|
|
def eta(self):
|
|
eta = timezone.now()
|
|
completeper = self.completeper()
|
|
if self.completeper() > 0:
|
|
eta += ((eta - self.started_on)*(100-completeper))/completeper
|
|
return eta
|
|
|
|
def has_images(self):
|
|
"""
|
|
Returns True if at least one of the targets for this build has an
|
|
image file associated with it, False otherwise
|
|
"""
|
|
targets = Target.objects.filter(build_id=self.id)
|
|
has_images = False
|
|
for target in targets:
|
|
if target.has_images():
|
|
has_images = True
|
|
break
|
|
return has_images
|
|
|
|
def has_image_recipes(self):
|
|
"""
|
|
Returns True if a build has any targets which were built from
|
|
image recipes.
|
|
"""
|
|
image_recipes = self.get_image_recipes()
|
|
return len(image_recipes) > 0
|
|
|
|
def get_image_file_extensions(self):
|
|
"""
|
|
Get string of file name extensions for images produced by this build;
|
|
note that this is the actual list of extensions stored on Target objects
|
|
for this build, and not the value of IMAGE_FSTYPES.
|
|
|
|
Returns comma-separated string, e.g. "vmdk, ext4"
|
|
"""
|
|
extensions = []
|
|
|
|
targets = Target.objects.filter(build_id = self.id)
|
|
for target in targets:
|
|
if not target.is_image:
|
|
continue
|
|
|
|
target_image_files = Target_Image_File.objects.filter(
|
|
target_id=target.id)
|
|
|
|
for target_image_file in target_image_files:
|
|
extensions.append(target_image_file.suffix)
|
|
|
|
extensions = list(set(extensions))
|
|
extensions.sort()
|
|
|
|
return ', '.join(extensions)
|
|
|
|
def get_image_fstypes(self):
|
|
"""
|
|
Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
|
|
list of image file suffixes.
|
|
"""
|
|
image_fstypes = Variable.objects.get(
|
|
build=self, variable_name='IMAGE_FSTYPES').variable_value
|
|
return list(set(re.split(r' {1,}', image_fstypes)))
|
|
|
|
def get_sorted_target_list(self):
|
|
tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
|
|
return( tgts );
|
|
|
|
def get_recipes(self):
|
|
"""
|
|
Get the recipes related to this build;
|
|
note that the related layer versions and layers are also prefetched
|
|
by this query, as this queryset can be sorted by these objects in the
|
|
build recipes view; prefetching them here removes the need
|
|
for another query in that view
|
|
"""
|
|
layer_versions = Layer_Version.objects.filter(build=self)
|
|
criteria = Q(layer_version__id__in=layer_versions)
|
|
return Recipe.objects.filter(criteria) \
|
|
.select_related('layer_version', 'layer_version__layer')
|
|
|
|
def get_image_recipes(self):
|
|
"""
|
|
Returns a list of image Recipes (custom and built-in) related to this
|
|
build, sorted by name; note that this has to be done in two steps, as
|
|
there's no way to get all the custom image recipes and image recipes
|
|
in one query
|
|
"""
|
|
custom_image_recipes = self.get_custom_image_recipes()
|
|
custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True)
|
|
|
|
not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \
|
|
Q(is_image=True)
|
|
|
|
built_image_recipes = self.get_recipes().filter(not_custom_image_recipes)
|
|
|
|
# append to the custom image recipes and sort
|
|
customisable_image_recipes = list(
|
|
itertools.chain(custom_image_recipes, built_image_recipes)
|
|
)
|
|
|
|
return sorted(customisable_image_recipes, key=lambda recipe: recipe.name)
|
|
|
|
def get_custom_image_recipes(self):
|
|
"""
|
|
Returns a queryset of CustomImageRecipes related to this build,
|
|
sorted by name
|
|
"""
|
|
built_recipe_names = self.get_recipes().values_list('name', flat=True)
|
|
criteria = Q(name__in=built_recipe_names) & Q(project=self.project)
|
|
queryset = CustomImageRecipe.objects.filter(criteria).order_by('name')
|
|
return queryset
|
|
|
|
def get_outcome_text(self):
|
|
return Build.BUILD_OUTCOME[int(self.outcome)][1]
|
|
|
|
@property
|
|
def failed_tasks(self):
|
|
""" Get failed tasks for the build """
|
|
tasks = self.task_build.all()
|
|
return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED)
|
|
|
|
@property
|
|
def errors(self):
|
|
return (self.logmessage_set.filter(level=LogMessage.ERROR) |
|
|
self.logmessage_set.filter(level=LogMessage.EXCEPTION) |
|
|
self.logmessage_set.filter(level=LogMessage.CRITICAL))
|
|
|
|
@property
|
|
def warnings(self):
|
|
return self.logmessage_set.filter(level=LogMessage.WARNING)
|
|
|
|
@property
|
|
def timespent(self):
|
|
return self.completed_on - self.started_on
|
|
|
|
@property
|
|
def timespent_seconds(self):
|
|
return self.timespent.total_seconds()
|
|
|
|
@property
|
|
def target_labels(self):
|
|
"""
|
|
Sorted (a-z) "target1:task, target2, target3" etc. string for all
|
|
targets in this build
|
|
"""
|
|
targets = self.target_set.all()
|
|
target_labels = [target.target +
|
|
(':' + target.task if target.task else '')
|
|
for target in targets]
|
|
target_labels.sort()
|
|
|
|
return target_labels
|
|
|
|
def get_buildrequest(self):
|
|
buildrequest = None
|
|
if hasattr(self, 'buildrequest'):
|
|
buildrequest = self.buildrequest
|
|
return buildrequest
|
|
|
|
def is_queued(self):
|
|
from bldcontrol.models import BuildRequest
|
|
buildrequest = self.get_buildrequest()
|
|
if buildrequest:
|
|
return buildrequest.state == BuildRequest.REQ_QUEUED
|
|
else:
|
|
return False
|
|
|
|
def is_cancelling(self):
|
|
from bldcontrol.models import BuildRequest
|
|
buildrequest = self.get_buildrequest()
|
|
if buildrequest:
|
|
return self.outcome == Build.IN_PROGRESS and \
|
|
buildrequest.state == BuildRequest.REQ_CANCELLING
|
|
else:
|
|
return False
|
|
|
|
def is_cloning(self):
|
|
"""
|
|
True if the build is still cloning repos
|
|
"""
|
|
return self.outcome == Build.IN_PROGRESS and \
|
|
self.repos_cloned < self.repos_to_clone
|
|
|
|
def is_parsing(self):
|
|
"""
|
|
True if the build is still parsing recipes
|
|
"""
|
|
return self.outcome == Build.IN_PROGRESS and \
|
|
self.recipes_parsed < self.recipes_to_parse
|
|
|
|
def is_starting(self):
|
|
"""
|
|
True if the build has no completed tasks yet and is still just starting
|
|
tasks.
|
|
|
|
Note that the mechanism for testing whether a Task is "done" is whether
|
|
its outcome field is set, as per the completeper() method.
|
|
"""
|
|
return self.outcome == Build.IN_PROGRESS and \
|
|
self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
|
|
|
|
|
|
def get_state(self):
|
|
"""
|
|
Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
|
|
'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
|
|
dependent on the BuildRequest state).
|
|
|
|
This works around the fact that we have BuildRequest states as well
|
|
as Build states, but really we just want to know the state of the build.
|
|
"""
|
|
if self.is_cancelling():
|
|
return 'Cancelling';
|
|
elif self.is_queued():
|
|
return 'Queued'
|
|
elif self.is_cloning():
|
|
return 'Cloning'
|
|
elif self.is_parsing():
|
|
return 'Parsing'
|
|
elif self.is_starting():
|
|
return 'Starting'
|
|
else:
|
|
return self.get_outcome_text()
|
|
|
|
def __str__(self):
|
|
return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
|
|
|
|
class ProjectTarget(models.Model):
|
|
project = models.ForeignKey(Project)
|
|
target = models.CharField(max_length=100)
|
|
task = models.CharField(max_length=100, null=True)
|
|
|
|
class Target(models.Model):
|
|
search_allowed_fields = ['target', 'file_name']
|
|
build = models.ForeignKey(Build)
|
|
target = models.CharField(max_length=100)
|
|
task = models.CharField(max_length=100, null=True)
|
|
is_image = models.BooleanField(default = False)
|
|
image_size = models.IntegerField(default=0)
|
|
license_manifest_path = models.CharField(max_length=500, null=True)
|
|
package_manifest_path = models.CharField(max_length=500, null=True)
|
|
|
|
def package_count(self):
|
|
return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
|
|
|
|
def __unicode__(self):
|
|
return self.target
|
|
|
|
def get_similar_targets(self):
|
|
"""
|
|
Get target sfor the same machine, task and target name
|
|
(e.g. 'core-image-minimal') from a successful build for this project
|
|
(but excluding this target).
|
|
|
|
Note that we only look for targets built by this project because
|
|
projects can have different configurations from each other, and put
|
|
their artifacts in different directories.
|
|
|
|
The possibility of error when retrieving candidate targets
|
|
is minimised by the fact that bitbake will rebuild artifacts if MACHINE
|
|
(or various other variables) change. In this case, there is no need to
|
|
clone artifacts from another target, as those artifacts will have
|
|
been re-generated for this target anyway.
|
|
"""
|
|
query = ~Q(pk=self.pk) & \
|
|
Q(target=self.target) & \
|
|
Q(build__machine=self.build.machine) & \
|
|
Q(build__outcome=Build.SUCCEEDED) & \
|
|
Q(build__project=self.build.project)
|
|
|
|
return Target.objects.filter(query)
|
|
|
|
def get_similar_target_with_image_files(self):
|
|
"""
|
|
Get the most recent similar target with Target_Image_Files associated
|
|
with it, for the purpose of cloning those files onto this target.
|
|
"""
|
|
similar_target = None
|
|
|
|
candidates = self.get_similar_targets()
|
|
if candidates.count() == 0:
|
|
return similar_target
|
|
|
|
task_subquery = Q(task=self.task)
|
|
|
|
# we can look for a 'build' task if this task is a 'populate_sdk_ext'
|
|
# task, as the latter also creates images; and vice versa; note that
|
|
# 'build' targets can have their task set to '';
|
|
# also note that 'populate_sdk' does not produce image files
|
|
image_tasks = [
|
|
'', # aka 'build'
|
|
'build',
|
|
'image',
|
|
'populate_sdk_ext'
|
|
]
|
|
if self.task in image_tasks:
|
|
task_subquery = Q(task__in=image_tasks)
|
|
|
|
# annotate with the count of files, to exclude any targets which
|
|
# don't have associated files
|
|
candidates = candidates.annotate(num_files=Count('target_image_file'))
|
|
|
|
query = task_subquery & Q(num_files__gt=0)
|
|
|
|
candidates = candidates.filter(query)
|
|
|
|
if candidates.count() > 0:
|
|
candidates.order_by('build__completed_on')
|
|
similar_target = candidates.last()
|
|
|
|
return similar_target
|
|
|
|
def get_similar_target_with_sdk_files(self):
|
|
"""
|
|
Get the most recent similar target with TargetSDKFiles associated
|
|
with it, for the purpose of cloning those files onto this target.
|
|
"""
|
|
similar_target = None
|
|
|
|
candidates = self.get_similar_targets()
|
|
if candidates.count() == 0:
|
|
return similar_target
|
|
|
|
# annotate with the count of files, to exclude any targets which
|
|
# don't have associated files
|
|
candidates = candidates.annotate(num_files=Count('targetsdkfile'))
|
|
|
|
query = Q(task=self.task) & Q(num_files__gt=0)
|
|
|
|
candidates = candidates.filter(query)
|
|
|
|
if candidates.count() > 0:
|
|
candidates.order_by('build__completed_on')
|
|
similar_target = candidates.last()
|
|
|
|
return similar_target
|
|
|
|
def clone_image_artifacts_from(self, target):
|
|
"""
|
|
Make clones of the Target_Image_Files and TargetKernelFile objects
|
|
associated with Target target, then associate them with this target.
|
|
|
|
Note that for Target_Image_Files, we only want files from the previous
|
|
build whose suffix matches one of the suffixes defined in this
|
|
target's build's IMAGE_FSTYPES configuration variable. This prevents the
|
|
Target_Image_File object for an ext4 image being associated with a
|
|
target for a project which didn't produce an ext4 image (for example).
|
|
|
|
Also sets the license_manifest_path and package_manifest_path
|
|
of this target to the same path as that of target being cloned from, as
|
|
the manifests are also build artifacts but are treated differently.
|
|
"""
|
|
|
|
image_fstypes = self.build.get_image_fstypes()
|
|
|
|
# filter out any image files whose suffixes aren't in the
|
|
# IMAGE_FSTYPES suffixes variable for this target's build
|
|
image_files = [target_image_file \
|
|
for target_image_file in target.target_image_file_set.all() \
|
|
if target_image_file.suffix in image_fstypes]
|
|
|
|
for image_file in image_files:
|
|
image_file.pk = None
|
|
image_file.target = self
|
|
image_file.save()
|
|
|
|
kernel_files = target.targetkernelfile_set.all()
|
|
for kernel_file in kernel_files:
|
|
kernel_file.pk = None
|
|
kernel_file.target = self
|
|
kernel_file.save()
|
|
|
|
self.license_manifest_path = target.license_manifest_path
|
|
self.package_manifest_path = target.package_manifest_path
|
|
self.save()
|
|
|
|
def clone_sdk_artifacts_from(self, target):
|
|
"""
|
|
Clone TargetSDKFile objects from target and associate them with this
|
|
target.
|
|
"""
|
|
sdk_files = target.targetsdkfile_set.all()
|
|
for sdk_file in sdk_files:
|
|
sdk_file.pk = None
|
|
sdk_file.target = self
|
|
sdk_file.save()
|
|
|
|
def has_images(self):
|
|
"""
|
|
Returns True if this target has one or more image files attached to it.
|
|
"""
|
|
return self.target_image_file_set.all().count() > 0
|
|
|
|
# kernel artifacts for a target: bzImage and modules*
|
|
class TargetKernelFile(models.Model):
|
|
target = models.ForeignKey(Target)
|
|
file_name = models.FilePathField()
|
|
file_size = models.IntegerField()
|
|
|
|
@property
|
|
def basename(self):
|
|
return os.path.basename(self.file_name)
|
|
|
|
# SDK artifacts for a target: sh and manifest files
|
|
class TargetSDKFile(models.Model):
|
|
target = models.ForeignKey(Target)
|
|
file_name = models.FilePathField()
|
|
file_size = models.IntegerField()
|
|
|
|
@property
|
|
def basename(self):
|
|
return os.path.basename(self.file_name)
|
|
|
|
class Target_Image_File(models.Model):
|
|
# valid suffixes for image files produced by a build
|
|
SUFFIXES = {
|
|
'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz',
|
|
'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4',
|
|
'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2',
|
|
'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo',
|
|
'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
|
|
'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma'
|
|
}
|
|
|
|
target = models.ForeignKey(Target)
|
|
file_name = models.FilePathField(max_length=254)
|
|
file_size = models.IntegerField()
|
|
|
|
@property
|
|
def suffix(self):
|
|
"""
|
|
Suffix for image file, minus leading "."
|
|
"""
|
|
for suffix in Target_Image_File.SUFFIXES:
|
|
if self.file_name.endswith(suffix):
|
|
return suffix
|
|
|
|
filename, suffix = os.path.splitext(self.file_name)
|
|
suffix = suffix.lstrip('.')
|
|
return suffix
|
|
|
|
class Target_File(models.Model):
|
|
ITYPE_REGULAR = 1
|
|
ITYPE_DIRECTORY = 2
|
|
ITYPE_SYMLINK = 3
|
|
ITYPE_SOCKET = 4
|
|
ITYPE_FIFO = 5
|
|
ITYPE_CHARACTER = 6
|
|
ITYPE_BLOCK = 7
|
|
ITYPES = ( (ITYPE_REGULAR ,'regular'),
|
|
( ITYPE_DIRECTORY ,'directory'),
|
|
( ITYPE_SYMLINK ,'symlink'),
|
|
( ITYPE_SOCKET ,'socket'),
|
|
( ITYPE_FIFO ,'fifo'),
|
|
( ITYPE_CHARACTER ,'character'),
|
|
( ITYPE_BLOCK ,'block'),
|
|
)
|
|
|
|
target = models.ForeignKey(Target)
|
|
path = models.FilePathField()
|
|
size = models.IntegerField()
|
|
inodetype = models.IntegerField(choices = ITYPES)
|
|
permission = models.CharField(max_length=16)
|
|
owner = models.CharField(max_length=128)
|
|
group = models.CharField(max_length=128)
|
|
directory = models.ForeignKey('Target_File', related_name="directory_set", null=True)
|
|
sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True)
|
|
|
|
|
|
class Task(models.Model):
|
|
|
|
SSTATE_NA = 0
|
|
SSTATE_MISS = 1
|
|
SSTATE_FAILED = 2
|
|
SSTATE_RESTORED = 3
|
|
|
|
SSTATE_RESULT = (
|
|
(SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
|
|
(SSTATE_MISS, 'File not in cache'), # the sstate object was not found
|
|
(SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
|
|
(SSTATE_RESTORED, 'Succeeded'), # successfully restored
|
|
)
|
|
|
|
CODING_NA = 0
|
|
CODING_PYTHON = 2
|
|
CODING_SHELL = 3
|
|
|
|
TASK_CODING = (
|
|
(CODING_NA, 'N/A'),
|
|
(CODING_PYTHON, 'Python'),
|
|
(CODING_SHELL, 'Shell'),
|
|
)
|
|
|
|
OUTCOME_NA = -1
|
|
OUTCOME_SUCCESS = 0
|
|
OUTCOME_COVERED = 1
|
|
OUTCOME_CACHED = 2
|
|
OUTCOME_PREBUILT = 3
|
|
OUTCOME_FAILED = 4
|
|
OUTCOME_EMPTY = 5
|
|
|
|
TASK_OUTCOME = (
|
|
(OUTCOME_NA, 'Not Available'),
|
|
(OUTCOME_SUCCESS, 'Succeeded'),
|
|
(OUTCOME_COVERED, 'Covered'),
|
|
(OUTCOME_CACHED, 'Cached'),
|
|
(OUTCOME_PREBUILT, 'Prebuilt'),
|
|
(OUTCOME_FAILED, 'Failed'),
|
|
(OUTCOME_EMPTY, 'Empty'),
|
|
)
|
|
|
|
TASK_OUTCOME_HELP = (
|
|
(OUTCOME_SUCCESS, 'This task successfully completed'),
|
|
(OUTCOME_COVERED, 'This task did not run because its output is provided by another task'),
|
|
(OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'),
|
|
(OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'),
|
|
(OUTCOME_FAILED, 'This task did not complete'),
|
|
(OUTCOME_EMPTY, 'This task has no executable content'),
|
|
(OUTCOME_NA, ''),
|
|
)
|
|
|
|
search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ]
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(Task, self).__init__(*args, **kwargs)
|
|
try:
|
|
self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text
|
|
except HelpText.DoesNotExist:
|
|
self._helptext = None
|
|
|
|
def get_related_setscene(self):
|
|
return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene")
|
|
|
|
def get_outcome_text(self):
|
|
return Task.TASK_OUTCOME[int(self.outcome) + 1][1]
|
|
|
|
def get_outcome_help(self):
|
|
return Task.TASK_OUTCOME_HELP[int(self.outcome)][1]
|
|
|
|
def get_sstate_text(self):
|
|
if self.sstate_result==Task.SSTATE_NA:
|
|
return ''
|
|
else:
|
|
return Task.SSTATE_RESULT[int(self.sstate_result)][1]
|
|
|
|
def get_executed_display(self):
|
|
if self.task_executed:
|
|
return "Executed"
|
|
return "Not Executed"
|
|
|
|
def get_description(self):
|
|
return self._helptext
|
|
|
|
build = models.ForeignKey(Build, related_name='task_build')
|
|
order = models.IntegerField(null=True)
|
|
task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed
|
|
outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
|
|
sstate_checksum = models.CharField(max_length=100, blank=True)
|
|
path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
|
|
recipe = models.ForeignKey('Recipe', related_name='tasks')
|
|
task_name = models.CharField(max_length=100)
|
|
source_url = models.FilePathField(max_length=255, blank=True)
|
|
work_directory = models.FilePathField(max_length=255, blank=True)
|
|
script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
|
|
line_number = models.IntegerField(default=0)
|
|
|
|
# start/end times
|
|
started = models.DateTimeField(null=True)
|
|
ended = models.DateTimeField(null=True)
|
|
|
|
# in seconds; this is stored to enable sorting
|
|
elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
|
|
|
|
# in bytes; note that disk_io is stored to enable sorting
|
|
disk_io = models.IntegerField(null=True)
|
|
disk_io_read = models.IntegerField(null=True)
|
|
disk_io_write = models.IntegerField(null=True)
|
|
|
|
# in seconds
|
|
cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
|
|
cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
|
|
|
|
sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
|
|
message = models.CharField(max_length=240)
|
|
logfile = models.FilePathField(max_length=255, blank=True)
|
|
|
|
outcome_text = property(get_outcome_text)
|
|
sstate_text = property(get_sstate_text)
|
|
|
|
def __unicode__(self):
|
|
return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name)
|
|
|
|
class Meta:
|
|
ordering = ('order', 'recipe' ,)
|
|
unique_together = ('build', 'recipe', 'task_name', )
|
|
|
|
|
|
class Task_Dependency(models.Model):
|
|
task = models.ForeignKey(Task, related_name='task_dependencies_task')
|
|
depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends')
|
|
|
|
class Package(models.Model):
|
|
search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name']
|
|
build = models.ForeignKey('Build', null=True)
|
|
recipe = models.ForeignKey('Recipe', null=True)
|
|
name = models.CharField(max_length=100)
|
|
installed_name = models.CharField(max_length=100, default='')
|
|
version = models.CharField(max_length=100, blank=True)
|
|
revision = models.CharField(max_length=32, blank=True)
|
|
summary = models.TextField(blank=True)
|
|
description = models.TextField(blank=True)
|
|
size = models.IntegerField(default=0)
|
|
installed_size = models.IntegerField(default=0)
|
|
section = models.CharField(max_length=80, blank=True)
|
|
license = models.CharField(max_length=80, blank=True)
|
|
|
|
@property
|
|
def is_locale_package(self):
|
|
""" Returns True if this package is identifiable as a locale package """
|
|
if self.name.find('locale') != -1:
|
|
return True
|
|
return False
|
|
|
|
@property
|
|
def is_packagegroup(self):
|
|
""" Returns True is this package is identifiable as a packagegroup """
|
|
if self.name.find('packagegroup') != -1:
|
|
return True
|
|
return False
|
|
|
|
class CustomImagePackage(Package):
|
|
# CustomImageRecipe fields to track pacakges appended,
|
|
# included and excluded from a CustomImageRecipe
|
|
recipe_includes = models.ManyToManyField('CustomImageRecipe',
|
|
related_name='includes_set')
|
|
recipe_excludes = models.ManyToManyField('CustomImageRecipe',
|
|
related_name='excludes_set')
|
|
recipe_appends = models.ManyToManyField('CustomImageRecipe',
|
|
related_name='appends_set')
|
|
|
|
|
|
class Package_DependencyManager(models.Manager):
|
|
use_for_related_fields = True
|
|
TARGET_LATEST = "use-latest-target-for-target"
|
|
|
|
def get_queryset(self):
|
|
return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id'))
|
|
|
|
def for_target_or_none(self, target):
|
|
""" filter the dependencies to be displayed by the supplied target
|
|
if no dependences are found for the target then try None as the target
|
|
which will return the dependences calculated without the context of a
|
|
target e.g. non image recipes.
|
|
|
|
returns: { size, packages }
|
|
"""
|
|
package_dependencies = self.all_depends().order_by('depends_on__name')
|
|
|
|
if target is self.TARGET_LATEST:
|
|
installed_deps =\
|
|
package_dependencies.filter(~Q(target__target=None))
|
|
else:
|
|
installed_deps =\
|
|
package_dependencies.filter(Q(target__target=target))
|
|
|
|
packages_list = None
|
|
total_size = 0
|
|
|
|
# If we have installed depdencies for this package and target then use
|
|
# these to display
|
|
if installed_deps.count() > 0:
|
|
packages_list = installed_deps
|
|
total_size = installed_deps.aggregate(
|
|
Sum('depends_on__size'))['depends_on__size__sum']
|
|
else:
|
|
new_list = []
|
|
package_names = []
|
|
|
|
# Find dependencies for the package that we know about even if
|
|
# it's not installed on a target e.g. from a non-image recipe
|
|
for p in package_dependencies.filter(Q(target=None)):
|
|
if p.depends_on.name in package_names:
|
|
continue
|
|
else:
|
|
package_names.append(p.depends_on.name)
|
|
new_list.append(p.pk)
|
|
# while we're here we may as well total up the size to
|
|
# avoid iterating again
|
|
total_size += p.depends_on.size
|
|
|
|
# We want to return a queryset here for consistency so pick the
|
|
# deps from the new_list
|
|
packages_list = package_dependencies.filter(Q(pk__in=new_list))
|
|
|
|
return {'packages': packages_list,
|
|
'size': total_size}
|
|
|
|
def all_depends(self):
|
|
""" Returns just the depends packages and not any other dep_type
|
|
Note that this is for any target
|
|
"""
|
|
return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) |
|
|
Q(dep_type=Package_Dependency.TYPE_TRDEPENDS))
|
|
|
|
|
|
class Package_Dependency(models.Model):
|
|
TYPE_RDEPENDS = 0
|
|
TYPE_TRDEPENDS = 1
|
|
TYPE_RRECOMMENDS = 2
|
|
TYPE_TRECOMMENDS = 3
|
|
TYPE_RSUGGESTS = 4
|
|
TYPE_RPROVIDES = 5
|
|
TYPE_RREPLACES = 6
|
|
TYPE_RCONFLICTS = 7
|
|
' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
|
|
DEPENDS_TYPE = (
|
|
(TYPE_RDEPENDS, "depends"),
|
|
(TYPE_TRDEPENDS, "depends"),
|
|
(TYPE_TRECOMMENDS, "recommends"),
|
|
(TYPE_RRECOMMENDS, "recommends"),
|
|
(TYPE_RSUGGESTS, "suggests"),
|
|
(TYPE_RPROVIDES, "provides"),
|
|
(TYPE_RREPLACES, "replaces"),
|
|
(TYPE_RCONFLICTS, "conflicts"),
|
|
)
|
|
""" Indexed by dep_type, in view order, key for short name and help
|
|
description which when viewed will be printf'd with the
|
|
package name.
|
|
"""
|
|
DEPENDS_DICT = {
|
|
TYPE_RDEPENDS : ("depends", "%s is required to run %s"),
|
|
TYPE_TRDEPENDS : ("depends", "%s is required to run %s"),
|
|
TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"),
|
|
TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"),
|
|
TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"),
|
|
TYPE_RPROVIDES : ("provides", "%s is provided by %s"),
|
|
TYPE_RREPLACES : ("replaces", "%s is replaced by %s"),
|
|
TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"),
|
|
}
|
|
|
|
package = models.ForeignKey(Package, related_name='package_dependencies_source')
|
|
depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency
|
|
dep_type = models.IntegerField(choices=DEPENDS_TYPE)
|
|
target = models.ForeignKey(Target, null=True)
|
|
objects = Package_DependencyManager()
|
|
|
|
class Target_Installed_Package(models.Model):
|
|
target = models.ForeignKey(Target)
|
|
package = models.ForeignKey(Package, related_name='buildtargetlist_package')
|
|
|
|
|
|
class Package_File(models.Model):
|
|
package = models.ForeignKey(Package, related_name='buildfilelist_package')
|
|
path = models.FilePathField(max_length=255, blank=True)
|
|
size = models.IntegerField()
|
|
|
|
|
|
class Recipe(models.Model):
|
|
search_allowed_fields = ['name', 'version', 'file_path', 'section',
|
|
'summary', 'description', 'license',
|
|
'layer_version__layer__name',
|
|
'layer_version__branch', 'layer_version__commit',
|
|
'layer_version__local_path',
|
|
'layer_version__layer_source']
|
|
|
|
up_date = models.DateTimeField(null=True, default=None)
|
|
|
|
name = models.CharField(max_length=100, blank=True)
|
|
version = models.CharField(max_length=100, blank=True)
|
|
layer_version = models.ForeignKey('Layer_Version',
|
|
related_name='recipe_layer_version')
|
|
summary = models.TextField(blank=True)
|
|
description = models.TextField(blank=True)
|
|
section = models.CharField(max_length=100, blank=True)
|
|
license = models.CharField(max_length=200, blank=True)
|
|
homepage = models.URLField(blank=True)
|
|
bugtracker = models.URLField(blank=True)
|
|
file_path = models.FilePathField(max_length=255)
|
|
pathflags = models.CharField(max_length=200, blank=True)
|
|
is_image = models.BooleanField(default=False)
|
|
|
|
def __unicode__(self):
|
|
return "Recipe " + self.name + ":" + self.version
|
|
|
|
def get_vcs_recipe_file_link_url(self):
|
|
return self.layer_version.get_vcs_file_link_url(self.file_path)
|
|
|
|
def get_description_or_summary(self):
|
|
if self.description:
|
|
return self.description
|
|
elif self.summary:
|
|
return self.summary
|
|
else:
|
|
return ""
|
|
|
|
class Meta:
|
|
unique_together = (("layer_version", "file_path", "pathflags"), )
|
|
|
|
|
|
class Recipe_DependencyManager(models.Manager):
|
|
use_for_related_fields = True
|
|
|
|
def get_queryset(self):
|
|
return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id'))
|
|
|
|
class Provides(models.Model):
|
|
name = models.CharField(max_length=100)
|
|
recipe = models.ForeignKey(Recipe)
|
|
|
|
class Recipe_Dependency(models.Model):
|
|
TYPE_DEPENDS = 0
|
|
TYPE_RDEPENDS = 1
|
|
|
|
DEPENDS_TYPE = (
|
|
(TYPE_DEPENDS, "depends"),
|
|
(TYPE_RDEPENDS, "rdepends"),
|
|
)
|
|
recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe')
|
|
depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends')
|
|
via = models.ForeignKey(Provides, null=True, default=None)
|
|
dep_type = models.IntegerField(choices=DEPENDS_TYPE)
|
|
objects = Recipe_DependencyManager()
|
|
|
|
|
|
class Machine(models.Model):
|
|
search_allowed_fields = ["name", "description", "layer_version__layer__name"]
|
|
up_date = models.DateTimeField(null = True, default = None)
|
|
|
|
layer_version = models.ForeignKey('Layer_Version')
|
|
name = models.CharField(max_length=255)
|
|
description = models.CharField(max_length=255)
|
|
|
|
def get_vcs_machine_file_link_url(self):
|
|
path = 'conf/machine/'+self.name+'.conf'
|
|
|
|
return self.layer_version.get_vcs_file_link_url(path)
|
|
|
|
def __unicode__(self):
|
|
return "Machine " + self.name + "(" + self.description + ")"
|
|
|
|
|
|
|
|
|
|
|
|
class BitbakeVersion(models.Model):
|
|
|
|
name = models.CharField(max_length=32, unique = True)
|
|
giturl = GitURLField()
|
|
branch = models.CharField(max_length=32)
|
|
dirpath = models.CharField(max_length=255)
|
|
|
|
def __unicode__(self):
|
|
return "%s (Branch: %s)" % (self.name, self.branch)
|
|
|
|
|
|
class Release(models.Model):
|
|
""" A release is a project template, used to pre-populate Project settings with a configuration set """
|
|
name = models.CharField(max_length=32, unique = True)
|
|
description = models.CharField(max_length=255)
|
|
bitbake_version = models.ForeignKey(BitbakeVersion)
|
|
branch_name = models.CharField(max_length=50, default = "")
|
|
helptext = models.TextField(null=True)
|
|
|
|
def __unicode__(self):
|
|
return "%s (%s)" % (self.name, self.branch_name)
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
class ReleaseDefaultLayer(models.Model):
|
|
release = models.ForeignKey(Release)
|
|
layer_name = models.CharField(max_length=100, default="")
|
|
|
|
|
|
class LayerSource(object):
|
|
""" Where the layer metadata came from """
|
|
TYPE_LOCAL = 0
|
|
TYPE_LAYERINDEX = 1
|
|
TYPE_IMPORTED = 2
|
|
TYPE_BUILD = 3
|
|
|
|
SOURCE_TYPE = (
|
|
(TYPE_LOCAL, "local"),
|
|
(TYPE_LAYERINDEX, "layerindex"),
|
|
(TYPE_IMPORTED, "imported"),
|
|
(TYPE_BUILD, "build"),
|
|
)
|
|
|
|
def types_dict():
|
|
""" Turn the TYPES enums into a simple dictionary """
|
|
dictionary = {}
|
|
for key in LayerSource.__dict__:
|
|
if "TYPE" in key:
|
|
dictionary[key] = getattr(LayerSource, key)
|
|
return dictionary
|
|
|
|
|
|
class Layer(models.Model):
|
|
|
|
up_date = models.DateTimeField(null=True, default=timezone.now)
|
|
|
|
name = models.CharField(max_length=100)
|
|
layer_index_url = models.URLField()
|
|
vcs_url = GitURLField(default=None, null=True)
|
|
local_source_dir = models.TextField(null=True, default=None)
|
|
vcs_web_url = models.URLField(null=True, default=None)
|
|
vcs_web_tree_base_url = models.URLField(null=True, default=None)
|
|
vcs_web_file_base_url = models.URLField(null=True, default=None)
|
|
|
|
summary = models.TextField(help_text='One-line description of the layer',
|
|
null=True, default=None)
|
|
description = models.TextField(null=True, default=None)
|
|
|
|
def __unicode__(self):
|
|
return "%s / %s " % (self.name, self.summary)
|
|
|
|
|
|
class Layer_Version(models.Model):
|
|
"""
|
|
A Layer_Version either belongs to a single project or no project
|
|
"""
|
|
search_allowed_fields = ["layer__name", "layer__summary",
|
|
"layer__description", "layer__vcs_url",
|
|
"dirpath", "release__name", "commit", "branch"]
|
|
|
|
build = models.ForeignKey(Build, related_name='layer_version_build',
|
|
default=None, null=True)
|
|
|
|
layer = models.ForeignKey(Layer, related_name='layer_version_layer')
|
|
|
|
layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
|
|
default=0)
|
|
|
|
up_date = models.DateTimeField(null=True, default=timezone.now)
|
|
|
|
# To which metadata release does this layer version belong to
|
|
release = models.ForeignKey(Release, null=True, default=None)
|
|
|
|
branch = models.CharField(max_length=80)
|
|
commit = models.CharField(max_length=100)
|
|
# If the layer is in a subdir
|
|
dirpath = models.CharField(max_length=255, null=True, default=None)
|
|
|
|
# if -1, this is a default layer
|
|
priority = models.IntegerField(default=0)
|
|
|
|
# where this layer exists on the filesystem
|
|
local_path = models.FilePathField(max_length=1024, default="/")
|
|
|
|
# Set if this layer is restricted to a particular project
|
|
project = models.ForeignKey('Project', null=True, default=None)
|
|
|
|
# code lifted, with adaptations, from the layerindex-web application
|
|
# https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
|
|
def _handle_url_path(self, base_url, path):
|
|
import re, posixpath
|
|
if base_url:
|
|
if self.dirpath:
|
|
if path:
|
|
extra_path = self.dirpath + '/' + path
|
|
# Normalise out ../ in path for usage URL
|
|
extra_path = posixpath.normpath(extra_path)
|
|
# Minor workaround to handle case where subdirectory has been added between branches
|
|
# (should probably support usage URL per branch to handle this... sigh...)
|
|
if extra_path.startswith('../'):
|
|
extra_path = extra_path[3:]
|
|
else:
|
|
extra_path = self.dirpath
|
|
else:
|
|
extra_path = path
|
|
branchname = self.release.name
|
|
url = base_url.replace('%branch%', branchname)
|
|
|
|
# If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
|
|
if extra_path:
|
|
extra_path = extra_path.replace('%', '%25')
|
|
|
|
if '%path%' in base_url:
|
|
if extra_path:
|
|
url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url)
|
|
else:
|
|
url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url)
|
|
return url.replace('%path%', extra_path)
|
|
else:
|
|
return url + extra_path
|
|
return None
|
|
|
|
def get_vcs_link_url(self):
|
|
if self.layer.vcs_web_url is None:
|
|
return None
|
|
return self.layer.vcs_web_url
|
|
|
|
def get_vcs_file_link_url(self, file_path=""):
|
|
if self.layer.vcs_web_file_base_url is None:
|
|
return None
|
|
return self._handle_url_path(self.layer.vcs_web_file_base_url,
|
|
file_path)
|
|
|
|
def get_vcs_dirpath_link_url(self):
|
|
if self.layer.vcs_web_tree_base_url is None:
|
|
return None
|
|
return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
|
|
|
|
def get_vcs_reference(self):
|
|
if self.commit is not None and len(self.commit) > 0:
|
|
return self.commit
|
|
if self.branch is not None and len(self.branch) > 0:
|
|
return self.branch
|
|
if self.release is not None:
|
|
return self.release.name
|
|
return 'N/A'
|
|
|
|
def get_detailspage_url(self, project_id=None):
|
|
""" returns the url to the layer details page uses own project
|
|
field if project_id is not specified """
|
|
|
|
if project_id is None:
|
|
project_id = self.project.pk
|
|
|
|
return reverse('layerdetails', args=(project_id, self.pk))
|
|
|
|
def get_alldeps(self, project_id):
|
|
"""Get full list of unique layer dependencies."""
|
|
def gen_layerdeps(lver, project, depth):
|
|
if depth == 0:
|
|
return
|
|
for ldep in lver.dependencies.all():
|
|
yield ldep.depends_on
|
|
# get next level of deps recursively calling gen_layerdeps
|
|
for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
|
|
yield subdep
|
|
|
|
project = Project.objects.get(pk=project_id)
|
|
result = []
|
|
projectlvers = [player.layercommit for player in
|
|
project.projectlayer_set.all()]
|
|
# protect against infinite layer dependency loops
|
|
maxdepth = 20
|
|
for dep in gen_layerdeps(self, project, maxdepth):
|
|
# filter out duplicates and layers already belonging to the project
|
|
if dep not in result + projectlvers:
|
|
result.append(dep)
|
|
|
|
return sorted(result, key=lambda x: x.layer.name)
|
|
|
|
def __unicode__(self):
|
|
return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
|
|
|
|
def __str__(self):
|
|
if self.release:
|
|
release = self.release.name
|
|
else:
|
|
release = "No release set"
|
|
|
|
return "%d %s (%s)" % (self.pk, self.layer.name, release)
|
|
|
|
|
|
class LayerVersionDependency(models.Model):
|
|
|
|
layer_version = models.ForeignKey(Layer_Version,
|
|
related_name="dependencies")
|
|
depends_on = models.ForeignKey(Layer_Version,
|
|
related_name="dependees")
|
|
|
|
class ProjectLayer(models.Model):
|
|
project = models.ForeignKey(Project)
|
|
layercommit = models.ForeignKey(Layer_Version, null=True)
|
|
optional = models.BooleanField(default = True)
|
|
|
|
def __unicode__(self):
|
|
return "%s, %s" % (self.project.name, self.layercommit)
|
|
|
|
class Meta:
|
|
unique_together = (("project", "layercommit"),)
|
|
|
|
class CustomImageRecipe(Recipe):
|
|
|
|
# CustomImageRecipe's belong to layers called:
|
|
LAYER_NAME = "toaster-custom-images"
|
|
|
|
search_allowed_fields = ['name']
|
|
base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe')
|
|
project = models.ForeignKey(Project)
|
|
last_updated = models.DateTimeField(null=True, default=None)
|
|
|
|
def get_last_successful_built_target(self):
|
|
""" Return the last successful built target object if one exists
|
|
otherwise return None """
|
|
return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
|
|
Q(build__project=self.project) &
|
|
Q(target=self.name)).last()
|
|
|
|
def update_package_list(self):
|
|
""" Update the package list from the last good build of this
|
|
CustomImageRecipe
|
|
"""
|
|
# Check if we're aldready up-to-date or not
|
|
target = self.get_last_successful_built_target()
|
|
if target == None:
|
|
# So we've never actually built this Custom recipe but what about
|
|
# the recipe it's based on?
|
|
target = \
|
|
Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
|
|
Q(build__project=self.project) &
|
|
Q(target=self.base_recipe.name)).last()
|
|
if target == None:
|
|
return
|
|
|
|
if target.build.completed_on == self.last_updated:
|
|
return
|
|
|
|
self.includes_set.clear()
|
|
|
|
excludes_list = self.excludes_set.values_list('name', flat=True)
|
|
appends_list = self.appends_set.values_list('name', flat=True)
|
|
|
|
built_packages_list = \
|
|
target.target_installed_package_set.values_list('package__name',
|
|
flat=True)
|
|
for built_package in built_packages_list:
|
|
# Is the built package in the custom packages list?
|
|
if built_package in excludes_list:
|
|
continue
|
|
|
|
if built_package in appends_list:
|
|
continue
|
|
|
|
cust_img_p = \
|
|
CustomImagePackage.objects.get(name=built_package)
|
|
self.includes_set.add(cust_img_p)
|
|
|
|
|
|
self.last_updated = target.build.completed_on
|
|
self.save()
|
|
|
|
def get_all_packages(self):
|
|
"""Get the included packages and any appended packages"""
|
|
self.update_package_list()
|
|
|
|
return CustomImagePackage.objects.filter((Q(recipe_appends=self) |
|
|
Q(recipe_includes=self)) &
|
|
~Q(recipe_excludes=self))
|
|
|
|
def get_base_recipe_file(self):
|
|
"""Get the base recipe file path if it exists on the file system"""
|
|
path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
|
|
self.base_recipe.file_path)
|
|
|
|
path_schema_two = self.base_recipe.file_path
|
|
|
|
if os.path.exists(path_schema_one):
|
|
return path_schema_one
|
|
|
|
# The path may now be the full path if the recipe has been built
|
|
if os.path.exists(path_schema_two):
|
|
return path_schema_two
|
|
|
|
return None
|
|
|
|
def generate_recipe_file_contents(self):
|
|
"""Generate the contents for the recipe file."""
|
|
# If we have no excluded packages we only need to _append
|
|
if self.excludes_set.count() == 0:
|
|
packages_conf = "IMAGE_INSTALL_append = \" "
|
|
|
|
for pkg in self.appends_set.all():
|
|
packages_conf += pkg.name+' '
|
|
else:
|
|
packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \""
|
|
# We add all the known packages to be built by this recipe apart
|
|
# from locale packages which are are controlled with IMAGE_LINGUAS.
|
|
for pkg in self.get_all_packages().exclude(
|
|
name__icontains="locale"):
|
|
packages_conf += pkg.name+' '
|
|
|
|
packages_conf += "\""
|
|
|
|
base_recipe_path = self.get_base_recipe_file()
|
|
if base_recipe_path:
|
|
base_recipe = open(base_recipe_path, 'r').read()
|
|
else:
|
|
raise IOError("Based on recipe file not found: %s" %
|
|
base_recipe_path)
|
|
|
|
# Add a special case for when the recipe we have based a custom image
|
|
# recipe on requires another recipe.
|
|
# For example:
|
|
# "require core-image-minimal.bb" is changed to:
|
|
# "require recipes-core/images/core-image-minimal.bb"
|
|
|
|
req_search = re.search(r'(require\s+)(.+\.bb\s*$)',
|
|
base_recipe,
|
|
re.MULTILINE)
|
|
if req_search:
|
|
require_filename = req_search.group(2).strip()
|
|
|
|
corrected_location = Recipe.objects.filter(
|
|
Q(layer_version=self.base_recipe.layer_version) &
|
|
Q(file_path__icontains=require_filename)).last().file_path
|
|
|
|
new_require_line = "require %s" % corrected_location
|
|
|
|
base_recipe = base_recipe.replace(req_search.group(0),
|
|
new_require_line)
|
|
|
|
info = {
|
|
"date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
"base_recipe": base_recipe,
|
|
"recipe_name": self.name,
|
|
"base_recipe_name": self.base_recipe.name,
|
|
"license": self.license,
|
|
"summary": self.summary,
|
|
"description": self.description,
|
|
"packages_conf": packages_conf.strip()
|
|
}
|
|
|
|
recipe_contents = ("# Original recipe %(base_recipe_name)s \n"
|
|
"%(base_recipe)s\n\n"
|
|
"# Recipe %(recipe_name)s \n"
|
|
"# Customisation Generated by Toaster on %(date)s\n"
|
|
"SUMMARY = \"%(summary)s\"\n"
|
|
"DESCRIPTION = \"%(description)s\"\n"
|
|
"LICENSE = \"%(license)s\"\n"
|
|
"%(packages_conf)s") % info
|
|
|
|
return recipe_contents
|
|
|
|
class ProjectVariable(models.Model):
|
|
project = models.ForeignKey(Project)
|
|
name = models.CharField(max_length=100)
|
|
value = models.TextField(blank = True)
|
|
|
|
class Variable(models.Model):
|
|
search_allowed_fields = ['variable_name', 'variable_value',
|
|
'vhistory__file_name', "description"]
|
|
build = models.ForeignKey(Build, related_name='variable_build')
|
|
variable_name = models.CharField(max_length=100)
|
|
variable_value = models.TextField(blank=True)
|
|
changed = models.BooleanField(default=False)
|
|
human_readable_name = models.CharField(max_length=200)
|
|
description = models.TextField(blank=True)
|
|
|
|
class VariableHistory(models.Model):
|
|
variable = models.ForeignKey(Variable, related_name='vhistory')
|
|
value = models.TextField(blank=True)
|
|
file_name = models.FilePathField(max_length=255)
|
|
line_number = models.IntegerField(null=True)
|
|
operation = models.CharField(max_length=64)
|
|
|
|
class HelpText(models.Model):
|
|
VARIABLE = 0
|
|
HELPTEXT_AREA = ((VARIABLE, 'variable'), )
|
|
|
|
build = models.ForeignKey(Build, related_name='helptext_build')
|
|
area = models.IntegerField(choices=HELPTEXT_AREA)
|
|
key = models.CharField(max_length=100)
|
|
text = models.TextField()
|
|
|
|
class LogMessage(models.Model):
|
|
EXCEPTION = -1 # used to signal self-toaster-exceptions
|
|
INFO = 0
|
|
WARNING = 1
|
|
ERROR = 2
|
|
CRITICAL = 3
|
|
|
|
LOG_LEVEL = (
|
|
(INFO, "info"),
|
|
(WARNING, "warn"),
|
|
(ERROR, "error"),
|
|
(CRITICAL, "critical"),
|
|
(EXCEPTION, "toaster exception")
|
|
)
|
|
|
|
build = models.ForeignKey(Build)
|
|
task = models.ForeignKey(Task, blank = True, null=True)
|
|
level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
|
|
message = models.TextField(blank=True, null=True)
|
|
pathname = models.FilePathField(max_length=255, blank=True)
|
|
lineno = models.IntegerField(null=True)
|
|
|
|
def __str__(self):
|
|
return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build))
|
|
|
|
def invalidate_cache(**kwargs):
|
|
from django.core.cache import cache
|
|
try:
|
|
cache.clear()
|
|
except Exception as e:
|
|
logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
|
|
|
|
def signal_runbuilds():
|
|
"""Send SIGUSR1 to runbuilds process"""
|
|
try:
|
|
with open(os.path.join(os.getenv('BUILDDIR', '.'),
|
|
'.runbuilds.pid')) as pidf:
|
|
os.kill(int(pidf.read()), SIGUSR1)
|
|
except FileNotFoundError:
|
|
logger.info("Stopping existing runbuilds: no current process found")
|
|
|
|
class Distro(models.Model):
|
|
search_allowed_fields = ["name", "description", "layer_version__layer__name"]
|
|
up_date = models.DateTimeField(null = True, default = None)
|
|
|
|
layer_version = models.ForeignKey('Layer_Version')
|
|
name = models.CharField(max_length=255)
|
|
description = models.CharField(max_length=255)
|
|
|
|
def get_vcs_distro_file_link_url(self):
|
|
path = self.name+'.conf'
|
|
return self.layer_version.get_vcs_file_link_url(path)
|
|
|
|
def __unicode__(self):
|
|
return "Distro " + self.name + "(" + self.description + ")"
|
|
|
|
django.db.models.signals.post_save.connect(invalidate_cache)
|
|
django.db.models.signals.post_delete.connect(invalidate_cache)
|
|
django.db.models.signals.m2m_changed.connect(invalidate_cache)
|