citadel/scripts/lib/wic/plugins/imager/direct.py

590 lines
23 KiB
Python
Raw Normal View History

# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Copyright (c) 2013, Intel Corporation.
# All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# DESCRIPTION
# This implements the 'direct' imager plugin class for 'wic'
#
# AUTHORS
# Tom Zanussi <tom.zanussi (at] linux.intel.com>
#
import logging
import os
import random
import shutil
import tempfile
import uuid
from time import strftime
from oe.path import copyhardlinktree
from wic import WicError
from wic.filemap import sparse_copy
from wic.ksparser import KickStart, KickStartError
from wic.pluginbase import PluginMgr, ImagerPlugin
from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd
logger = logging.getLogger('wic')
class DirectPlugin(ImagerPlugin):
"""
Install a system into a file containing a partitioned disk image.
An image file is formatted with a partition table, each partition
created from a rootfs or other OpenEmbedded build artifact and dd'ed
into the virtual disk. The disk image can subsequently be dd'ed onto
media and used on actual hardware.
"""
name = 'direct'
def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
native_sysroot, oe_builddir, options):
try:
self.ks = KickStart(wks_file)
except KickStartError as err:
raise WicError(str(err))
# parse possible 'rootfs=name' items
self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
self.bootimg_dir = bootimg_dir
self.kernel_dir = kernel_dir
self.native_sysroot = native_sysroot
self.oe_builddir = oe_builddir
self.outdir = options.outdir
self.compressor = options.compressor
self.bmap = options.bmap
self.no_fstab_update = options.no_fstab_update
self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
strftime("%Y%m%d%H%M"))
self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
self._image = None
self.ptable_format = self.ks.bootloader.ptable
self.parts = self.ks.partitions
# as a convenience, set source to the boot partition source
# instead of forcing it to be set via bootloader --source
for part in self.parts:
if not self.ks.bootloader.source and part.mountpoint == "/boot":
self.ks.bootloader.source = part.source
break
image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
self._image = PartitionedImage(image_path, self.ptable_format,
self.parts, self.native_sysroot)
def do_create(self):
"""
Plugin entry point.
"""
try:
self.create()
self.assemble()
self.finalize()
self.print_info()
finally:
self.cleanup()
def _write_fstab(self, image_rootfs):
"""overriden to generate fstab (temporarily) in rootfs. This is called
from _create, make sure it doesn't get called from
BaseImage.create()
"""
if not image_rootfs:
return
fstab_path = image_rootfs + "/etc/fstab"
if not os.path.isfile(fstab_path):
return
with open(fstab_path) as fstab:
fstab_lines = fstab.readlines()
if self._update_fstab(fstab_lines, self.parts):
# copy rootfs dir to workdir to update fstab
# as rootfs can be used by other tasks and can't be modified
new_rootfs = os.path.realpath(os.path.join(self.workdir, "rootfs_copy"))
copyhardlinktree(image_rootfs, new_rootfs)
fstab_path = os.path.join(new_rootfs, 'etc/fstab')
os.unlink(fstab_path)
with open(fstab_path, "w") as fstab:
fstab.writelines(fstab_lines)
return new_rootfs
def _update_fstab(self, fstab_lines, parts):
"""Assume partition order same as in wks"""
updated = False
for part in parts:
if not part.realnum or not part.mountpoint \
or part.mountpoint == "/":
continue
if part.use_uuid:
device_name = "PARTUUID=%s" % part.uuid
else:
# mmc device partitions are named mmcblk0p1, mmcblk0p2..
prefix = 'p' if part.disk.startswith('mmcblk') else ''
device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
opts = part.fsopts if part.fsopts else "defaults"
line = "\t".join([device_name, part.mountpoint, part.fstype,
opts, "0", "0"]) + "\n"
fstab_lines.append(line)
updated = True
return updated
def _full_path(self, path, name, extention):
""" Construct full file path to a file we generate. """
return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
#
# Actual implemention
#
def create(self):
"""
For 'wic', we already have our build artifacts - we just create
filesystems from the artifacts directly and combine them into
a partitioned image.
"""
if self.no_fstab_update:
new_rootfs = None
else:
new_rootfs = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
if new_rootfs:
# rootfs was copied to update fstab
self.rootfs_dir['ROOTFS_DIR'] = new_rootfs
for part in self.parts:
# get rootfs size from bitbake variable if it's not set in .ks file
if not part.size:
# and if rootfs name is specified for the partition
image_name = self.rootfs_dir.get(part.rootfs_dir)
if image_name and os.path.sep not in image_name:
# Bitbake variable ROOTFS_SIZE is calculated in
# Image._get_rootfs_size method from meta/lib/oe/image.py
# using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
# IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
if rsize_bb:
part.size = int(round(float(rsize_bb)))
self._image.prepare(self)
self._image.layout_partitions()
self._image.create()
def assemble(self):
"""
Assemble partitions into disk image
"""
self._image.assemble()
def finalize(self):
"""
Finalize the disk image.
For example, prepare the image to be bootable by e.g.
creating and installing a bootloader configuration.
"""
source_plugin = self.ks.bootloader.source
disk_name = self.parts[0].disk
if source_plugin:
plugin = PluginMgr.get_plugins('source')[source_plugin]
plugin.do_install_disk(self._image, disk_name, self, self.workdir,
self.oe_builddir, self.bootimg_dir,
self.kernel_dir, self.native_sysroot)
full_path = self._image.path
# Generate .bmap
if self.bmap:
logger.debug("Generating bmap file for %s", disk_name)
python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3')
bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool')
exec_native_cmd("%s %s create %s -o %s.bmap" % \
(python, bmaptool, full_path, full_path), self.native_sysroot)
# Compress the image
if self.compressor:
logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
exec_cmd("%s %s" % (self.compressor, full_path))
def print_info(self):
"""
Print the image(s) and artifacts used, for the user.
"""
msg = "The new image(s) can be found here:\n"
extension = "direct" + {"gzip": ".gz",
"bzip2": ".bz2",
"xz": ".xz",
None: ""}.get(self.compressor)
full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
msg += ' %s\n\n' % full_path
msg += 'The following build artifacts were used to create the image(s):\n'
for part in self.parts:
if part.rootfs_dir is None:
continue
if part.mountpoint == '/':
suffix = ':'
else:
suffix = '["%s"]:' % (part.mountpoint or part.label)
msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir)
msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
logger.info(msg)
@property
def rootdev(self):
"""
Get root device name to use as a 'root' parameter
in kernel command line.
Assume partition order same as in wks
"""
for part in self.parts:
if part.mountpoint == "/":
if part.uuid:
return "PARTUUID=%s" % part.uuid
else:
suffix = 'p' if part.disk.startswith('mmcblk') else ''
return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
def cleanup(self):
if self._image:
self._image.cleanup()
# Move results to the output dir
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
for fname in os.listdir(self.workdir):
path = os.path.join(self.workdir, fname)
if os.path.isfile(path):
shutil.move(path, os.path.join(self.outdir, fname))
# remove work directory
shutil.rmtree(self.workdir, ignore_errors=True)
# Overhead of the MBR partitioning scheme (just one sector)
MBR_OVERHEAD = 1
# Overhead of the GPT partitioning scheme
GPT_OVERHEAD = 34
# Size of a sector in bytes
SECTOR_SIZE = 512
class PartitionedImage():
"""
Partitioned image in a file.
"""
def __init__(self, path, ptable_format, partitions, native_sysroot=None):
self.path = path # Path to the image file
self.numpart = 0 # Number of allocated partitions
self.realpart = 0 # Number of partitions in the partition table
self.offset = 0 # Offset of next partition (in sectors)
self.min_size = 0 # Minimum required disk size to fit
# all partitions (in bytes)
self.ptable_format = ptable_format # Partition table format
# Disk system identifier
self.identifier = random.SystemRandom().randint(1, 0xffffffff)
self.partitions = partitions
self.partimages = []
# Size of a sector used in calculations
self.sector_size = SECTOR_SIZE
self.native_sysroot = native_sysroot
# calculate the real partition number, accounting for partitions not
# in the partition table and logical partitions
realnum = 0
for part in self.partitions:
if part.no_table:
part.realnum = 0
else:
realnum += 1
if self.ptable_format == 'msdos' and realnum > 3 and len(partitions) > 4:
part.realnum = realnum + 1
continue
part.realnum = realnum
# generate parition UUIDs
for part in self.partitions:
if not part.uuid and part.use_uuid:
if self.ptable_format == 'gpt':
part.uuid = str(uuid.uuid4())
else: # msdos partition table
part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
def prepare(self, imager):
"""Prepare an image. Call prepare method of all image partitions."""
for part in self.partitions:
# need to create the filesystems in order to get their
# sizes before we can add them and do the layout.
part.prepare(imager, imager.workdir, imager.oe_builddir,
imager.rootfs_dir, imager.bootimg_dir,
imager.kernel_dir, imager.native_sysroot)
# Converting kB to sectors for parted
part.size_sec = part.disk_size * 1024 // self.sector_size
def layout_partitions(self):
""" Layout the partitions, meaning calculate the position of every
partition on the disk. The 'ptable_format' parameter defines the
partition table format and may be "msdos". """
logger.debug("Assigning %s partitions to disks", self.ptable_format)
# The number of primary and logical partitions. Extended partition and
# partitions not listed in the table are not included.
num_real_partitions = len([p for p in self.partitions if not p.no_table])
# Go through partitions in the order they are added in .ks file
for num in range(len(self.partitions)):
part = self.partitions[num]
if self.ptable_format == 'msdos' and part.part_name:
raise WicError("setting custom partition name is not " \
"implemented for msdos partitions")
if self.ptable_format == 'msdos' and part.part_type:
# The --part-type can also be implemented for MBR partitions,
# in which case it would map to the 1-byte "partition type"
# filed at offset 3 of the partition entry.
raise WicError("setting custom partition type is not " \
"implemented for msdos partitions")
# Get the disk where the partition is located
self.numpart += 1
if not part.no_table:
self.realpart += 1
if self.numpart == 1:
if self.ptable_format == "msdos":
overhead = MBR_OVERHEAD
elif self.ptable_format == "gpt":
overhead = GPT_OVERHEAD
# Skip one sector required for the partitioning scheme overhead
self.offset += overhead
if self.realpart > 3 and num_real_partitions > 4:
# Reserve a sector for EBR for every logical partition
# before alignment is performed.
if self.ptable_format == "msdos":
self.offset += 1
if part.align:
# If not first partition and we do have alignment set we need
# to align the partition.
# FIXME: This leaves a empty spaces to the disk. To fill the
# gaps we could enlargea the previous partition?
# Calc how much the alignment is off.
align_sectors = self.offset % (part.align * 1024 // self.sector_size)
if align_sectors:
# If partition is not aligned as required, we need
# to move forward to the next alignment point
align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
logger.debug("Realignment for %s%s with %s sectors, original"
" offset %s, target alignment is %sK.",
part.disk, self.numpart, align_sectors,
self.offset, part.align)
# increase the offset so we actually start the partition on right alignment
self.offset += align_sectors
part.start = self.offset
self.offset += part.size_sec
part.type = 'primary'
if not part.no_table:
part.num = self.realpart
else:
part.num = 0
if self.ptable_format == "msdos":
# only count the partitions that are in partition table
if num_real_partitions > 4:
if self.realpart > 3:
part.type = 'logical'
part.num = self.realpart + 1
logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
"sectors (%d bytes).", part.mountpoint, part.disk,
part.num, part.start, self.offset - 1, part.size_sec,
part.size_sec * self.sector_size)
# Once all the partitions have been layed out, we can calculate the
# minumim disk size
self.min_size = self.offset
if self.ptable_format == "gpt":
self.min_size += GPT_OVERHEAD
self.min_size *= self.sector_size
def _create_partition(self, device, parttype, fstype, start, size):
""" Create a partition on an image described by the 'device' object. """
# Start is included to the size so we need to substract one from the end.
end = start + size - 1
logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
parttype, start, end, size)
cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
if fstype:
cmd += " %s" % fstype
cmd += " %d %d" % (start, end)
return exec_native_cmd(cmd, self.native_sysroot)
def create(self):
logger.debug("Creating sparse file %s", self.path)
with open(self.path, 'w') as sparse:
os.ftruncate(sparse.fileno(), self.min_size)
logger.debug("Initializing partition table for %s", self.path)
exec_native_cmd("parted -s %s mklabel %s" %
(self.path, self.ptable_format), self.native_sysroot)
logger.debug("Set disk identifier %x", self.identifier)
with open(self.path, 'r+b') as img:
img.seek(0x1B8)
img.write(self.identifier.to_bytes(4, 'little'))
logger.debug("Creating partitions")
for part in self.partitions:
if part.num == 0:
continue
if self.ptable_format == "msdos" and part.num == 5:
# Create an extended partition (note: extended
# partition is described in MBR and contains all
# logical partitions). The logical partitions save a
# sector for an EBR just before the start of a
# partition. The extended partition must start one
# sector before the start of the first logical
# partition. This way the first EBR is inside of the
# extended partition. Since the extended partitions
# starts a sector before the first logical partition,
# add a sector at the back, so that there is enough
# room for all logical partitions.
self._create_partition(self.path, "extended",
None, part.start - 1,
self.offset - part.start + 1)
if part.fstype == "swap":
parted_fs_type = "linux-swap"
elif part.fstype == "vfat":
parted_fs_type = "fat32"
elif part.fstype == "msdos":
parted_fs_type = "fat16"
if not part.system_id:
part.system_id = '0x6' # FAT16
else:
# Type for ext2/ext3/ext4/btrfs
parted_fs_type = "ext2"
# Boot ROM of OMAP boards require vfat boot partition to have an
# even number of sectors.
if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
and part.size_sec % 2:
logger.debug("Subtracting one sector from '%s' partition to "
"get even number of sectors for the partition",
part.mountpoint)
part.size_sec -= 1
self._create_partition(self.path, part.type,
parted_fs_type, part.start, part.size_sec)
if part.part_name:
logger.debug("partition %d: set name to %s",
part.num, part.part_name)
exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
(part.num, part.part_name,
self.path), self.native_sysroot)
if part.part_type:
logger.debug("partition %d: set type UID to %s",
part.num, part.part_type)
exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
(part.num, part.part_type,
self.path), self.native_sysroot)
if part.uuid and self.ptable_format == "gpt":
logger.debug("partition %d: set UUID to %s",
part.num, part.uuid)
exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
(part.num, part.uuid, self.path),
self.native_sysroot)
if part.label and self.ptable_format == "gpt":
logger.debug("partition %d: set name to %s",
part.num, part.label)
exec_native_cmd("parted -s %s name %d %s" % \
(self.path, part.num, part.label),
self.native_sysroot)
if part.active:
flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
flag_name, part.num, self.path)
exec_native_cmd("parted -s %s set %d %s on" % \
(self.path, part.num, flag_name),
self.native_sysroot)
if part.system_id:
exec_native_cmd("sfdisk --part-type %s %s %s" % \
(self.path, part.num, part.system_id),
self.native_sysroot)
def cleanup(self):
# remove partition images
for image in set(self.partimages):
os.remove(image)
def assemble(self):
logger.debug("Installing partitions")
for part in self.partitions:
source = part.source_file
if source:
# install source_file contents into a partition
sparse_copy(source, self.path, seek=part.start * self.sector_size)
logger.debug("Installed %s in partition %d, sectors %d-%d, "
"size %d sectors", source, part.num, part.start,
part.start + part.size_sec - 1, part.size_sec)
partimage = self.path + '.p%d' % part.num
os.rename(source, partimage)
self.partimages.append(partimage)