590 lines
23 KiB
Python
590 lines
23 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
#
|
|
# Copyright (c) 2013, Intel Corporation.
|
|
# All rights reserved.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
#
|
|
# DESCRIPTION
|
|
# This implements the 'direct' imager plugin class for 'wic'
|
|
#
|
|
# AUTHORS
|
|
# Tom Zanussi <tom.zanussi (at] linux.intel.com>
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import random
|
|
import shutil
|
|
import tempfile
|
|
import uuid
|
|
|
|
from time import strftime
|
|
|
|
from oe.path import copyhardlinktree
|
|
|
|
from wic import WicError
|
|
from wic.filemap import sparse_copy
|
|
from wic.ksparser import KickStart, KickStartError
|
|
from wic.pluginbase import PluginMgr, ImagerPlugin
|
|
from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd
|
|
|
|
logger = logging.getLogger('wic')
|
|
|
|
class DirectPlugin(ImagerPlugin):
|
|
"""
|
|
Install a system into a file containing a partitioned disk image.
|
|
|
|
An image file is formatted with a partition table, each partition
|
|
created from a rootfs or other OpenEmbedded build artifact and dd'ed
|
|
into the virtual disk. The disk image can subsequently be dd'ed onto
|
|
media and used on actual hardware.
|
|
"""
|
|
name = 'direct'
|
|
|
|
def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
|
|
native_sysroot, oe_builddir, options):
|
|
try:
|
|
self.ks = KickStart(wks_file)
|
|
except KickStartError as err:
|
|
raise WicError(str(err))
|
|
|
|
# parse possible 'rootfs=name' items
|
|
self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
|
|
self.bootimg_dir = bootimg_dir
|
|
self.kernel_dir = kernel_dir
|
|
self.native_sysroot = native_sysroot
|
|
self.oe_builddir = oe_builddir
|
|
|
|
self.outdir = options.outdir
|
|
self.compressor = options.compressor
|
|
self.bmap = options.bmap
|
|
self.no_fstab_update = options.no_fstab_update
|
|
|
|
self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
|
|
strftime("%Y%m%d%H%M"))
|
|
self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
|
|
self._image = None
|
|
self.ptable_format = self.ks.bootloader.ptable
|
|
self.parts = self.ks.partitions
|
|
|
|
# as a convenience, set source to the boot partition source
|
|
# instead of forcing it to be set via bootloader --source
|
|
for part in self.parts:
|
|
if not self.ks.bootloader.source and part.mountpoint == "/boot":
|
|
self.ks.bootloader.source = part.source
|
|
break
|
|
|
|
image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
|
|
self._image = PartitionedImage(image_path, self.ptable_format,
|
|
self.parts, self.native_sysroot)
|
|
|
|
def do_create(self):
|
|
"""
|
|
Plugin entry point.
|
|
"""
|
|
try:
|
|
self.create()
|
|
self.assemble()
|
|
self.finalize()
|
|
self.print_info()
|
|
finally:
|
|
self.cleanup()
|
|
|
|
def _write_fstab(self, image_rootfs):
|
|
"""overriden to generate fstab (temporarily) in rootfs. This is called
|
|
from _create, make sure it doesn't get called from
|
|
BaseImage.create()
|
|
"""
|
|
if not image_rootfs:
|
|
return
|
|
|
|
fstab_path = image_rootfs + "/etc/fstab"
|
|
if not os.path.isfile(fstab_path):
|
|
return
|
|
|
|
with open(fstab_path) as fstab:
|
|
fstab_lines = fstab.readlines()
|
|
|
|
if self._update_fstab(fstab_lines, self.parts):
|
|
# copy rootfs dir to workdir to update fstab
|
|
# as rootfs can be used by other tasks and can't be modified
|
|
new_rootfs = os.path.realpath(os.path.join(self.workdir, "rootfs_copy"))
|
|
copyhardlinktree(image_rootfs, new_rootfs)
|
|
fstab_path = os.path.join(new_rootfs, 'etc/fstab')
|
|
|
|
os.unlink(fstab_path)
|
|
|
|
with open(fstab_path, "w") as fstab:
|
|
fstab.writelines(fstab_lines)
|
|
|
|
return new_rootfs
|
|
|
|
def _update_fstab(self, fstab_lines, parts):
|
|
"""Assume partition order same as in wks"""
|
|
updated = False
|
|
for part in parts:
|
|
if not part.realnum or not part.mountpoint \
|
|
or part.mountpoint == "/":
|
|
continue
|
|
|
|
if part.use_uuid:
|
|
device_name = "PARTUUID=%s" % part.uuid
|
|
else:
|
|
# mmc device partitions are named mmcblk0p1, mmcblk0p2..
|
|
prefix = 'p' if part.disk.startswith('mmcblk') else ''
|
|
device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
|
|
|
|
opts = part.fsopts if part.fsopts else "defaults"
|
|
line = "\t".join([device_name, part.mountpoint, part.fstype,
|
|
opts, "0", "0"]) + "\n"
|
|
|
|
fstab_lines.append(line)
|
|
updated = True
|
|
|
|
return updated
|
|
|
|
def _full_path(self, path, name, extention):
|
|
""" Construct full file path to a file we generate. """
|
|
return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
|
|
|
|
#
|
|
# Actual implemention
|
|
#
|
|
def create(self):
|
|
"""
|
|
For 'wic', we already have our build artifacts - we just create
|
|
filesystems from the artifacts directly and combine them into
|
|
a partitioned image.
|
|
"""
|
|
if self.no_fstab_update:
|
|
new_rootfs = None
|
|
else:
|
|
new_rootfs = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
|
|
if new_rootfs:
|
|
# rootfs was copied to update fstab
|
|
self.rootfs_dir['ROOTFS_DIR'] = new_rootfs
|
|
|
|
for part in self.parts:
|
|
# get rootfs size from bitbake variable if it's not set in .ks file
|
|
if not part.size:
|
|
# and if rootfs name is specified for the partition
|
|
image_name = self.rootfs_dir.get(part.rootfs_dir)
|
|
if image_name and os.path.sep not in image_name:
|
|
# Bitbake variable ROOTFS_SIZE is calculated in
|
|
# Image._get_rootfs_size method from meta/lib/oe/image.py
|
|
# using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
|
|
# IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
|
|
rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
|
|
if rsize_bb:
|
|
part.size = int(round(float(rsize_bb)))
|
|
|
|
self._image.prepare(self)
|
|
self._image.layout_partitions()
|
|
self._image.create()
|
|
|
|
def assemble(self):
|
|
"""
|
|
Assemble partitions into disk image
|
|
"""
|
|
self._image.assemble()
|
|
|
|
def finalize(self):
|
|
"""
|
|
Finalize the disk image.
|
|
|
|
For example, prepare the image to be bootable by e.g.
|
|
creating and installing a bootloader configuration.
|
|
"""
|
|
source_plugin = self.ks.bootloader.source
|
|
disk_name = self.parts[0].disk
|
|
if source_plugin:
|
|
plugin = PluginMgr.get_plugins('source')[source_plugin]
|
|
plugin.do_install_disk(self._image, disk_name, self, self.workdir,
|
|
self.oe_builddir, self.bootimg_dir,
|
|
self.kernel_dir, self.native_sysroot)
|
|
|
|
full_path = self._image.path
|
|
# Generate .bmap
|
|
if self.bmap:
|
|
logger.debug("Generating bmap file for %s", disk_name)
|
|
python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3')
|
|
bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool')
|
|
exec_native_cmd("%s %s create %s -o %s.bmap" % \
|
|
(python, bmaptool, full_path, full_path), self.native_sysroot)
|
|
# Compress the image
|
|
if self.compressor:
|
|
logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
|
|
exec_cmd("%s %s" % (self.compressor, full_path))
|
|
|
|
def print_info(self):
|
|
"""
|
|
Print the image(s) and artifacts used, for the user.
|
|
"""
|
|
msg = "The new image(s) can be found here:\n"
|
|
|
|
extension = "direct" + {"gzip": ".gz",
|
|
"bzip2": ".bz2",
|
|
"xz": ".xz",
|
|
None: ""}.get(self.compressor)
|
|
full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
|
|
msg += ' %s\n\n' % full_path
|
|
|
|
msg += 'The following build artifacts were used to create the image(s):\n'
|
|
for part in self.parts:
|
|
if part.rootfs_dir is None:
|
|
continue
|
|
if part.mountpoint == '/':
|
|
suffix = ':'
|
|
else:
|
|
suffix = '["%s"]:' % (part.mountpoint or part.label)
|
|
msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir)
|
|
|
|
msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
|
|
msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
|
|
msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
|
|
|
|
logger.info(msg)
|
|
|
|
@property
|
|
def rootdev(self):
|
|
"""
|
|
Get root device name to use as a 'root' parameter
|
|
in kernel command line.
|
|
|
|
Assume partition order same as in wks
|
|
"""
|
|
for part in self.parts:
|
|
if part.mountpoint == "/":
|
|
if part.uuid:
|
|
return "PARTUUID=%s" % part.uuid
|
|
else:
|
|
suffix = 'p' if part.disk.startswith('mmcblk') else ''
|
|
return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
|
|
|
|
def cleanup(self):
|
|
if self._image:
|
|
self._image.cleanup()
|
|
|
|
# Move results to the output dir
|
|
if not os.path.exists(self.outdir):
|
|
os.makedirs(self.outdir)
|
|
|
|
for fname in os.listdir(self.workdir):
|
|
path = os.path.join(self.workdir, fname)
|
|
if os.path.isfile(path):
|
|
shutil.move(path, os.path.join(self.outdir, fname))
|
|
|
|
# remove work directory
|
|
shutil.rmtree(self.workdir, ignore_errors=True)
|
|
|
|
# Overhead of the MBR partitioning scheme (just one sector)
|
|
MBR_OVERHEAD = 1
|
|
|
|
# Overhead of the GPT partitioning scheme
|
|
GPT_OVERHEAD = 34
|
|
|
|
# Size of a sector in bytes
|
|
SECTOR_SIZE = 512
|
|
|
|
class PartitionedImage():
|
|
"""
|
|
Partitioned image in a file.
|
|
"""
|
|
|
|
def __init__(self, path, ptable_format, partitions, native_sysroot=None):
|
|
self.path = path # Path to the image file
|
|
self.numpart = 0 # Number of allocated partitions
|
|
self.realpart = 0 # Number of partitions in the partition table
|
|
self.offset = 0 # Offset of next partition (in sectors)
|
|
self.min_size = 0 # Minimum required disk size to fit
|
|
# all partitions (in bytes)
|
|
self.ptable_format = ptable_format # Partition table format
|
|
# Disk system identifier
|
|
self.identifier = random.SystemRandom().randint(1, 0xffffffff)
|
|
|
|
self.partitions = partitions
|
|
self.partimages = []
|
|
# Size of a sector used in calculations
|
|
self.sector_size = SECTOR_SIZE
|
|
self.native_sysroot = native_sysroot
|
|
|
|
# calculate the real partition number, accounting for partitions not
|
|
# in the partition table and logical partitions
|
|
realnum = 0
|
|
for part in self.partitions:
|
|
if part.no_table:
|
|
part.realnum = 0
|
|
else:
|
|
realnum += 1
|
|
if self.ptable_format == 'msdos' and realnum > 3 and len(partitions) > 4:
|
|
part.realnum = realnum + 1
|
|
continue
|
|
part.realnum = realnum
|
|
|
|
# generate parition UUIDs
|
|
for part in self.partitions:
|
|
if not part.uuid and part.use_uuid:
|
|
if self.ptable_format == 'gpt':
|
|
part.uuid = str(uuid.uuid4())
|
|
else: # msdos partition table
|
|
part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
|
|
|
|
def prepare(self, imager):
|
|
"""Prepare an image. Call prepare method of all image partitions."""
|
|
for part in self.partitions:
|
|
# need to create the filesystems in order to get their
|
|
# sizes before we can add them and do the layout.
|
|
part.prepare(imager, imager.workdir, imager.oe_builddir,
|
|
imager.rootfs_dir, imager.bootimg_dir,
|
|
imager.kernel_dir, imager.native_sysroot)
|
|
|
|
# Converting kB to sectors for parted
|
|
part.size_sec = part.disk_size * 1024 // self.sector_size
|
|
|
|
def layout_partitions(self):
|
|
""" Layout the partitions, meaning calculate the position of every
|
|
partition on the disk. The 'ptable_format' parameter defines the
|
|
partition table format and may be "msdos". """
|
|
|
|
logger.debug("Assigning %s partitions to disks", self.ptable_format)
|
|
|
|
# The number of primary and logical partitions. Extended partition and
|
|
# partitions not listed in the table are not included.
|
|
num_real_partitions = len([p for p in self.partitions if not p.no_table])
|
|
|
|
# Go through partitions in the order they are added in .ks file
|
|
for num in range(len(self.partitions)):
|
|
part = self.partitions[num]
|
|
|
|
if self.ptable_format == 'msdos' and part.part_name:
|
|
raise WicError("setting custom partition name is not " \
|
|
"implemented for msdos partitions")
|
|
|
|
if self.ptable_format == 'msdos' and part.part_type:
|
|
# The --part-type can also be implemented for MBR partitions,
|
|
# in which case it would map to the 1-byte "partition type"
|
|
# filed at offset 3 of the partition entry.
|
|
raise WicError("setting custom partition type is not " \
|
|
"implemented for msdos partitions")
|
|
|
|
# Get the disk where the partition is located
|
|
self.numpart += 1
|
|
if not part.no_table:
|
|
self.realpart += 1
|
|
|
|
if self.numpart == 1:
|
|
if self.ptable_format == "msdos":
|
|
overhead = MBR_OVERHEAD
|
|
elif self.ptable_format == "gpt":
|
|
overhead = GPT_OVERHEAD
|
|
|
|
# Skip one sector required for the partitioning scheme overhead
|
|
self.offset += overhead
|
|
|
|
if self.realpart > 3 and num_real_partitions > 4:
|
|
# Reserve a sector for EBR for every logical partition
|
|
# before alignment is performed.
|
|
if self.ptable_format == "msdos":
|
|
self.offset += 1
|
|
|
|
if part.align:
|
|
# If not first partition and we do have alignment set we need
|
|
# to align the partition.
|
|
# FIXME: This leaves a empty spaces to the disk. To fill the
|
|
# gaps we could enlargea the previous partition?
|
|
|
|
# Calc how much the alignment is off.
|
|
align_sectors = self.offset % (part.align * 1024 // self.sector_size)
|
|
|
|
if align_sectors:
|
|
# If partition is not aligned as required, we need
|
|
# to move forward to the next alignment point
|
|
align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
|
|
|
|
logger.debug("Realignment for %s%s with %s sectors, original"
|
|
" offset %s, target alignment is %sK.",
|
|
part.disk, self.numpart, align_sectors,
|
|
self.offset, part.align)
|
|
|
|
# increase the offset so we actually start the partition on right alignment
|
|
self.offset += align_sectors
|
|
|
|
part.start = self.offset
|
|
self.offset += part.size_sec
|
|
|
|
part.type = 'primary'
|
|
if not part.no_table:
|
|
part.num = self.realpart
|
|
else:
|
|
part.num = 0
|
|
|
|
if self.ptable_format == "msdos":
|
|
# only count the partitions that are in partition table
|
|
if num_real_partitions > 4:
|
|
if self.realpart > 3:
|
|
part.type = 'logical'
|
|
part.num = self.realpart + 1
|
|
|
|
logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
|
|
"sectors (%d bytes).", part.mountpoint, part.disk,
|
|
part.num, part.start, self.offset - 1, part.size_sec,
|
|
part.size_sec * self.sector_size)
|
|
|
|
# Once all the partitions have been layed out, we can calculate the
|
|
# minumim disk size
|
|
self.min_size = self.offset
|
|
if self.ptable_format == "gpt":
|
|
self.min_size += GPT_OVERHEAD
|
|
|
|
self.min_size *= self.sector_size
|
|
|
|
def _create_partition(self, device, parttype, fstype, start, size):
|
|
""" Create a partition on an image described by the 'device' object. """
|
|
|
|
# Start is included to the size so we need to substract one from the end.
|
|
end = start + size - 1
|
|
logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
|
|
parttype, start, end, size)
|
|
|
|
cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
|
|
if fstype:
|
|
cmd += " %s" % fstype
|
|
cmd += " %d %d" % (start, end)
|
|
|
|
return exec_native_cmd(cmd, self.native_sysroot)
|
|
|
|
def create(self):
|
|
logger.debug("Creating sparse file %s", self.path)
|
|
with open(self.path, 'w') as sparse:
|
|
os.ftruncate(sparse.fileno(), self.min_size)
|
|
|
|
logger.debug("Initializing partition table for %s", self.path)
|
|
exec_native_cmd("parted -s %s mklabel %s" %
|
|
(self.path, self.ptable_format), self.native_sysroot)
|
|
|
|
logger.debug("Set disk identifier %x", self.identifier)
|
|
with open(self.path, 'r+b') as img:
|
|
img.seek(0x1B8)
|
|
img.write(self.identifier.to_bytes(4, 'little'))
|
|
|
|
logger.debug("Creating partitions")
|
|
|
|
for part in self.partitions:
|
|
if part.num == 0:
|
|
continue
|
|
|
|
if self.ptable_format == "msdos" and part.num == 5:
|
|
# Create an extended partition (note: extended
|
|
# partition is described in MBR and contains all
|
|
# logical partitions). The logical partitions save a
|
|
# sector for an EBR just before the start of a
|
|
# partition. The extended partition must start one
|
|
# sector before the start of the first logical
|
|
# partition. This way the first EBR is inside of the
|
|
# extended partition. Since the extended partitions
|
|
# starts a sector before the first logical partition,
|
|
# add a sector at the back, so that there is enough
|
|
# room for all logical partitions.
|
|
self._create_partition(self.path, "extended",
|
|
None, part.start - 1,
|
|
self.offset - part.start + 1)
|
|
|
|
if part.fstype == "swap":
|
|
parted_fs_type = "linux-swap"
|
|
elif part.fstype == "vfat":
|
|
parted_fs_type = "fat32"
|
|
elif part.fstype == "msdos":
|
|
parted_fs_type = "fat16"
|
|
if not part.system_id:
|
|
part.system_id = '0x6' # FAT16
|
|
else:
|
|
# Type for ext2/ext3/ext4/btrfs
|
|
parted_fs_type = "ext2"
|
|
|
|
# Boot ROM of OMAP boards require vfat boot partition to have an
|
|
# even number of sectors.
|
|
if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
|
|
and part.size_sec % 2:
|
|
logger.debug("Subtracting one sector from '%s' partition to "
|
|
"get even number of sectors for the partition",
|
|
part.mountpoint)
|
|
part.size_sec -= 1
|
|
|
|
self._create_partition(self.path, part.type,
|
|
parted_fs_type, part.start, part.size_sec)
|
|
|
|
if part.part_name:
|
|
logger.debug("partition %d: set name to %s",
|
|
part.num, part.part_name)
|
|
exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
|
|
(part.num, part.part_name,
|
|
self.path), self.native_sysroot)
|
|
|
|
if part.part_type:
|
|
logger.debug("partition %d: set type UID to %s",
|
|
part.num, part.part_type)
|
|
exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
|
|
(part.num, part.part_type,
|
|
self.path), self.native_sysroot)
|
|
|
|
if part.uuid and self.ptable_format == "gpt":
|
|
logger.debug("partition %d: set UUID to %s",
|
|
part.num, part.uuid)
|
|
exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
|
|
(part.num, part.uuid, self.path),
|
|
self.native_sysroot)
|
|
|
|
if part.label and self.ptable_format == "gpt":
|
|
logger.debug("partition %d: set name to %s",
|
|
part.num, part.label)
|
|
exec_native_cmd("parted -s %s name %d %s" % \
|
|
(self.path, part.num, part.label),
|
|
self.native_sysroot)
|
|
|
|
if part.active:
|
|
flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
|
|
logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
|
|
flag_name, part.num, self.path)
|
|
exec_native_cmd("parted -s %s set %d %s on" % \
|
|
(self.path, part.num, flag_name),
|
|
self.native_sysroot)
|
|
if part.system_id:
|
|
exec_native_cmd("sfdisk --part-type %s %s %s" % \
|
|
(self.path, part.num, part.system_id),
|
|
self.native_sysroot)
|
|
|
|
def cleanup(self):
|
|
# remove partition images
|
|
for image in set(self.partimages):
|
|
os.remove(image)
|
|
|
|
def assemble(self):
|
|
logger.debug("Installing partitions")
|
|
|
|
for part in self.partitions:
|
|
source = part.source_file
|
|
if source:
|
|
# install source_file contents into a partition
|
|
sparse_copy(source, self.path, seek=part.start * self.sector_size)
|
|
|
|
logger.debug("Installed %s in partition %d, sectors %d-%d, "
|
|
"size %d sectors", source, part.num, part.start,
|
|
part.start + part.size_sec - 1, part.size_sec)
|
|
|
|
partimage = self.path + '.p%d' % part.num
|
|
os.rename(source, partimage)
|
|
self.partimages.append(partimage)
|