From 1a587bb6ccf53ed87445abe9567c5d5914b3a249 Mon Sep 17 00:00:00 2001 From: Bruce Leidl Date: Thu, 12 Sep 2019 16:31:38 -0400 Subject: [PATCH] refactor configure and launch of VM to be much simpler and nicer --- rust/src/main.rs | 54 ++-------- rust/src/memory/manager.rs | 4 + rust/src/vm/config.rs | 185 ++++++++++++++++++++++++++++++++++ rust/src/vm/error.rs | 11 +- rust/src/vm/kernel_cmdline.rs | 26 +---- rust/src/vm/mod.rs | 172 +++++++++++++++++++------------ rust/src/vm/run.rs | 18 +++- 7 files changed, 332 insertions(+), 138 deletions(-) create mode 100644 rust/src/vm/config.rs diff --git a/rust/src/main.rs b/rust/src/main.rs index 63914a6..b9f9f83 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,10 +1,8 @@ #![allow(non_snake_case)] -extern crate libc; -extern crate byteorder; -extern crate termios; - +#[macro_use] extern crate lazy_static; +#[macro_use] mod log; mod vm; mod memory; #[macro_use] @@ -12,48 +10,14 @@ mod system; mod devices; mod kvm; mod virtio; +mod disk; - +pub use log::{Logger,LogLevel}; use std::env; -use std::path::PathBuf; + fn main() { - - let mut config = vm::VmConfig::new(); - config.ram_size_megs(1024); - match find_kernel() { - Some(path) => config.kernel_path(&path), - None => { println!("Could not find kernel"); return; } - } - match find_init() { - Some(path) => config.init_path(&path), - None => { println!("Could not find init"); return; } - } - match vm::Vm::open(config) { - Ok(vm) => { - vm.start().unwrap(); - }, - Err(e) => println!("error :( {}", e) - } -} - -fn find_init() -> Option { - let mut cwd = env::current_dir().unwrap(); - if cwd.join("rust/target/release/ph-init").exists() { - cwd.push("rust/target/release/ph-init"); - return Some(cwd) - } - if cwd.join("rust/target/debug/ph-init").exists() { - cwd.push("rust/target/debug/ph-init"); - return Some(cwd) - } - None -} - -fn find_kernel() -> Option { - let mut cwd = env::current_dir().unwrap(); - if cwd.join("kernel/ph_linux").exists() { - cwd.push("kernel/ph_linux"); - return Some(cwd) - } - None + vm::VmConfig::new(env::args()) + .ram_size_megs(1024) + .use_realmfs("/home/user/Shared/main-realmfs.img") + .boot(); } diff --git a/rust/src/memory/manager.rs b/rust/src/memory/manager.rs index 9797c74..40cca56 100644 --- a/rust/src/memory/manager.rs +++ b/rust/src/memory/manager.rs @@ -26,6 +26,10 @@ impl MemoryManager { &self.ram } + pub fn kvm_mut(&mut self) -> &mut Kvm { + &mut self.kvm + } + pub fn kvm(&self) -> &Kvm { &self.kvm } diff --git a/rust/src/vm/config.rs b/rust/src/vm/config.rs new file mode 100644 index 0000000..2b2d7ea --- /dev/null +++ b/rust/src/vm/config.rs @@ -0,0 +1,185 @@ +use std::path::{PathBuf, Path}; +use crate::vm::{Vm, Result, ErrorKind}; +use std::{env, process}; + +pub enum RootFS { + SelfRoot, + RealmFSImage(PathBuf), + RawImage(PathBuf), + RawOffset(PathBuf, usize), +} + +pub struct VmConfig { + ram_size: usize, + ncpus: usize, + verbose: bool, + launch_systemd: bool, + kernel_path: Option, + init_path: Option, + init_cmd: Option, + rootfs: RootFS, +} + +#[allow(dead_code)] +impl VmConfig { + pub fn new(args: env::Args) -> VmConfig { + let mut config = VmConfig { + ram_size: 256 * 1024 * 1024, + ncpus: 1, + verbose: false, + launch_systemd: false, + kernel_path: None, + init_path: None, + init_cmd: None, + rootfs: RootFS::SelfRoot, + }; + config.parse_args(args); + config + } + + pub fn ram_size_megs(mut self, megs: usize) -> Self { + self.ram_size = megs * 1024 * 1024; + self + } + + pub fn num_cpus(mut self, ncpus: usize) -> Self { + self.ncpus = ncpus; + self + } + + pub fn init_cmdline(mut self, val: &str) -> Self { + self.init_cmd = Some(val.to_owned()); + self + } + + pub fn kernel_path>(mut self, path: P) -> Self { + self.kernel_path = Some(path.into()); + self + } + + pub fn init_path>(mut self, path: P) -> Self { + self.init_path = Some(path.into()); + self + } + + pub fn use_realmfs>(mut self, path: P) -> Self { + self.rootfs = RootFS::RealmFSImage(path.into()); + self + } + + pub fn use_rawdisk>(mut self, path: P) -> Self { + self.rootfs = RootFS::RawImage(path.into()); + self + } + + pub fn use_rawdisk_with_offset>(mut self, path: P, offset: usize) -> Self { + self.rootfs = RootFS::RawOffset(path.into(), offset); + self + } + + pub fn use_systemd(mut self) -> Self { + self.launch_systemd = true; + self + } + + pub fn boot(self) { + match Vm::open(self) { + Ok(vm) => if let Err(err) = vm.start() { + notify!("Error starting VM: {}", err); + } + Err(e) => notify!("Error creating VM: {}", e), + } + } + + pub fn ram_size(&self) -> usize { + self.ram_size + } + + pub fn ncpus(&self) -> usize { + self.ncpus + } + + pub fn verbose(&self) -> bool { + self.verbose + } + + pub fn launch_systemd(&self) -> bool { + self.launch_systemd + } + + pub fn get_kernel_path(&self) -> Result { + match self.kernel_path { + Some(ref path) if path.exists() => return Ok(path.to_path_buf()), + None => if let Some(path) = Self::search_kernel() { + return Ok(path) + } + _ => {}, + } + Err(ErrorKind::KernelNotFound.into()) + } + + pub fn get_init_path(&self) -> Result { + match self.init_path { + Some(ref path) if path.exists() => return Ok(path.to_path_buf()), + None => if let Some(path) = Self::search_init() { + return Ok(path) + } + _ => {}, + } + Err(ErrorKind::InitNotFound.into()) + } + + pub fn get_init_cmdline(&self) -> Option<&str> { + self.init_cmd.as_ref().map(|s| s.as_str()) + } + + pub fn rootfs(&self) -> &RootFS { + &self.rootfs + } + + fn search_init() -> Option { + Self::search_binary("ph-init", &[ + "rust/target/release", "rust/target/debug", + "target/debug", "target/release" + ]) + } + + fn search_kernel() -> Option { + Self::search_binary("ph_linux", &["kernel", "../kernel"]) + } + + fn search_binary(name: &str, paths: &[&str]) -> Option { + let cwd = match env::current_dir() { + Ok(cwd) => cwd, + _ => return None, + }; + + for p in paths { + let p = Path::new(p).join(name); + let current = if p.is_absolute() { + p + } else { + cwd.join(p) + }; + if current.exists() { + return Some(current); + } + } + None + } + + fn parse_args(&mut self, args: env::Args) { + for arg in args.skip(1) { + self.parse_one_arg(&arg); + } + } + + fn parse_one_arg(&mut self, arg: &str) { + if arg == "-v" { + self.verbose = true; + } else { + eprintln!("Unrecognized command line argument: {}", arg); + process::exit(1); + } + } +} diff --git a/rust/src/vm/error.rs b/rust/src/vm/error.rs index b71d172..41e781a 100644 --- a/rust/src/vm/error.rs +++ b/rust/src/vm/error.rs @@ -1,4 +1,4 @@ -use std::result; +use std::{result, io}; use std::error; use std::fmt; use std::str; @@ -10,6 +10,8 @@ pub type Result = result::Result; #[derive(Debug)] pub enum ErrorKind { + KernelNotFound, + InitNotFound, InvalidAddress(u64), InvalidMappingOffset(usize), RegisterMemoryFailed, @@ -22,12 +24,15 @@ pub enum ErrorKind { CreateVmFailed, BadVersion, EventFdError, - DiskImageOpen(disk::Error) + DiskImageOpen(disk::Error), + TerminalTermios(io::Error), } impl ErrorKind { fn as_str(&self) -> &'static str { match *self { + ErrorKind::KernelNotFound => "Could not find kernel image", + ErrorKind::InitNotFound => "Could not find init image", ErrorKind::InvalidAddress(..) => "Invalid guest memory address", ErrorKind::InvalidMappingOffset(..) => "Invalid memory mapping offset", ErrorKind::RegisterMemoryFailed => "Failed to register memory region", @@ -41,6 +46,7 @@ impl ErrorKind { ErrorKind::BadVersion => "unexpected kvm api version", ErrorKind::EventFdError => "eventfd error", ErrorKind::DiskImageOpen(_) => "failed to open disk image", + ErrorKind::TerminalTermios(_) => "failed termios", } } } @@ -52,6 +58,7 @@ impl fmt::Display for ErrorKind { ErrorKind::InvalidMappingOffset(offset) => write!(f, "{}: 0x{:x}", self.as_str(), offset), ErrorKind::IoctlFailed(name) => write!(f, "Ioctl {} failed", name), ErrorKind::DiskImageOpen(ref e) => write!(f, "failed to open disk image: {}", e), + ErrorKind::TerminalTermios(ref e) => write!(f, "error reading/restoring terminal state: {}", e), _ => write!(f, "{}", self.as_str()), } } diff --git a/rust/src/vm/kernel_cmdline.rs b/rust/src/vm/kernel_cmdline.rs index 4286718..a2f8e9d 100644 --- a/rust/src/vm/kernel_cmdline.rs +++ b/rust/src/vm/kernel_cmdline.rs @@ -5,11 +5,7 @@ use crate::memory::{GuestRam,KERNEL_CMDLINE_ADDRESS}; use super::Result; -fn add_defaults(cmdline: &mut KernelCmdLine, rdonly_root: bool, verbose: bool) { - let root_mount_type = if rdonly_root { "ro" } else { "rw" }; - - let output = if verbose {"earlyprintk=serial"} else {"quiet"}; - +fn add_defaults(cmdline: &mut KernelCmdLine) { cmdline .push("noapic") .push("noacpi") @@ -24,10 +20,6 @@ fn add_defaults(cmdline: &mut KernelCmdLine, rdonly_root: bool, verbose: bool) { .push_set_true("rcuupdate.rcu_normal_after_boot") .push_set_val("console", "hvc0") - .push(root_mount_type) - .push_set_val("rootfstype", "9p") - .push_set_val("rootflags", "trans=virtio,version=9p2000.L,cache=loose") - .push_set_true("i8042.direct") .push_set_true("i8042.dumbkbd") .push_set_true("i8042.nopnp") @@ -37,11 +29,7 @@ fn add_defaults(cmdline: &mut KernelCmdLine, rdonly_root: bool, verbose: bool) { .push_set_val("iommu", "off") .push("cryptomgr.notests") - .push(output) - - .push_set_val("8250.nr_uarts", "0") - //.push_set_val("init", "/home/user/virt/init"); - .push_set_val("init", "/phinit"); + .push_set_val("8250.nr_uarts", "0"); } @@ -55,13 +43,12 @@ impl KernelCmdLine { KernelCmdLine { address: KERNEL_CMDLINE_ADDRESS, buffer: OsString::new() } } - pub fn new_default(verbose: bool) -> KernelCmdLine { + pub fn new_default() -> KernelCmdLine { let mut cmdline = KernelCmdLine::new(); - add_defaults(&mut cmdline, true, verbose); + add_defaults(&mut cmdline); cmdline } - pub fn push(&mut self, option: &str) -> &mut Self { if !self.buffer.is_empty() { self.buffer.push(" "); @@ -89,13 +76,8 @@ impl KernelCmdLine { pub fn write_to_memory(&self, memory: &GuestRam) -> Result<()> { let bs = self.buffer.as_bytes(); let len = bs.len(); - //println!("Kernel CmdLine: {:?}", self.buffer); - //println!("writing {} command line bytes to 0x{:x}", len + 1, KERNEL_CMDLINE_ADDRESS); memory.write_bytes(KERNEL_CMDLINE_ADDRESS, bs)?; memory.write_int(KERNEL_CMDLINE_ADDRESS + len as u64, 0u8)?; Ok(()) } - - - } diff --git a/rust/src/vm/mod.rs b/rust/src/vm/mod.rs index 158d351..f48202e 100644 --- a/rust/src/vm/mod.rs +++ b/rust/src/vm/mod.rs @@ -1,22 +1,21 @@ -use std::sync::Arc; use std::thread; -use std::path::{PathBuf,Path}; -use std::env; use self::io::IoDispatcher; use crate::virtio::VirtioBus; use crate::devices; +use crate::disk; use crate::memory::{GuestRam, KVM_KERNEL_LOAD_ADDRESS, MemoryManager, SystemAllocator, AddressRange}; use crate::kvm::*; - mod run; pub mod io; mod setup; mod error; mod kernel_cmdline; +mod config; +pub use config::VmConfig; pub use self::error::{Result,Error,ErrorKind}; @@ -24,48 +23,20 @@ pub use self::error::{Result,Error,ErrorKind}; use self::run::KvmRunArea; use self::kernel_cmdline::KernelCmdLine; - -pub struct VmConfig { - ram_size: usize, - ncpus: usize, - kernel_path: PathBuf, - init_path: PathBuf, -} - -#[allow(dead_code)] -impl VmConfig { - pub fn new() -> VmConfig { - VmConfig { - ram_size: 256 * 1024 * 1024, - ncpus: 1, - kernel_path: PathBuf::new(), - init_path: PathBuf::new(), - } - } - - pub fn ram_size_megs(&mut self, megs: usize) { - self.ram_size = megs * 1024 * 1024; - } - - pub fn num_cpus(&mut self, ncpus: usize) { - self.ncpus = ncpus; - } - - pub fn kernel_path(&mut self, path: &Path) { - self.kernel_path = path.to_path_buf(); - } - - pub fn init_path(&mut self, path: &Path) { - self.init_path = path.to_path_buf(); - } +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use crate::disk::OpenType; +use termios::Termios; +use crate::vm::config::RootFS; +use std::path::Path; -} pub struct Vm { - kvm: Kvm, + _config: VmConfig, memory: MemoryManager, io_dispatcher: Arc, - _virtio: VirtioBus, + termios: Option, + _virtio: Arc, } static REQUIRED_EXTENSIONS: &[u32] = &[ @@ -89,55 +60,122 @@ fn get_base_dev_pfn(mem_size: u64) -> u64 { } impl Vm { - pub fn open(config: VmConfig) -> Result { - let mut kvm = Kvm::open(&REQUIRED_EXTENSIONS)?; - + fn create_kvm() -> Result { + let kvm = Kvm::open(&REQUIRED_EXTENSIONS)?; kvm.set_tss_addr(0xFFFbd000)?; kvm.create_pit2()?; - - let ram = GuestRam::new(config.ram_size, &kvm)?; - let dev_addr_start = get_base_dev_pfn(config.ram_size as u64) * 4096; + kvm.create_irqchip()?; + Ok(kvm) + } + fn create_memory_manager(ram_size: usize) -> Result { + let kvm = Self::create_kvm()?; + let ram = GuestRam::new(ram_size, &kvm)?; + let dev_addr_start = get_base_dev_pfn(ram_size as u64) * 4096; let dev_addr_size = u64::max_value() - dev_addr_start; let allocator = SystemAllocator::new(AddressRange::new(dev_addr_start,dev_addr_size as usize)); - let memory = MemoryManager::new(kvm.clone(), ram, allocator); + Ok(MemoryManager::new(kvm, ram, allocator)) + } - kvm.create_irqchip()?; + fn setup_virtio(config: &VmConfig, cmdline: &mut KernelCmdLine, virtio: &mut VirtioBus) -> Result<()> { + devices::VirtioSerial::create(virtio)?; + devices::VirtioRandom::create(virtio)?; + devices::VirtioWayland::create(virtio)?; + let init_path = config.get_init_path()?; + devices::VirtioP9::create(virtio, "home", "/home/user", &init_path)?; - let verbose = env::args().any(|arg| arg == "-v"); - let cmdline = KernelCmdLine::new_default(verbose); + Self::setup_rootfs(config, cmdline, virtio, &init_path) + } - cmdline.write_to_memory(memory.guest_ram())?; - let path = PathBuf::from(&config.kernel_path); - setup::kernel::load_pm_kernel(memory.guest_ram(), &path, cmdline.address(), cmdline.size())?; + fn setup_rootfs(config: &VmConfig, cmdline: &mut KernelCmdLine, virtio: &mut VirtioBus, init_path: &Path) -> Result<()> { + match config.rootfs() { + RootFS::SelfRoot => { + devices::VirtioP9::create(virtio, "/dev/root", "/", &init_path)?; + notify!("9p root"); + cmdline.push_set_val("root", "/dev/root"); + cmdline.push("ro"); + cmdline.push_set_val("rootfstype", "9p"); + cmdline.push_set_val("rootflags", "trans=virtio,version=9p2000.L,cache=loose"); + cmdline.push_set_val("init", "/phinit"); + }, + RootFS::RealmFSImage(ref path) => { + let disk = disk::RealmFSImage::open(path, false) + .map_err(ErrorKind::DiskImageOpen)?; + devices::VirtioBlock::create(virtio, disk)?; + cmdline.push_set_val("root", "/dev/vda"); + cmdline.push("rw"); + cmdline.push_set_val("init", "/usr/bin/ph-init"); + }, + RootFS::RawImage(ref path) => { + let disk = disk::RawDiskImage::open(path, OpenType::MemoryOverlay).map_err(ErrorKind::DiskImageOpen)?; + devices::VirtioBlock::create(virtio, disk)?; + cmdline.push_set_val("root", "/dev/vda"); + cmdline.push("rw"); + }, + RootFS::RawOffset(path, offset) => { + let disk = disk::RawDiskImage::open_with_offset(path, OpenType::ReadWrite, *offset) + .map_err(ErrorKind::DiskImageOpen)?; + devices::VirtioBlock::create(virtio, disk)?; + cmdline.push_set_val("root", "/dev/vda"); + cmdline.push("rw"); + cmdline.push_set_val("init", "/usr/bin/ph-init"); + } + }; + Ok(()) + } + + pub fn open(config: VmConfig) -> Result { + let kernel_path = config.get_kernel_path()?; + + let mut memory = Self::create_memory_manager(config.ram_size())?; + + let mut cmdline = KernelCmdLine::new_default(); + + setup::kernel::load_pm_kernel(memory.guest_ram(), &kernel_path, cmdline.address(), cmdline.size())?; let io_dispatch = IoDispatcher::new(); - kvm.create_vcpus(config.ncpus)?; + memory.kvm_mut().create_vcpus(config.ncpus())?; devices::rtc::Rtc::register(io_dispatch.clone()); - if verbose { - devices::serial::SerialDevice::register(kvm.clone(),io_dispatch.clone(), 0); + if config.verbose() { + cmdline.push("earlyprintk=serial"); + devices::serial::SerialDevice::register(memory.kvm().clone(),io_dispatch.clone(), 0); + } else { + cmdline.push("quiet"); } - let mut virtio = VirtioBus::new(memory.clone(), io_dispatch.clone(), kvm.clone()); - devices::VirtioSerial::create(&mut virtio)?; - devices::VirtioRandom::create(&mut virtio)?; - devices::VirtioP9::create(&mut virtio, "/dev/root", "/", &config.init_path)?; + let saved= Termios::from_fd(0) + .map_err(ErrorKind::TerminalTermios)?; + let termios = Some(saved); - setup::mptable::setup_mptable(&memory, config.ncpus, virtio.pci_irqs())?; + let mut virtio = VirtioBus::new(memory.clone(), io_dispatch.clone(), memory.kvm().clone()); + Self::setup_virtio(&config, &mut cmdline, &mut virtio)?; + + if config.launch_systemd() { + cmdline.push("phinit.run_systemd"); + } + if let Some(init_cmd) = config.get_init_cmdline() { + cmdline.push_set_val("init", init_cmd); + } + + cmdline.write_to_memory(memory.guest_ram())?; + + setup::mptable::setup_mptable(memory.guest_ram(), config.ncpus(), virtio.pci_irqs())?; Ok(Vm { - kvm, + _config: config, memory, io_dispatcher: io_dispatch, - _virtio: virtio, + termios, + _virtio: Arc::new(virtio), }) } pub fn start(&self) -> Result<()> { + let shutdown = Arc::new(AtomicBool::new(false)); let mut handles = Vec::new(); - for vcpu in self.kvm.get_vcpus() { + for vcpu in self.memory.kvm().get_vcpus() { setup::cpu::setup_protected_mode(&vcpu, KVM_KERNEL_LOAD_ADDRESS + 0x200, self.memory.guest_ram())?; let mut run_area = KvmRunArea::new(vcpu, shutdown.clone(), self.io_dispatcher.clone())?; let h = thread::spawn(move || run_area.run()); @@ -147,6 +185,10 @@ impl Vm { for h in handles { h.join().expect("..."); } + if let Some(termios) = self.termios { + let _ = termios::tcsetattr(0, termios::TCSANOW, &termios) + .map_err(ErrorKind::TerminalTermios)?; + } Ok(()) } diff --git a/rust/src/vm/run.rs b/rust/src/vm/run.rs index 765afef..bff9825 100644 --- a/rust/src/vm/run.rs +++ b/rust/src/vm/run.rs @@ -4,6 +4,7 @@ use crate::kvm::KvmVcpu; use crate::memory::Mapping; use super::Result; use super::io::IoDispatcher; +use std::sync::atomic::{AtomicBool, Ordering}; const KVM_EXIT_UNKNOWN:u32 = 0; const KVM_EXIT_IO:u32 = 2; @@ -17,6 +18,7 @@ pub struct KvmRunArea { vcpu: KvmVcpu, io: Arc, mapping: Mapping, + shutdown: Arc, } pub struct IoExitData { @@ -34,13 +36,14 @@ pub struct MmioExitData { } impl KvmRunArea { - pub fn new(vcpu: KvmVcpu, io_dispatcher: Arc) -> Result { + pub fn new(vcpu: KvmVcpu, shutdown: Arc, io_dispatcher: Arc) -> Result { let size = vcpu.get_vcpu_mmap_size()?; let mapping = Mapping::new_from_fd(vcpu.raw_fd(), size)?; Ok(KvmRunArea{ vcpu, io: io_dispatcher, mapping, + shutdown, }) } @@ -97,6 +100,9 @@ impl KvmRunArea { } else { self.handle_exit(); } + if self.shutdown.load(Ordering::Relaxed) { + return; + } } } @@ -106,8 +112,8 @@ impl KvmRunArea { KVM_EXIT_IO => { self.handle_exit_io() }, KVM_EXIT_MMIO => { self.handle_exit_mmio() }, KVM_EXIT_INTR => { println!("intr")}, - KVM_EXIT_SHUTDOWN => { println!("shut"); - self.handle_problem(); + KVM_EXIT_SHUTDOWN => { + self.handle_shutdown(); }, KVM_EXIT_SYSTEM_EVENT => { println!("event")}, KVM_EXIT_INTERNAL_ERROR => { @@ -120,7 +126,11 @@ impl KvmRunArea { } } - fn handle_problem(&mut self) { + fn handle_shutdown(&mut self) { + self.shutdown.store(true, Ordering::Relaxed); + } + + fn _handle_problem(&mut self) { let regs = self.vcpu.get_regs().unwrap(); let sregs = self.vcpu.get_sregs().unwrap(); println!("REGS:\n{:?}", regs);