Initial implementation of RealmFS

Cargo.lock

@@ -132,7 +132,9 @@ version = "0.1.0"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.47 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libcitadel 0.1.0",
  "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)",
  "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",

citadel-realms/Cargo.toml

@@ -6,6 +6,7 @@ homepage = "http://github.com/subgraph/citadel"
 edition = "2018"
 
 [dependencies]
+libcitadel = { path = "../libcitadel" }
 libc = "0.2"
 clap = "2.30.0"
 failure = "0.1.1"
@@ -14,3 +15,4 @@ serde_derive = "1.0.27"
 serde = "1.0.27"
 termcolor = "0.3"
 walkdir = "2"
+lazy_static = "1.2.0"

citadel-realms/src/appimg.rs

@@ -1,43 +0,0 @@
-use std::path::Path;
-use std::process::Command;
-
-use crate::Realm;
-use crate::Result;
-
-const BASE_APPIMG_PATH: &str = "/storage/appimg/base.appimg";
-const BTRFS_COMMAND: &str = "/usr/bin/btrfs";
-
-pub fn clone_base_appimg(target_realm: &Realm) -> Result<()> {
-    if !Path::new(BASE_APPIMG_PATH).exists() {
-        bail!("base appimg does not exist at {}", BASE_APPIMG_PATH);
-    }
-    let target = format!("/realms/realm-{}/rootfs", target_realm.name());
-    let target_path = Path::new(&target);
-
-    if target_path.exists() {
-        bail!("cannot create clone of base appimg for realm '{}' because rootfs directory already exists at {}", 
-              target_realm.name(), target);
-    }
-
-    if !target_path.parent().unwrap().exists() {
-        bail!("cannot create clone of base appimg for realm '{}' because realm directory /realms/realm-{} does not exist.", 
-              target_realm.name(), target_realm.name());
-    }
-
-    Command::new(BTRFS_COMMAND)
-        .args(&["subvolume", "snapshot", BASE_APPIMG_PATH, &target ])
-        .status()
-        .map_err(|e| format_err!("failed to execute {}: {}", BTRFS_COMMAND, e))?;
-    Ok(())
-
-}
-
-pub fn delete_rootfs_subvolume(realm: &Realm) -> Result<()> {
-    let path = realm.base_path().join("rootfs");
-    Command::new(BTRFS_COMMAND)
-        .args(&["subvolume", "delete", path.to_str().unwrap() ])
-        .status()
-        .map_err(|e| format_err!("failed to execute {}: {}", BTRFS_COMMAND, e))?;
-    Ok(())
-}
-

citadel-realms/src/main.rs

@@ -1,5 +1,6 @@
 #[macro_use] extern crate failure;
 #[macro_use] extern crate serde_derive;
+#[macro_use] extern crate lazy_static;
 
 use failure::Error;
 use clap::{App,Arg,ArgMatches,SubCommand};
@@ -33,13 +34,15 @@ mod util;
 mod systemd;
 mod config;
 mod network;
-mod appimg;
+
+use libcitadel::RealmFS;
 
 use crate::realm::{Realm,RealmSymlinks};
 use crate::manager::RealmManager;
 use crate::config::RealmConfig;
 use crate::systemd::Systemd;
 use crate::network::NetworkConfig;
+use crate::config::GLOBAL_CONFIG;
 
 fn main() {
     let app = App::new("citadel-realms")

citadel-realms/src/realm.rs

@@ -5,9 +5,10 @@ use std::cell::{RefCell,Cell};
 use std::fs::{self,File};
 use std::os::unix::fs::{symlink,MetadataExt};
 
-use crate::{RealmConfig,Result,Systemd,NetworkConfig};
+use libcitadel::{CommandLine,RealmFS};
+
+use crate::{RealmConfig,Result,Systemd,NetworkConfig,GLOBAL_CONFIG};
 use crate::util::*;
-use crate::appimg::*;
 
 const REALMS_BASE_PATH: &str = "/realms";
 const REALMS_RUN_PATH: &str = "/run/realms";
@@ -45,8 +46,7 @@ impl Realm {
 
     fn load_config(&mut self) -> Result<()> {
         let path = self.base_path().join("config");
-        self.config = RealmConfig::load_or_default(&path)
+        self.config = RealmConfig::load_or_default(&path);
-            .map_err(|e| format_err!("failed to load realm config file {}: {}", path.display(), e))?;
         Ok(())
     }
 
@@ -105,11 +105,89 @@ impl Realm {
     }
 
     pub fn start(&self) -> Result<()> {
+        self.setup_realmfs(self.config.realmfs())?;
         self.systemd.start_realm(self)?;
         info!("Started realm '{}'", self.name());
         Ok(())
     }
 
+    fn setup_realmfs(&self, name: &str) -> Result<()> {
+        let mut realmfs = self.get_named_realmfs(name)?;
+        self.setup_rootfs_link(&realmfs)?;
+
+        info!("Starting realm with realmfs = {}", name);
+        if !realmfs.is_mounted() {
+            if realmfs.is_sealed() {
+                realmfs.mount_verity()?;
+            } else {
+                if CommandLine::sealed() {
+                    bail!("Cannot start realm because realmfs {} is not sealed and citadel.sealed is set", name);
+                }
+                realmfs.mount_rw()?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Return named RealmFS instance if it already exists.
+    /// Otherwise, create it as a fork of the 'default' image.
+    /// The default image is either 'base' or some other name
+    /// from the global realm config file.
+    ///
+    /// If the default image does not exist, then create that too
+    /// as a fork of 'base' image.
+    fn get_named_realmfs(&self, name: &str) -> Result<RealmFS> {
+        if RealmFS::named_image_exists(name) {
+            return RealmFS::load_by_name(name);
+        }
+
+        if CommandLine::sealed() {
+            bail!("Realm {} needs RealmFS {} which does not exist and cannot be created in sealed realmfs mode", self.name(), name);
+        }
+
+        let default = GLOBAL_CONFIG.realmfs();
+
+        let default_image = if RealmFS::named_image_exists(default) {
+            RealmFS::load_by_name(default)?
+        } else {
+            // If default image name is something other than 'base' and does
+            // not exist, create it as a fork of 'base'
+            let base = RealmFS::load_by_name("base")?;
+            base.fork(default)?
+        };
+
+        // Requested name might be the default image that was just created, if so return it.
+        let image = if name == default {
+            default_image
+        } else {
+            default_image.fork(name)?
+        };
+        Ok(image)
+    }
+
+    // Make sure rootfs in realm directory is a symlink pointing to the correct realmfs mountpoint
+    fn setup_rootfs_link(&self, realmfs: &RealmFS) -> Result<()> {
+        let mountpoint = realmfs.mountpoint();
+        let rootfs = self.base_path().join("rootfs");
+
+        if rootfs.exists() {
+            let link = fs::read_link(&rootfs)?;
+            if link == mountpoint {
+                return Ok(())
+            }
+            fs::remove_file(&rootfs)?;
+        }
+        symlink(mountpoint, rootfs)?;
+        Ok(())
+    }
+
+    pub fn readonly_rootfs(&self) -> bool {
+        if CommandLine::sealed() {
+            return true
+        }
+        !self.config.realmfs_write()
+    }
+
     pub fn stop(&self) -> Result<()> {
         self.systemd.stop_realm(self)?;
         if self.is_current() {
@@ -158,9 +236,6 @@ impl Realm {
         self.create_home_directory()
             .map_err(|e| format_err!("failed to create realm home directory {}: {}", self.base_path().join("home").display(), e))?;
 
-        // This must be last step because if an error is returned caller assumes that subvolume was
-        // never created and does not need to be removed.
-        clone_base_appimg(self)?;
         Ok(())
     }
 
@@ -183,8 +258,6 @@ impl Realm {
         if self.is_running()? {
             self.stop()?;
         }
-        info!("removing rootfs subvolume for '{}'", self.name());
-        delete_rootfs_subvolume(self)?;
 
         info!("removing realm directory {}", self.base_path().display());
         fs::remove_dir_all(self.base_path())?;

citadel-realms/src/systemd.rs

@@ -283,9 +283,9 @@ impl Systemd {
 
     fn generate_nspawn_file(&self, realm: &Realm) -> Result<String> {
         Ok(NSPAWN_FILE_TEMPLATE
-           .replace("$EXTRA_BIND_MOUNTS", &self.generate_extra_bind_mounts(realm)?)
+            .replace("$EXTRA_BIND_MOUNTS", &self.generate_extra_bind_mounts(realm)?)
-
+            .replace("$EXTRA_FILE_OPTIONS", &self.generate_extra_file_options(realm)?)
-           .replace("$NETWORK_CONFIG", &self.generate_network_config(realm)?))
+            .replace("$NETWORK_CONFIG", &self.generate_network_config(realm)?))
     }
 
     fn generate_extra_bind_mounts(&self, realm: &Realm) -> Result<String> {
@@ -327,6 +327,15 @@ impl Systemd {
         Ok(s)
     }
 
+    fn generate_extra_file_options(&self, realm: &Realm) -> Result<String> {
+        let mut s = String::new();
+        if realm.readonly_rootfs() {
+            writeln!(s, "ReadOnly=true")?;
+            writeln!(s, "Overlay=+/var::/var")?;
+        }
+        Ok(s)
+    }
+
     fn generate_network_config(&self, realm: &Realm) -> Result<String> {
         let mut s = String::new();
         if realm.config().network() {
@@ -363,6 +372,8 @@ BindReadOnly=/storage/citadel-state/resolv.conf:/etc/resolv.conf
 
 $EXTRA_BIND_MOUNTS
 
+$EXTRA_FILE_OPTIONS
+
 "###;
 
 pub const REALM_SERVICE_TEMPLATE: &str = r###"
@@ -372,7 +383,7 @@ Wants=citadel-desktopd.service
 
 [Service]
 Environment=SYSTEMD_NSPAWN_SHARE_NS_IPC=1
-ExecStart=/usr/bin/systemd-nspawn --quiet --notify-ready=yes --keep-unit --machine=$REALM_NAME --link-journal=try-guest --directory=$ROOTFS
+ExecStart=/usr/bin/systemd-nspawn --quiet --notify-ready=yes --keep-unit --machine=$REALM_NAME --link-journal=auto --directory=$ROOTFS
 
 KillMode=mixed
 Type=notify

libcitadel/src/lib.rs

@@ -58,6 +58,7 @@ mod resource;
 pub mod util;
 pub mod verity;
 mod mount;
+mod realmfs;
 
 pub use crate::config::OsRelease;
 pub use crate::blockdev::BlockDev;
@@ -67,6 +68,7 @@ pub use crate::partition::Partition;
 pub use crate::resource::ResourceImage;
 pub use crate::keys::{KeyPair,PublicKey};
 pub use crate::mount::Mount;
+pub use crate::realmfs::RealmFS;
 
 const DEVKEYS_HEX: &str =
     "3053020101300506032b6570042204206ed2849c6c5168e1aebc50005ac3d4a4e84af4889e4e0189bb4c787e6ee0be49a1230321006b652764c62a1de35e7e37af2b743e9a5b82cee2211cf3091d2514441b417f5f";

libcitadel/src/realmfs.rs

@@ -0,0 +1,224 @@
+
+use std::path::{Path,PathBuf};
+use std::fs;
+use std::io::Write;
+
+use crate::{ImageHeader,MetaInfo,Mount,Result,util,verity};
+
+const BASE_PATH: &'static str = "/storage/realms/realmfs-images";
+const RUN_DIRECTORY: &str = "/run/images";
+const MAX_REALMFS_NAME_LEN: usize = 40;
+
+
+pub struct RealmFS {
+    path: PathBuf,
+    mountpoint: PathBuf,
+    header: ImageHeader,
+    metainfo: MetaInfo,
+}
+
+impl RealmFS {
+
+    /// Locate a RealmFS image by name in the default location using the standard name convention
+    pub fn load_by_name(name: &str) -> Result<RealmFS> {
+        if !util::is_valid_name(name, MAX_REALMFS_NAME_LEN) {
+            bail!("Invalid realmfs name '{}'", name);
+        }
+        let path = Path::new(BASE_PATH).join(format!("{}-realmfs.img", name));
+        if !path.exists() {
+            bail!("No image found at {}", path.display());
+        }
+
+        RealmFS::load_from_path(path, name)
+    }
+
+    pub fn named_image_exists(name: &str) -> bool {
+        if !util::is_valid_name(name, MAX_REALMFS_NAME_LEN) {
+            return false;
+        }
+        let path = Path::new(BASE_PATH).join(format!("{}-realmfs.img", name));
+        path.exists()
+    }
+
+    /// Load RealmFS image from an exact path.
+    pub fn load_from_path<P: AsRef<Path>>(path: P, name: &str) -> Result<RealmFS> {
+        let path = path.as_ref().to_owned();
+        let header = ImageHeader::from_file(&path)?;
+        if !header.is_magic_valid() {
+            bail!("Image file {} does not have a valid header", path.display());
+        }
+        let metainfo = header.metainfo()?;
+        let mountpoint = PathBuf::from(format!("{}/{}-realmfs.mountpoint", RUN_DIRECTORY, name));
+
+        Ok(RealmFS{
+            path,
+            mountpoint,
+            header,
+            metainfo,
+        })
+    }
+
+    pub fn mount_rw(&mut self) -> Result<()> {
+        // XXX fail if already verity mounted?
+        // XXX strip dm-verity tree if present?
+        // XXX just remount if not verity mounted, but currently ro mounted?
+        self.mount(false)
+    }
+
+    pub fn mount_ro(&mut self) -> Result<()> {
+        self.mount(true)
+    }
+
+    fn mount(&mut self, read_only: bool) -> Result<()> {
+        let flags = if read_only {
+            Some("-oro")
+        } else {
+            Some("-orw")
+        };
+        if !self.mountpoint.exists() {
+            fs::create_dir_all(self.mountpoint())?;
+        }
+        let loopdev = self.create_loopdev()?;
+        util::mount(&loopdev.to_string_lossy(), self.mountpoint(), flags)
+    }
+
+    pub fn mount_verity(&self) -> Result<()> {
+        if self.is_mounted() {
+            bail!("RealmFS image is already mounted");
+        }
+        if !self.is_sealed() {
+            bail!("Cannot verity mount RealmFS image because it's not sealed");
+        }
+        if !self.mountpoint.exists() {
+            fs::create_dir_all(self.mountpoint())?;
+        }
+        let dev = self.setup_verity_device()?;
+        util::mount(&dev.to_string_lossy(), &self.mountpoint, Some("-oro"))
+    }
+
+    fn setup_verity_device(&self) -> Result<PathBuf> {
+
+        // TODO verify signature
+
+        if !self.header.has_flag(ImageHeader::FLAG_HASH_TREE) {
+            self.generate_verity()?;
+        }
+        verity::setup_image_device(&self.path, &self.metainfo)
+    }
+
+    pub fn create_loopdev(&self) -> Result<PathBuf> {
+        let args = format!("--offset 4096 -f --show {}", self.path.display());
+        let output = util::exec_cmdline_with_output("/sbin/losetup", args)?;
+        Ok(PathBuf::from(output))
+    }
+
+    pub fn is_mounted(&self) -> bool {
+        match Mount::is_target_mounted(self.mountpoint()) {
+            Ok(val) => val,
+            Err(e) => {
+                warn!("Error reading /proc/mounts: {}", e);
+                false
+            }
+        }
+    }
+
+    pub fn fork(&self, new_name: &str) -> Result<RealmFS> {
+        if !util::is_valid_name(new_name, MAX_REALMFS_NAME_LEN) {
+            bail!("Invalid realmfs name '{}'", new_name);
+        }
+
+        // during install the images have a different base directory
+        let mut new_path = self.path.clone();
+        new_path.pop();
+        new_path.push(format!("{}-realmfs.img", new_name));
+
+        if new_path.exists() {
+            bail!("RealmFS image for name {} already exists", new_name);
+        }
+
+        let args = format!("--reflink=auto {} {}", self.path.display(), new_path.display());
+        util::exec_cmdline("/usr/bin/cp", args)?;
+
+        let header = ImageHeader::new();
+        header.set_metainfo_bytes(&self.generate_fork_metainfo(new_name));
+        header.write_header_to(&new_path)?;
+
+        let realmfs = RealmFS::load_from_path(&new_path, new_name)?;
+
+        // forking unseals since presumably the image is being forked to modify it
+        realmfs.truncate_verity()?;
+        Ok(realmfs)
+    }
+
+
+    fn generate_fork_metainfo(&self, name: &str) -> Vec<u8> {
+        let mut v = Vec::new();
+        writeln!(v, "image-type = \"realmfs\"").unwrap();
+        writeln!(v, "realmfs-name = \"{}\"", name).unwrap();
+        writeln!(v, "nblocks = {}", self.metainfo.nblocks()).unwrap();
+        v
+    }
+
+    // Remove verity tree from image file by truncating file to the number of blocks in metainfo
+    fn truncate_verity(&self) -> Result<()> {
+        let verity_flag = self.header.has_flag(ImageHeader::FLAG_HASH_TREE);
+        if verity_flag {
+            self.header.clear_flag(ImageHeader::FLAG_HASH_TREE);
+            self.header.write_header_to(&self.path)?;
+        }
+
+        let meta = self.path.metadata()?;
+        let expected = (self.metainfo.nblocks() + 1) * 4096;
+        let actual = meta.len() as usize;
+
+        if actual > expected {
+            if !verity_flag {
+                warn!("RealmFS length was greater than length indicated by metainfo but FLAG_HASH_TREE not set");
+            }
+            let f = fs::OpenOptions::new().write(true).open(&self.path)?;
+            f.set_len(expected as u64)?;
+        } else if actual < expected {
+            bail!("RealmFS image {} is shorter than length indicated by metainfo", self.path.display());
+        }
+        if verity_flag {
+            warn!("FLAG_HASH_TREE was set but RealmFS image file length matched metainfo length");
+        }
+        Ok(())
+    }
+
+    pub fn is_sealed(&self) -> bool {
+        !self.metainfo.verity_root().is_empty()
+    }
+
+    fn generate_verity(&self) -> Result<()> {
+        self.truncate_verity()?;
+        verity::generate_image_hashtree(&self.path, &self.metainfo)?;
+        self.header.set_flag(ImageHeader::FLAG_HASH_TREE);
+        self.header.write_header_to(&self.path)?;
+        Ok(())
+    }
+
+    pub fn create_overlay(&self, basedir: &Path) -> Result<PathBuf> {
+        if !self.is_mounted() {
+            bail!("Cannot create overlay until realmfs is mounted");
+        }
+        let workdir = basedir.join("workdir");
+        let upperdir = basedir.join("upperdir");
+        let mountpoint = basedir.join("mountpoint");
+        fs::create_dir_all(&workdir)?;
+        fs::create_dir_all(&upperdir)?;
+        fs::create_dir_all(&mountpoint)?;
+        let args = format!("-t overlay realmfs-overlay -olowerdir={},upperdir={},workdir={} {}",
+            self.mountpoint.display(),
+            upperdir.display(),
+            workdir.display(),
+            mountpoint.display());
+
+        util::exec_cmdline("/usr/bin/mount", args)?;
+        Ok(mountpoint)
+    }
+
+    pub fn mountpoint(&self) -> &Path {
+        &self.mountpoint
+    }
+}