From 3dbfda2c40aac28e2a1df1dddae039113df1c6f0 Mon Sep 17 00:00:00 2001 From: Bruce Leidl Date: Fri, 27 May 2022 18:06:52 -0400 Subject: [PATCH] Use less memory when decompressing images for install 1. Delete source image before decompressing temporary file 2. Decompress the images serially instead of spawning a thread for each one. --- citadel-tool/src/boot/live.rs | 29 +++++++++++++++++++++-------- citadel-tool/src/boot/rootfs.rs | 2 +- citadel-tool/src/image/mod.rs | 2 +- citadel-tool/src/update/mod.rs | 2 +- libcitadel/src/exec.rs | 2 +- libcitadel/src/resource.rs | 12 ++++++++---- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/citadel-tool/src/boot/live.rs b/citadel-tool/src/boot/live.rs index 5dc2671..6cef9ab 100644 --- a/citadel-tool/src/boot/live.rs +++ b/citadel-tool/src/boot/live.rs @@ -20,7 +20,7 @@ pub fn live_rootfs() -> Result<()> { } pub fn live_setup() -> Result<()> { - decompress_images()?; + decompress_images(true)?; info!("Starting live setup"); let live = Installer::new_livesetup(); live.run() @@ -131,14 +131,21 @@ fn find_rootfs_image() -> Result { bail!("unable to find rootfs resource image in {}", IMAGE_DIRECTORY) } -fn decompress_images() -> Result<()> { +fn decompress_images(sync: bool) -> Result<()> { info!("Decompressing images"); let mut threads = Vec::new(); util::read_directory("/run/citadel/images", |dent| { if dent.path().extension() == Some(OsStr::new("img")) { if let Ok(image) = ResourceImage::from_path(&dent.path()) { if image.is_compressed() { - threads.push(decompress_one_image(image)); + if sync { + if let Err(err) = decompress_one_image_sync(image) { + warn!("Error: {}", err); + } + + } else { + threads.push(decompress_one_image(image)); + } } } } @@ -146,22 +153,28 @@ fn decompress_images() -> Result<()> { })?; for t in threads { - t.join().unwrap()?; + if let Err(err) = t.join().unwrap() { + warn!("Error: {}", err); + } } info!("Finished decompressing images"); Ok(()) - } -fn decompress_one_image(image: ResourceImage) -> JoinHandle> { - thread::spawn(move || { +fn decompress_one_image_sync(image: ResourceImage) -> Result<()> { let start = Instant::now(); info!("Decompressing {}", image.path().display()); - image.decompress()?; + image.decompress(true) + .map_err(|e| format_err!("Failed to decompress image file {}: {}", image.path().display(), e))?; cmd!("/usr/bin/du", "-h {}", image.path().display())?; info!("Decompress {:?} finished in {} seconds", image.path().file_name().unwrap(), start.elapsed().as_secs()); Ok(()) +} + +fn decompress_one_image(image: ResourceImage) -> JoinHandle> { + thread::spawn(move || { + decompress_one_image_sync(image) }) } diff --git a/citadel-tool/src/boot/rootfs.rs b/citadel-tool/src/boot/rootfs.rs index 55b853c..3a74cb7 100644 --- a/citadel-tool/src/boot/rootfs.rs +++ b/citadel-tool/src/boot/rootfs.rs @@ -23,7 +23,7 @@ pub fn setup_rootfs_resource(rootfs: &ResourceImage) -> Result<()> { fn setup_resource_unverified(img: &ResourceImage) -> Result<()> { if img.is_compressed() { - img.decompress()?; + img.decompress(false)?; } let loopdev = LoopDevice::create(img.path(), Some(4096), true)?; info!("Loop device created: {}", loopdev); diff --git a/citadel-tool/src/image/mod.rs b/citadel-tool/src/image/mod.rs index 46de9e4..884ed12 100644 --- a/citadel-tool/src/image/mod.rs +++ b/citadel-tool/src/image/mod.rs @@ -275,7 +275,7 @@ fn decompress(arg_matches: &ArgMatches) -> Result<()> { if !img.is_compressed() { info!("Image is not compressed, not decompressing."); } else { - img.decompress()?; + img.decompress(false)?; } Ok(()) } diff --git a/citadel-tool/src/update/mod.rs b/citadel-tool/src/update/mod.rs index 4e992e9..69844ae 100644 --- a/citadel-tool/src/update/mod.rs +++ b/citadel-tool/src/update/mod.rs @@ -122,7 +122,7 @@ fn install_image(path: &Path, flags: u32) -> Result<()> { // dmverity hash tree. fn prepare_image(image: &ResourceImage, flags: u32) -> Result<()> { if image.is_compressed() { - image.decompress()?; + image.decompress(false)?; } if flags & FLAG_SKIP_SHA == 0 { diff --git a/libcitadel/src/exec.rs b/libcitadel/src/exec.rs index 55313b6..7a02c1d 100644 --- a/libcitadel/src/exec.rs +++ b/libcitadel/src/exec.rs @@ -54,7 +54,7 @@ impl Exec { .map_err(context!("failed to execute command {}", self.cmd_name))?; for line in BufReader::new(result.stderr.as_slice()).lines() { - verbose!(" {}", line.unwrap()); + warn!(" {}", line.unwrap()); } self.check_cmd_status(result.status) } diff --git a/libcitadel/src/resource.rs b/libcitadel/src/resource.rs index e1d0081..5d744ba 100644 --- a/libcitadel/src/resource.rs +++ b/libcitadel/src/resource.rs @@ -138,7 +138,7 @@ impl ResourceImage { self.header.has_flag(ImageHeader::FLAG_HASH_TREE) } - pub fn decompress(&self) -> Result<()> { + pub fn decompress(&self, early_remove: bool) -> Result<()> { if !self.is_compressed() { return Ok(()) } @@ -154,6 +154,10 @@ impl ResourceImage { io::copy(&mut reader, &mut out) .map_err(context!("error copying image file {:?} to temporary file", self.path()))?; + if early_remove { + util::remove_file(self.path())?; + } + util::xz_decompress(xzfile)?; let tmpfile = self.path.with_extension("tmp"); util::rename(&tmpfile, self.path())?; @@ -218,7 +222,7 @@ impl ResourceImage { return Ok(()) } if self.is_compressed() { - self.decompress()?; + self.decompress(false)?; } info!("Generating dm-verity hash tree for image {}", self.path.display()); let verity = self.verity()?; @@ -239,7 +243,7 @@ impl ResourceImage { pub fn generate_shasum(&self) -> Result { if self.is_compressed() { - self.decompress()?; + self.decompress(false)?; } info!("Calculating sha256 of image"); let output = util::exec_cmdline_pipe_input("sha256sum", "-", self.path(), util::FileRange::Range{offset: 4096, len: self.metainfo().nblocks() * 4096}) @@ -256,7 +260,7 @@ impl ResourceImage { info!("loop mounting image to {} (noverity)", self.mount_path().display()); if self.is_compressed() { - self.decompress()?; + self.decompress(false)?; } let loopdev = LoopDevice::create(self.path(), Some(4096), true)?;