From 53a5bce2a204035c387898328263a0d4f333decb Mon Sep 17 00:00:00 2001 From: Bruce Leidl Date: Thu, 11 Jan 2024 08:53:48 -0500 Subject: [PATCH] Replace our memory manager implementation with vm-memory. This is a large intrusive refactor but it will make it easier to adopt other components from rust-vmm ecosystem in the future. Our old implementation was also potentially problematic due to an API which directly hands out [u8] slices to guest memory. (See VolatileSlice and 'volatile_memory' module in vm-memory for discussion about this issue) Since the old memory manager also handled graphics memory buffers this was rewritten into a separate component and moved into 'io' module. Uses of FileDesc class were removed and replaced by use of the standard File type instead which accomplishes most of the same goals. MemoryFd was removed and replaced with memfd crate. --- Cargo.lock | 249 ++++++++++++++++++-- Cargo.toml | 3 + src/audio/pulse/client.rs | 19 +- src/audio/pulse/context.rs | 10 +- src/audio/pulse/stream.rs | 16 +- src/devices/ac97/ac97.rs | 12 +- src/devices/ac97/ac97_bus_master.rs | 27 ++- src/devices/virtio_9p/file.rs | 46 +++- src/devices/virtio_9p/mod.rs | 8 +- src/devices/virtio_9p/pdu.rs | 12 +- src/devices/virtio_9p/server.rs | 24 +- src/devices/virtio_block.rs | 7 +- src/devices/virtio_rng.rs | 4 +- src/devices/virtio_wl/device.rs | 26 ++- src/devices/virtio_wl/mod.rs | 31 +-- src/devices/virtio_wl/pipe.rs | 28 ++- src/devices/virtio_wl/shm.rs | 53 ++--- src/devices/virtio_wl/socket.rs | 33 +-- src/devices/virtio_wl/vfd.rs | 107 +++++---- src/disk/memory.rs | 41 ++-- src/disk/mod.rs | 9 +- src/disk/raw.rs | 19 +- src/disk/realmfs.rs | 5 +- src/{memory => io}/address.rs | 0 src/io/manager.rs | 43 ++-- src/io/mod.rs | 6 +- src/io/pci/config.rs | 23 +- src/io/pci/mod.rs | 3 +- src/io/shm_mapper.rs | 341 ++++++++++++++++++++++++++++ src/io/virtio/device.rs | 8 +- src/io/virtio/queues.rs | 24 +- src/io/virtio/vq/chain.rs | 37 ++- src/io/virtio/vq/descriptor.rs | 22 +- src/io/virtio/vq/splitqueue.rs | 53 ++--- src/io/virtio/vq/virtqueue.rs | 4 +- src/lib.rs | 1 - src/memory/allocator.rs | 93 -------- src/memory/manager.rs | 159 ------------- src/memory/mmap.rs | 172 -------------- src/memory/mod.rs | 47 ---- src/memory/ram.rs | 145 ------------ src/{memory => system}/drm.rs | 32 ++- src/system/filedesc.rs | 143 ------------ src/system/memfd.rs | 76 ------- src/system/mod.rs | 22 +- src/vm/arch/error.rs | 7 +- src/vm/arch/mod.rs | 4 +- src/vm/arch/x86/kernel.rs | 27 ++- src/vm/arch/x86/memory.rs | 62 ++--- src/vm/arch/x86/mptable.rs | 7 +- src/vm/arch/x86/setup.rs | 55 ++--- src/vm/mod.rs | 2 +- src/vm/setup.rs | 18 +- 53 files changed, 1087 insertions(+), 1338 deletions(-) rename src/{memory => io}/address.rs (100%) create mode 100644 src/io/shm_mapper.rs delete mode 100644 src/memory/allocator.rs delete mode 100644 src/memory/manager.rs delete mode 100644 src/memory/mmap.rs delete mode 100644 src/memory/mod.rs delete mode 100644 src/memory/ram.rs rename src/{memory => system}/drm.rs (86%) delete mode 100644 src/system/filedesc.rs delete mode 100644 src/system/memfd.rs diff --git a/Cargo.lock b/Cargo.lock index a97ff5e..c416550 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a1eca3195b729bbd64e292ef2f5fff6b1c28504fed762ce2b1013dde4d8e92" +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "backtrace" version = "0.3.37" @@ -48,9 +54,15 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.1.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "byteorder" @@ -89,6 +101,16 @@ dependencies = [ "libdbus-sys", ] +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "failure" version = "0.1.5" @@ -135,7 +157,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24e40d6fd5d64e2082e0c796495c8ef5ad667a96d03e5aaa0becfd9d47bcbfb8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "inotify-sys", "libc", ] @@ -177,9 +199,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.62" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libcitadel" @@ -223,6 +245,33 @@ dependencies = [ "take_mut", ] +[[package]] +name = "libpulse-binding" +version = "2.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3557a2dfc380c8f061189a01c6ae7348354e0c9886038dc6c171219c08eaff" +dependencies = [ + "bitflags 1.3.2", + "libc", + "libpulse-sys", + "num-derive", + "num-traits", + "winapi", +] + +[[package]] +name = "libpulse-sys" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc19e110fbf42c17260d30f6d3dc545f58491c7830d38ecb9aaca96e26067a9b" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "pkg-config", + "winapi", +] + [[package]] name = "libsodium-sys" version = "0.2.4" @@ -237,19 +286,54 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "memfd" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +dependencies = [ + "rustix", +] + [[package]] name = "nix" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "319fffb13b63c0f4ff5a4e1c97566e7e741561ff5d03bf8bbf11653454bbd70b" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cc", "cfg-if", "libc", "void", ] +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.107", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + [[package]] name = "ph" version = "0.1.0" @@ -260,17 +344,21 @@ dependencies = [ "lazy_static", "libc", "libcitadel", + "libpulse-binding", + "memfd", "signal-hook", "termios", "thiserror", + "vm-allocator", + "vm-memory", "vmm-sys-util", ] [[package]] name = "pkg-config" -version = "0.3.16" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d5370d90f49f70bd033c3d75e87fc529fbfff9d6f7cccef07d6170079d91ea" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "proc-macro2" @@ -283,9 +371,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -301,11 +389,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.2" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.50", + "proc-macro2 1.0.69", ] [[package]] @@ -326,6 +414,19 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rustix" +version = "0.38.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "same-file" version = "1.0.5" @@ -347,8 +448,8 @@ version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b133a43a1ecd55d4086bd5b4dc6c1751c68b1bfbeba7a5040442022c7e7c02e" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.2", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.107", ] @@ -400,8 +501,19 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.2", + "proc-macro2 1.0.69", + "quote 1.0.33", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", "unicode-ident", ] @@ -446,22 +558,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.2", - "syn 1.0.107", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.38", ] [[package]] @@ -491,13 +603,34 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33dd455d0f96e90a75803cfeb7f948768c08d70a6de9a8d2362461935698bf95" +[[package]] +name = "vm-allocator" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "565b6886b7dd1b3bf34ec9243d90a97db4f2a83c2416caa52fcc95fd255d45e4" +dependencies = [ + "libc", + "thiserror", +] + +[[package]] +name = "vm-memory" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5376c9ee5ebe2103a310d8241936cfb93c946734b0479a4fa5bdf7a64abbacd8" +dependencies = [ + "libc", + "thiserror", + "winapi", +] + [[package]] name = "vmm-sys-util" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd64fe09d8e880e600c324e7d664760a17f56e9672b7495a86381b49e4f72f46" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", ] @@ -549,6 +682,72 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "xattr" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index faa6581..3babd79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,10 @@ lazy_static = "1.4.0" signal-hook = "0.1.10" thiserror = "1.0" vmm-sys-util = "0.11.1" +vm-memory = { version = "0.13.1", features = ["backend-mmap"] } +vm-allocator = "0.1.0" kvm-ioctls = "0.12.0" kvm-bindings = "0.6.0" +memfd = "0.6.4" pulse = { version = "2.27.1", package = "libpulse-binding" } libcitadel = { git = "https://github.com/brl/citadel-tools", rev="44d5ce660f1f5cf8a3ad1060b143926a99be5148" } diff --git a/src/audio/pulse/client.rs b/src/audio/pulse/client.rs index 94c666e..695ece8 100644 --- a/src/audio/pulse/client.rs +++ b/src/audio/pulse/client.rs @@ -1,27 +1,30 @@ use std::sync::mpsc; use std::thread; use pulse::sample::{Format, Spec}; +use vm_memory::GuestMemoryMmap; use crate::audio::pulse::context::PulseContext; use crate::audio::pulse::message::PulseMessageChannel; use crate::audio::pulse::Result; use crate::audio::{SampleFormat, StreamDirection}; use crate::audio::shm_streams::{GenericResult, NullShmStream, ShmStream, ShmStreamSource}; -use crate::memory::GuestRam; pub struct PulseClient { channel: PulseMessageChannel, } impl PulseClient { - pub fn connect(guest_ram: GuestRam) -> Result { + pub fn connect(guest_memory: &GuestMemoryMmap) -> Result { let (tx,rx) = mpsc::channel(); - let _ = thread::spawn(move || { - let mut ctx = PulseContext::new(guest_ram); - if let Err(err) = ctx.connect() { - warn!("PulseAudio Error: {}", err); - } else { - ctx.run(rx); + let _ = thread::spawn({ + let guest_memory = guest_memory.clone(); + move || { + let mut ctx = PulseContext::new(guest_memory); + if let Err(err) = ctx.connect() { + warn!("PulseAudio Error: {}", err); + } else { + ctx.run(rx); + } } }); Ok(PulseClient { diff --git a/src/audio/pulse/context.rs b/src/audio/pulse/context.rs index e66fd98..371b516 100644 --- a/src/audio/pulse/context.rs +++ b/src/audio/pulse/context.rs @@ -7,12 +7,12 @@ use pulse::mainloop::threaded::Mainloop; use pulse::proplist::{properties, Proplist}; use pulse::sample::Spec; use pulse::stream::Stream; -use crate::memory::GuestRam; +use vm_memory::GuestMemoryMmap; use crate::audio::pulse::{Result, PulseError, PulseStream}; use crate::audio::pulse::message::{PulseContextMessage, PulseContextRequest, PulseMessageChannel}; pub struct PulseContext { - guest_ram: GuestRam, + guest_memory: GuestMemoryMmap, mainloop: Rc>, context: Rc>, } @@ -34,7 +34,7 @@ impl PulseContext { self.mainloop.clone() } - pub fn new(guest_ram: GuestRam) -> Self { + pub fn new(guest_memory: GuestMemoryMmap) -> Self { let mainloop = Mainloop::new() .expect("Failed to create a pulseaudio mainloop"); @@ -51,7 +51,7 @@ impl PulseContext { ).expect("Failed to create a pulseaudio context"); PulseContext { - guest_ram, + guest_memory, mainloop: Rc::new(RefCell::new(mainloop)), context: Rc::new(RefCell::new(context)), } @@ -111,7 +111,7 @@ impl PulseContext { None) .expect("Failed to create pulseaudio stream"); - let ps = PulseStream::new_playback(stream, self.guest_ram.clone(), spec, buffer_size, channel); + let ps = PulseStream::new_playback(stream, self.guest_memory.clone(), spec, buffer_size, channel); self.mainloop_unlock(); ps } diff --git a/src/audio/pulse/stream.rs b/src/audio/pulse/stream.rs index 7ecfbc8..57019c5 100644 --- a/src/audio/pulse/stream.rs +++ b/src/audio/pulse/stream.rs @@ -2,12 +2,11 @@ use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::time::Duration; use pulse::sample::Spec; use pulse::stream::{FlagSet, SeekMode, State, Stream}; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; use crate::audio::pulse::{PulseError,Result}; use crate::audio::pulse::context::PulseContext; use crate::audio::pulse::message::PulseMessageChannel; use crate::audio::shm_streams::{BufferSet, GenericResult, ServerRequest, ShmStream}; -use crate::memory::GuestRam; - struct Available { byte_count: Mutex, cond: Condvar, @@ -52,7 +51,7 @@ impl Available { pub struct PulseStream { spec: Spec, buffer_size: usize, - guest_ram: GuestRam, + guest_memory: GuestMemoryMmap, stream: Arc>, avail: Arc, channel: PulseMessageChannel, @@ -105,7 +104,7 @@ impl PulseStream { result } - pub fn new_playback(mut stream: Stream, guest_ram: GuestRam, spec: Spec, buffer_size: usize, channel: PulseMessageChannel) -> Self { + pub fn new_playback(mut stream: Stream, guest_memory: GuestMemoryMmap, spec: Spec, buffer_size: usize, channel: PulseMessageChannel) -> Self { let avail = Arc::new(Available::new()); stream.set_write_callback(Some(Box::new({ @@ -119,7 +118,7 @@ impl PulseStream { PulseStream { spec, buffer_size, - guest_ram, + guest_memory, avail, stream, channel, @@ -166,12 +165,13 @@ impl ShmStream for PulseStream { impl BufferSet for PulseStream { fn callback(&self, address: u64, frames: usize) -> GenericResult<()> { self.uncork()?; - let bytes = self.guest_ram.slice(address, frames * self.frame_size())?; + let mut buffer = vec![0u8; frames * self.frame_size()]; + self.guest_memory.read_slice(&mut buffer, GuestAddress(address))?; self.channel.send_mainloop_lock()?; - self.stream().write_copy(bytes, 0, SeekMode::Relative)?; + self.stream().write_copy(&buffer, 0, SeekMode::Relative)?; self.channel.send_mainloop_unlock()?; - self.avail.decrement(bytes.len()); + self.avail.decrement(buffer.len()); Ok(()) } diff --git a/src/devices/ac97/ac97.rs b/src/devices/ac97/ac97.rs index 6c0d466..2db817c 100644 --- a/src/devices/ac97/ac97.rs +++ b/src/devices/ac97/ac97.rs @@ -5,13 +5,13 @@ use std::io; use thiserror::Error; +use vm_memory::GuestMemoryMmap; use crate::audio::pulse::{PulseClient, PulseError}; use crate::devices::ac97::ac97_bus_master::{Ac97BusMaster, AudioStreamSource}; use crate::devices::ac97::ac97_mixer::Ac97Mixer; use crate::devices::ac97::ac97_regs::{MASTER_REGS_SIZE, MIXER_REGS_SIZE}; use crate::devices::irq_event::IrqLevelEvent; use crate::io::pci::{PciBar, PciBarAllocation, PciConfiguration, PciDevice}; -use crate::memory::GuestRam; use crate::vm::KvmVm; @@ -48,7 +48,7 @@ impl Ac97Dev { /// default values. pub fn new( irq: u8, - mem: GuestRam, + mem: &GuestMemoryMmap, audio_server: AudioStreamSource, ) -> Self { let pci_config = PciConfiguration::new(irq, PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_5, PCI_CLASS_MULTIMEDIA_AUDIO); @@ -57,7 +57,7 @@ impl Ac97Dev { irq, pci_config, bus_master: Ac97BusMaster::new( - mem, + mem.clone(), audio_server, ), mixer: Ac97Mixer::new(), @@ -69,7 +69,7 @@ impl Ac97Dev { pub fn try_new( kvm_vm: &KvmVm, irq: u8, - mem: GuestRam, + mem: &GuestMemoryMmap, ) -> Result { let mut ac97 = Self::initialize_pulseaudio(irq, mem)?; let irq_event = IrqLevelEvent::register(kvm_vm, irq) @@ -78,8 +78,8 @@ impl Ac97Dev { Ok(ac97) } - fn initialize_pulseaudio(irq: u8, mem: GuestRam) -> Result { - let server = PulseClient::connect(mem.clone()) + fn initialize_pulseaudio(irq: u8, mem: &GuestMemoryMmap) -> Result { + let server = PulseClient::connect(mem) .map_err(Ac97Error::PulseError)?; Ok(Self::new( irq, diff --git a/src/devices/ac97/ac97_bus_master.rs b/src/devices/ac97/ac97_bus_master.rs index a6e58e6..1d1c409 100644 --- a/src/devices/ac97/ac97_bus_master.rs +++ b/src/devices/ac97/ac97_bus_master.rs @@ -13,13 +13,12 @@ use std::fmt::{Debug, Formatter}; use std::time::{Duration, Instant}; use thiserror::Error; +use vm_memory::{Bytes, guest_memory, GuestAddress, GuestMemoryMmap}; use crate::audio::shm_streams::{ShmStream, ShmStreamSource}; use crate::audio::{BoxError, SampleFormat, StreamControl, StreamDirection}; use crate::devices::ac97::ac97_mixer::Ac97Mixer; use crate::devices::ac97::ac97_regs::*; use crate::devices::irq_event::IrqLevelEvent; -use crate::memory::GuestRam; -use crate::system; const INPUT_SAMPLE_RATE: u32 = 48000; const DEVICE_INPUT_CHANNEL_COUNT: usize = 2; @@ -100,7 +99,7 @@ impl Ac97BusMasterRegs { pub(crate) enum GuestMemoryError { // Failure getting the address of the audio buffer. #[error("Failed to get the address of the audio buffer: {0}.")] - ReadingGuestBufferAddress(system::Error), + ReadingGuestBufferAddress(guest_memory::Error), } #[derive(Error, Debug)] @@ -178,7 +177,7 @@ impl AudioThreadInfo { /// interface compliant with the ICH bus master. pub struct Ac97BusMaster { // Keep guest memory as each function will use it for buffer descriptors. - mem: GuestRam, + mem: GuestMemoryMmap, regs: Arc>, acc_sema: u8, @@ -198,7 +197,7 @@ impl Ac97BusMaster { /// Creates an Ac97BusMaster` object that plays audio from `mem` to streams provided by /// `audio_server`. - pub fn new(mem: GuestRam, audio_server: AudioStreamSource) -> Self { + pub fn new(mem: GuestMemoryMmap, audio_server: AudioStreamSource) -> Self { Ac97BusMaster { mem, regs: Arc::new(Mutex::new(Ac97BusMasterRegs::new())), @@ -597,12 +596,12 @@ impl Ac97BusMaster { fn get_buffer_samples( func_regs: &Ac97FunctionRegs, - mem: &GuestRam, + mem: &GuestMemoryMmap, index: u8, ) -> GuestMemoryResult { let descriptor_addr = func_regs.bdbar + u32::from(index) * DESCRIPTOR_LENGTH as u32; let control_reg: u32 = mem - .read_int(u64::from(descriptor_addr) + 4) + .read_obj(GuestAddress(u64::from(descriptor_addr) + 4)) .map_err(GuestMemoryError::ReadingGuestBufferAddress)?; let buffer_samples = control_reg as usize & 0x0000_ffff; Ok(buffer_samples) @@ -612,14 +611,14 @@ fn get_buffer_samples( // function and registers. fn buffer_completed( regs: &mut Ac97BusMasterRegs, - mem: &GuestRam, + mem: &GuestMemoryMmap, func: Ac97Function, ) -> AudioResult<()> { // check if the completed descriptor wanted an interrupt on completion. let civ = regs.func_regs(func).civ; let descriptor_addr = regs.func_regs(func).bdbar + u32::from(civ) * DESCRIPTOR_LENGTH as u32; let control_reg: u32 = mem - .read_int(u64::from(descriptor_addr) + 4) + .read_obj(GuestAddress(u64::from(descriptor_addr) + 4)) .map_err(GuestMemoryError::ReadingGuestBufferAddress)?; let mut new_sr = regs.func_regs(func).sr & !SR_CELV; @@ -688,7 +687,7 @@ fn update_sr(regs: &mut Ac97BusMasterRegs, func: Ac97Function, val: u16) { // Returns the size in samples of the buffer pointed to by the CIV register. fn current_buffer_size( func_regs: &Ac97FunctionRegs, - mem: &GuestRam, + mem: &GuestMemoryMmap, ) -> GuestMemoryResult { let civ = func_regs.civ; get_buffer_samples(func_regs, mem, civ) @@ -747,12 +746,12 @@ impl Debug for GuestBuffer { fn get_buffer_address( func_regs: &Ac97FunctionRegs, - mem: &GuestRam, + mem: &GuestMemoryMmap, index: u8, ) -> GuestMemoryResult { let descriptor_addr = func_regs.bdbar + u32::from(index) * DESCRIPTOR_LENGTH as u32; let buffer_addr_reg: u32 = mem - .read_int(u64::from(descriptor_addr)) + .read_obj(GuestAddress(u64::from(descriptor_addr))) .map_err(GuestMemoryError::ReadingGuestBufferAddress)?; let buffer_addr = (buffer_addr_reg & !0x03u32) as u64; // The address must be aligned to four bytes. Ok(buffer_addr) @@ -765,7 +764,7 @@ fn get_buffer_address( // `civ + offset == LVI and the CELV flag is set. fn next_guest_buffer( regs: &Ac97BusMasterRegs, - mem: &GuestRam, + mem: &GuestMemoryMmap, func: Ac97Function, offset: usize, ) -> AudioResult> { @@ -812,7 +811,7 @@ fn next_guest_buffer( struct AudioWorker { func: Ac97Function, regs: Arc>, - mem: GuestRam, + mem: GuestMemoryMmap, thread_run: Arc, lvi_semaphore: Arc, message_interval: Duration, diff --git a/src/devices/virtio_9p/file.rs b/src/devices/virtio_9p/file.rs index b138e7e..20659e1 100644 --- a/src/devices/virtio_9p/file.rs +++ b/src/devices/virtio_9p/file.rs @@ -5,13 +5,13 @@ use std::path::{Path, PathBuf, Component}; use std::fs::{Metadata, File}; use std::os::unix::io::{RawFd,AsRawFd}; use std::os::linux::fs::MetadataExt; -use std::os::unix::fs::FileExt; use crate::devices::virtio_9p::{ pdu::PduParser, directory::Directory, filesystem::FileSystemOps, }; -use std::io::{Cursor, SeekFrom, Seek, Read}; +use std::io::{Cursor, SeekFrom, Seek}; use std::sync::{RwLock, Arc}; +use vm_memory::{ReadVolatile, VolatileSlice, WriteVolatile}; pub const P9_DOTL_RDONLY: u32 = 0o00000000; pub const P9_DOTL_WRONLY: u32 = 0o00000001; @@ -59,12 +59,14 @@ impl > Buffer { Buffer(Arc::new(RwLock::new(Cursor::new(bytes)))) } - pub fn read_at(&self, buffer: &mut [u8], offset: u64) -> io::Result { + pub fn read_at(&self, buffer: &mut VolatileSlice, offset: u64) -> io::Result { let mut lock = self.0.write().unwrap(); lock.seek(SeekFrom::Start(offset))?; - lock.read(buffer) + lock.read_volatile(buffer) + .map_err(io::Error::other) } - pub fn write_at(&self, _buffer: &[u8], _offset: u64) -> io::Result { + + pub fn write_at(&self, _buffer: &VolatileSlice, _offset: u64) -> io::Result { return Err(io::Error::from_raw_os_error(libc::EPERM)) } @@ -121,17 +123,31 @@ impl P9File { } } - pub fn read_at(&self, buffer: &mut [u8], offset: u64) -> io::Result { + pub fn read_at(&mut self, buffer: &mut VolatileSlice, offset: u64) -> io::Result { match self.file { - FileObject::File(ref f) => f.read_at(buffer,offset), + FileObject::File(ref mut f) => { + let pos = f.stream_position()?; + f.seek(SeekFrom::Start(offset))?; + let result = f.read_volatile(buffer) + .map_err(io::Error::other); + f.seek(SeekFrom::Start(pos))?; + result + }, FileObject::BufferFile(ref f) => f.read_at(buffer, offset), FileObject::NotAFile => Ok(0), } } - pub fn write_at(&self, buffer: &[u8], offset: u64) -> io::Result { + pub fn write_at(&mut self, buffer: &VolatileSlice, offset: u64) -> io::Result { match self.file { - FileObject::File(ref f) => f.write_at(buffer,offset), + FileObject::File(ref mut f) => { + let pos = f.stream_position()?; + f.seek(SeekFrom::Start(offset))?; + let result = f.write_volatile(buffer) + .map_err(io::Error::other); + f.seek(SeekFrom::Start(pos))?; + result + }, FileObject::BufferFile(ref f) => f.write_at(buffer, offset), FileObject::NotAFile => Ok(0), } @@ -328,6 +344,11 @@ impl Fids { self.fid(id) } + pub fn read_fid_mut(&mut self, pp: &mut PduParser) -> io::Result<&mut Fid> { + let id = pp.r32()?; + self.fid_mut(id) + } + pub fn read_new_path(&self, pp: &mut PduParser) -> io::Result { let fid = self.read_fid(pp)?; let name = pp.read_string()?; @@ -435,6 +456,13 @@ impl Fid { } } + pub fn file_mut(&mut self) -> io::Result<&mut P9File> { + match self.file.as_mut() { + Some(file) => Ok(file), + None => system_error(libc::EBADF), + } + } + pub fn join_name(&self, root: &Path, name: &str) -> io::Result { Self::path_join_name(self.qid, self.path(), root, name) } diff --git a/src/devices/virtio_9p/mod.rs b/src/devices/virtio_9p/mod.rs index a1c5445..a0e4c60 100644 --- a/src/devices/virtio_9p/mod.rs +++ b/src/devices/virtio_9p/mod.rs @@ -1,8 +1,8 @@ use std::thread; use std::path::{PathBuf, Path}; +use vm_memory::GuestMemoryMmap; -use crate::memory::GuestRam; use crate::devices::virtio_9p::server::Server; use crate::devices::virtio_9p::filesystem::{FileSystem, FileSystemOps}; use self::pdu::PduParser; @@ -85,13 +85,13 @@ impl VirtioDevice for VirtioP9 { let vq = queues.get_queue(0); let root_dir = self.root_dir.clone(); let filesystem = self.filesystem.clone(); - let ram = queues.memory().guest_ram().clone(); + let memory = queues.guest_memory().clone(); let debug = self.debug; - thread::spawn(move || run_device(ram, vq, &root_dir, filesystem, debug)); + thread::spawn(move || run_device(memory, vq, &root_dir, filesystem, debug)); } } -fn run_device(memory: GuestRam, vq: VirtQueue, root_dir: &Path, filesystem: T, debug: bool) { +fn run_device(memory: GuestMemoryMmap, vq: VirtQueue, root_dir: &Path, filesystem: T, debug: bool) { let mut server = Server::new(&root_dir, filesystem); if debug { diff --git a/src/devices/virtio_9p/pdu.rs b/src/devices/virtio_9p/pdu.rs index 1949503..e33518e 100644 --- a/src/devices/virtio_9p/pdu.rs +++ b/src/devices/virtio_9p/pdu.rs @@ -4,16 +4,16 @@ use std::ffi::OsStr; use libc; use byteorder::{LittleEndian,ReadBytesExt,WriteBytesExt}; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; use crate::devices::virtio_9p::file::Qid; use crate::io::Chain; -use crate::memory::GuestRam; const P9_HEADER_LEN: usize = 7; const P9_RLERROR: u8 = 7; pub struct PduParser<'a> { - memory: GuestRam, + memory: GuestMemoryMmap, pub chain: &'a mut Chain, size: u32, @@ -107,7 +107,7 @@ impl P9Attr { } impl <'a> PduParser<'a> { - pub fn new(chain: &'a mut Chain, memory: GuestRam) -> PduParser<'a> { + pub fn new(chain: &'a mut Chain, memory: GuestMemoryMmap) -> PduParser<'a> { PduParser{ memory, chain, size: 0, cmd: 0, tag: 0, reply_start_addr: 0 } } @@ -180,7 +180,7 @@ impl <'a> PduParser<'a> { } pub fn _w8_at(&self, offset: usize, val: u8) { - self.memory.write_int::(self.reply_start_addr + offset as u64, val).unwrap(); + self.memory.write_obj::(val, GuestAddress(self.reply_start_addr + offset as u64)).unwrap(); } #[allow(dead_code)] @@ -189,7 +189,7 @@ impl <'a> PduParser<'a> { } pub fn _w16_at(&self, offset: usize, val: u16) { - self.memory.write_int::(self.reply_start_addr + offset as u64, val).unwrap(); + self.memory.write_obj::(val, GuestAddress(self.reply_start_addr + offset as u64)).unwrap(); } pub fn w32_at(&self, offset: usize, val: u32) { @@ -197,7 +197,7 @@ impl <'a> PduParser<'a> { } pub fn _w32_at(&self, offset: usize, val: u32) { - self.memory.write_int::(self.reply_start_addr + offset as u64, val).unwrap(); + self.memory.write_obj::(val, GuestAddress(self.reply_start_addr + offset as u64)).unwrap(); } pub fn write_done(&mut self) -> io::Result<()> { diff --git a/src/devices/virtio_9p/server.rs b/src/devices/virtio_9p/server.rs index 9b43901..811b719 100644 --- a/src/devices/virtio_9p/server.rs +++ b/src/devices/virtio_9p/server.rs @@ -633,8 +633,8 @@ impl Server { pp.write_done() } - fn p9_read_args(&self, pp: &mut PduParser) -> io::Result<(&Fid, u64, u32)> { - let fid = self.read_fid(pp)?; + fn p9_read_args(&mut self, pp: &mut PduParser) -> io::Result<(&mut Fid, u64, u32)> { + let fid = self.fids.read_fid_mut(pp)?; let offset = pp.r64()?; let count = pp.r32()?; pp.read_done()?; @@ -642,13 +642,14 @@ impl Server { } fn p9_read(&mut self, pp: &mut PduParser) -> io::Result<()> { + let debug = self.debug; let (fid, offset, count) = self.p9_read_args(pp)?; - if self.debug { + if debug { notify!("p9_read({}, offset={}, count={})", fid, offset, count); } - let file = fid.file()?; + let file = fid.file_mut()?; // space for size field pp.w32(0)?; @@ -660,7 +661,8 @@ impl Server { break; } let rlen = cmp::min(current.len(), count as usize); - let n = file.read_at(&mut current[..rlen], offset + nread as u64)?; + let mut subslice = current.subslice(0, rlen).map_err(io::Error::other)?; + let n = file.read_at(&mut subslice, offset + nread as u64)?; if n == 0 { break; } @@ -671,24 +673,26 @@ impl Server { pp.write_done() } - fn p9_write_args(&self, pp: &mut PduParser) -> io::Result<(&Fid, u64, u32)> { - let fid = self.read_fid(pp)?; + fn p9_write_args(&mut self, pp: &mut PduParser) -> io::Result<(&mut Fid, u64, u32)> { + let fid = self.fids.read_fid_mut(pp)?; let offset = pp.r64()?; let count = pp.r32()?; Ok((fid, offset, count)) } fn p9_write(&mut self, pp: &mut PduParser) -> io::Result<()> { + let debug = self.debug; let (fid, offset, count) = self.p9_write_args(pp)?; - if self.debug { + if debug { notify!("p9_write({}, offset={}, count={})", fid, offset, count); } - let file = fid.file()?; + let file = fid.file_mut()?; let mut nread = 0; while nread < count { - let n = file.write_at(pp.chain.current_read_slice(), offset + nread as u64)?; + let buffer = pp.chain.current_read_slice(); + let n = file.write_at(&buffer, offset + nread as u64)?; if n == 0 { break; } diff --git a/src/devices/virtio_block.rs b/src/devices/virtio_block.rs index 0794cfc..8f6dc74 100644 --- a/src/devices/virtio_block.rs +++ b/src/devices/virtio_block.rs @@ -199,9 +199,10 @@ impl <'a,'b, D: DiskImage> MessageHandler<'a,'b, D> { return Ok(()) } let len = nsectors << SECTOR_SHIFT; - let buffer = &mut current[..len]; + let mut buffer = current.subslice(0, len) + .map_err(io::Error::other)?; - self.disk.read_sectors(self.sector, buffer) + self.disk.read_sectors(self.sector, &mut buffer) .map_err(Error::DiskRead)?; self.chain.inc_write_offset(len); self.sector += nsectors as u64; @@ -218,7 +219,7 @@ impl <'a,'b, D: DiskImage> MessageHandler<'a,'b, D> { if nsectors == 0 { return Ok(()) } - self.disk.write_sectors(self.sector, current) + self.disk.write_sectors(self.sector, ¤t) .map_err(Error::DiskWrite)?; self.chain.inc_read_offset(nsectors << SECTOR_SHIFT); diff --git a/src/devices/virtio_rng.rs b/src/devices/virtio_rng.rs index 846d062..d4302c2 100644 --- a/src/devices/virtio_rng.rs +++ b/src/devices/virtio_rng.rs @@ -16,12 +16,12 @@ impl VirtioRandom { } fn run(q: VirtQueue) { - let random = File::open("/dev/urandom").unwrap(); + let mut random = File::open("/dev/urandom").unwrap(); loop { q.on_each_chain(|mut chain| { while !chain.is_end_of_chain() { - let _ = chain.copy_from_reader(&random, 256).unwrap(); + let _ = chain.copy_from_reader(&mut random, 256).unwrap(); } }); } diff --git a/src/devices/virtio_wl/device.rs b/src/devices/virtio_wl/device.rs index ff5210d..ac359cc 100644 --- a/src/devices/virtio_wl/device.rs +++ b/src/devices/virtio_wl/device.rs @@ -3,13 +3,14 @@ use std::thread; use crate::system; use crate::system::EPoll; -use crate::memory::{MemoryManager, DrmDescriptor}; +use crate::system::drm::DrmDescriptor; use crate::devices::virtio_wl::{vfd::VfdManager, consts::*, Error, Result, VfdObject}; use crate::system::ioctl::ioctl_with_ref; use std::os::raw::{c_ulong, c_uint, c_ulonglong}; use vmm_sys_util::eventfd::EventFd; use crate::io::{Chain, FeatureBits, Queues, VirtioDevice, VirtioDeviceType, VirtQueue}; +use crate::io::shm_mapper::DeviceSharedMemoryManager; #[repr(C)] struct dma_buf_sync { @@ -19,14 +20,16 @@ const DMA_BUF_IOCTL_BASE: c_uint = 0x62; const DMA_BUF_IOCTL_SYNC: c_ulong = iow!(DMA_BUF_IOCTL_BASE, 0, ::std::mem::size_of::() as i32); pub struct VirtioWayland { + dev_shm_manager: Option, features: FeatureBits, enable_dmabuf: bool, } impl VirtioWayland { - pub fn new(enable_dmabuf: bool) -> Self { + pub fn new(enable_dmabuf: bool , dev_shm_manager: DeviceSharedMemoryManager) -> Self { let features = FeatureBits::new_default(VIRTIO_WL_F_TRANS_FLAGS as u64); VirtioWayland { + dev_shm_manager: Some(dev_shm_manager), features, enable_dmabuf } @@ -36,9 +39,9 @@ impl VirtioWayland { self.features.has_guest_bit(VIRTIO_WL_F_TRANS_FLAGS as u64) } - fn create_device(memory: MemoryManager, in_vq: VirtQueue, out_vq: VirtQueue, transition: bool, enable_dmabuf: bool) -> Result { + fn create_device(in_vq: VirtQueue, out_vq: VirtQueue, transition: bool, enable_dmabuf: bool, dev_shm_manager: DeviceSharedMemoryManager) -> Result { let kill_evt = EventFd::new(0).map_err(Error::EventFdCreate)?; - let dev = WaylandDevice::new(memory, in_vq, out_vq, kill_evt, transition, enable_dmabuf)?; + let dev = WaylandDevice::new(in_vq, out_vq, kill_evt, transition, enable_dmabuf, dev_shm_manager)?; Ok(dev) } } @@ -60,11 +63,11 @@ impl VirtioDevice for VirtioWayland { thread::spawn({ let transition = self.transition_flags(); let enable_dmabuf = self.enable_dmabuf; + let dev_shm_manager = self.dev_shm_manager.take().expect("No dev_shm_manager"); let in_vq = queues.get_queue(0); let out_vq = queues.get_queue(1); - let memory = queues.memory().clone(); move || { - let mut dev = match Self::create_device(memory, in_vq, out_vq,transition, enable_dmabuf) { + let mut dev = match Self::create_device(in_vq, out_vq,transition, enable_dmabuf, dev_shm_manager) { Err(e) => { warn!("Error creating virtio wayland device: {}", e); return; @@ -92,8 +95,9 @@ impl WaylandDevice { const KILL_TOKEN: u64 = 2; const VFDS_TOKEN: u64 = 3; - fn new(mm: MemoryManager, in_vq: VirtQueue, out_vq: VirtQueue, kill_evt: EventFd, use_transition: bool, enable_dmabuf: bool) -> Result { - let vfd_manager = VfdManager::new(mm, use_transition, in_vq, "/run/user/1000/wayland-0")?; + fn new(in_vq: VirtQueue, out_vq: VirtQueue, kill_evt: EventFd, use_transition: bool, enable_dmabuf: bool, dev_shm_manager: DeviceSharedMemoryManager) -> Result { + let vfd_manager = VfdManager::new(dev_shm_manager, use_transition, in_vq, "/run/user/1000/wayland-0")?; + Ok(WaylandDevice { vfd_manager, out_vq, @@ -219,7 +223,7 @@ impl <'a> MessageHandler<'a> { self.chain.w32(id)?; self.chain.w32(flags)?; self.chain.w64(pfn)?; - self.chain.w32(size as u32)?; + self.chain.w32(size)?; self.responded = true; Ok(()) } @@ -306,9 +310,9 @@ impl <'a> MessageHandler<'a> { }; if let Some(fds) = send_fds.as_ref() { - vfd.send_with_fds(data, fds)?; + vfd.send_with_fds(&data, fds)?; } else { - vfd.send(data)?; + vfd.send(&data)?; } self.send_ok() } diff --git a/src/devices/virtio_wl/mod.rs b/src/devices/virtio_wl/mod.rs index 8489a2a..cb924ca 100644 --- a/src/devices/virtio_wl/mod.rs +++ b/src/devices/virtio_wl/mod.rs @@ -1,11 +1,11 @@ use std::os::unix::io::RawFd; use std::{result, io}; +use std::fs::File; use thiserror::Error; +use vm_memory::{VolatileMemoryError, VolatileSlice}; use crate::system; -use crate::memory::Error as MemError; -use crate::system::FileDesc; mod vfd; mod shm; @@ -52,18 +52,21 @@ mod consts { } pub use device::VirtioWayland; +use crate::devices::virtio_wl::shm_mapper::SharedMemoryAllocation; +use crate::io::shm_mapper; + pub type Result = result::Result; pub struct VfdRecv { buf: Vec, - fds: Option>, + fds: Option>, } impl VfdRecv { fn new(buf: Vec) -> Self { VfdRecv { buf, fds: None } } - fn new_with_fds(buf: Vec, fds: Vec) -> Self { + fn new_with_fds(buf: Vec, fds: Vec) -> Self { VfdRecv { buf, fds: Some(fds) } } } @@ -73,11 +76,11 @@ pub trait VfdObject { fn send_fd(&self) -> Option { None } fn poll_fd(&self) -> Option { None } fn recv(&mut self) -> Result> { Ok(None) } - fn send(&mut self, _data: &[u8]) -> Result<()> { Err(Error::InvalidSendVfd) } - fn send_with_fds(&mut self, _data: &[u8], _fds: &[RawFd]) -> Result<()> { Err(Error::InvalidSendVfd) } + fn send(&mut self, _data: &VolatileSlice) -> Result<()> { Err(Error::InvalidSendVfd) } + fn send_with_fds(&mut self, _data: &VolatileSlice, _fds: &[RawFd]) -> Result<()> { Err(Error::InvalidSendVfd) } fn flags(&self) -> u32; - fn pfn_and_size(&self) -> Option<(u64, u64)> { None } - fn close(&mut self) -> Result<()>; + fn shared_memory(&self) -> Option { None } + fn close(&mut self) -> Result<()> { Ok(()) } } @@ -92,9 +95,9 @@ pub enum Error { #[error("unexpected virtio wayland command: {0}")] UnexpectedCommand(u32), #[error("failed to allocate shared memory: {0}")] - ShmAllocFailed(system::Error), - #[error("failed to register memory with hypervisor: {0}")] - RegisterMemoryFailed(MemError), + ShmAllocFailed(shm_mapper::Error), + #[error("failed to free shared memory allocation: {0}")] + ShmFreeFailed(shm_mapper::Error), #[error("failed to create pipes: {0}")] CreatePipesFailed(system::Error), #[error("error reading from socket: {0}")] @@ -105,6 +108,8 @@ pub enum Error { PipeReceive(io::Error), #[error("error writing to vfd: {0}")] SendVfd(io::Error), + #[error("error writing volatile memory to vfd: {0}")] + VolatileSendVfd(VolatileMemoryError), #[error("attempt to send to incorrect vfd type")] InvalidSendVfd, #[error("message has too many vfd ids: {0}")] @@ -115,8 +120,4 @@ pub enum Error { FailedPollAdd(system::Error), #[error("error calling dma sync: {0}")] DmaSync(system::ErrnoError), - #[error("failed creating DMA buf: {0}")] - DmaBuf(MemError), - #[error("failed creating DMA buf: {0}")] - DmaBufSize(system::Error), } diff --git a/src/devices/virtio_wl/pipe.rs b/src/devices/virtio_wl/pipe.rs index cbb1777..76b7a91 100644 --- a/src/devices/virtio_wl/pipe.rs +++ b/src/devices/virtio_wl/pipe.rs @@ -1,6 +1,10 @@ -use std::os::unix::io::{AsRawFd,RawFd}; +use std::fs::File; +use std::io::Read; +use std::os::fd::FromRawFd; +use std::os::unix::io::{AsRawFd, RawFd}; +use vm_memory::{VolatileSlice, WriteVolatile}; -use crate::system::{self,FileDesc}; +use crate::system; use crate::devices::virtio_wl::{ consts::{VIRTIO_WL_VFD_WRITE, VIRTIO_WL_VFD_READ, IN_BUFFER_LEN}, @@ -11,13 +15,13 @@ use crate::devices::virtio_wl::{ pub struct VfdPipe { vfd_id: u32, flags: u32, - local: Option, - remote: Option, + local: Option, + remote: Option, } impl VfdPipe { - pub fn new(vfd_id: u32, read_pipe: FileDesc, write_pipe: FileDesc, local_write: bool) -> Self { + pub fn new(vfd_id: u32, read_pipe: File, write_pipe: File, local_write: bool) -> Self { if local_write { VfdPipe { vfd_id, local: Some(write_pipe), remote: Some(read_pipe), flags: VIRTIO_WL_VFD_WRITE } } else { @@ -25,7 +29,7 @@ impl VfdPipe { } } - pub fn local_only(vfd_id: u32, local_pipe: FileDesc, flags: u32) -> Self { + pub fn local_only(vfd_id: u32, local_pipe: File, flags: u32) -> Self { VfdPipe { vfd_id, local: Some(local_pipe), remote: None, flags } } @@ -35,8 +39,8 @@ impl VfdPipe { if libc::pipe2(pipe_fds.as_mut_ptr(), libc::O_CLOEXEC) < 0 { return Err(Error::CreatePipesFailed(system::Error::last_os_error())); } - let read_pipe = FileDesc::new(pipe_fds[0]); - let write_pipe = FileDesc::new(pipe_fds[1]); + let read_pipe = File::from_raw_fd(pipe_fds[0]); + let write_pipe = File::from_raw_fd(pipe_fds[1]); Ok(Self::new(vfd_id, read_pipe, write_pipe, local_write)) } } @@ -58,7 +62,7 @@ impl VfdObject for VfdPipe { fn recv(&mut self) -> Result> { if let Some(pipe) = self.local.take() { let mut buf = vec![0; IN_BUFFER_LEN]; - let len = pipe.read(&mut buf[..IN_BUFFER_LEN]) + let len = (&pipe).read(&mut buf[..IN_BUFFER_LEN]) .map_err(Error::PipeReceive)?; buf.truncate(len); if buf.len() > 0 { @@ -69,9 +73,9 @@ impl VfdObject for VfdPipe { Ok(None) } - fn send(&mut self, data: &[u8]) -> Result<()> { - if let Some(pipe) = self.local.as_ref() { - pipe.write_all(data).map_err(Error::SendVfd) + fn send(&mut self, data: &VolatileSlice) -> Result<()> { + if let Some(pipe) = self.local.as_mut() { + pipe.write_all_volatile(data).map_err(Error::VolatileSendVfd) } else { Err(Error::InvalidSendVfd) } diff --git a/src/devices/virtio_wl/shm.rs b/src/devices/virtio_wl/shm.rs index facf4f3..f0a6a0b 100644 --- a/src/devices/virtio_wl/shm.rs +++ b/src/devices/virtio_wl/shm.rs @@ -1,20 +1,15 @@ -use std::os::unix::io::{AsRawFd,RawFd}; - -use crate::memory::{MemoryManager, DrmDescriptor}; -use crate::system::MemoryFd; +use std::os::unix::io::RawFd; use crate::devices::virtio_wl::{ consts::{VIRTIO_WL_VFD_MAP, VIRTIO_WL_VFD_WRITE}, Error, Result, VfdObject }; +use crate::io::shm_mapper::{DeviceSharedMemoryManager, SharedMemoryAllocation}; pub struct VfdSharedMemory { vfd_id: u32, flags: u32, - mm: MemoryManager, - memfd: Option, - slot: u32, - pfn: u64, + shm: SharedMemoryAllocation, } impl VfdSharedMemory { @@ -23,28 +18,22 @@ impl VfdSharedMemory { (n + mask) & !mask } - pub fn new(vfd_id: u32, transition_flags: bool, mm: MemoryManager, memfd: MemoryFd, slot: u32, pfn: u64) -> Self { + pub fn new(vfd_id: u32, transition_flags: bool, shm: SharedMemoryAllocation) -> Self { let flags = if transition_flags { 0 } else { VIRTIO_WL_VFD_WRITE | VIRTIO_WL_VFD_MAP}; - let memfd = Some(memfd); - VfdSharedMemory { vfd_id, flags, mm, memfd, slot, pfn } + VfdSharedMemory { vfd_id, flags, shm } } - pub fn create(vfd_id: u32, transition_flags: bool, size: u32, mm: &MemoryManager) -> Result { + pub fn create(vfd_id: u32, transition_flags: bool, size: u32, dev_shm_manager: &DeviceSharedMemoryManager) -> Result { let size = Self::round_to_page_size(size as usize); - let memfd = MemoryFd::new_memfd(size, true) + let shm = dev_shm_manager.allocate_buffer(size) .map_err(Error::ShmAllocFailed)?; - let (pfn, slot) = mm.register_device_memory(memfd.as_raw_fd(), size) - .map_err(Error::RegisterMemoryFailed)?; - Ok(Self::new(vfd_id, transition_flags, mm.clone(), memfd, slot, pfn)) + Ok(Self::new(vfd_id, transition_flags, shm)) } - pub fn create_dmabuf(vfd_id: u32, tflags: bool, width: u32, height: u32, format: u32, mm: &MemoryManager) -> Result<(Self, DrmDescriptor)> { - let (pfn, slot, fd, desc) = mm.allocate_drm_buffer(width, height, format) - .map_err(Error::DmaBuf)?; - let memfd = MemoryFd::from_filedesc(fd) - .map_err(Error::DmaBufSize)?; - let vfd = Self::new(vfd_id, tflags, mm.clone(), memfd, slot, pfn); - Ok((vfd, desc)) + pub fn create_dmabuf(vfd_id: u32, tflags: bool, width: u32, height: u32, format: u32, dev_shm_manager: &DeviceSharedMemoryManager) -> Result { + let shm = dev_shm_manager.allocate_drm_buffer(width, height, format) + .map_err(Error::ShmAllocFailed)?; + Ok(Self::new(vfd_id, tflags, shm)) } } @@ -54,26 +43,14 @@ impl VfdObject for VfdSharedMemory { } fn send_fd(&self) -> Option { - self.memfd.as_ref().map(AsRawFd::as_raw_fd) + Some(self.shm.raw_fd()) } fn flags(&self) -> u32 { self.flags } - fn pfn_and_size(&self) -> Option<(u64, u64)> { - if let Some(memfd) = self.memfd.as_ref() { - Some((self.pfn, memfd.size() as u64)) - } else { - None - } - } - - fn close(&mut self) -> Result<()> { - if let Some(_) = self.memfd.take() { - self.mm.unregister_device_memory(self.slot) - .map_err(Error::RegisterMemoryFailed)?; - } - Ok(()) + fn shared_memory(&self) -> Option { + Some(self.shm) } } diff --git a/src/devices/virtio_wl/socket.rs b/src/devices/virtio_wl/socket.rs index 634f83f..bfbce31 100644 --- a/src/devices/virtio_wl/socket.rs +++ b/src/devices/virtio_wl/socket.rs @@ -1,8 +1,11 @@ -use std::io::{self,Write}; +use std::fs::File; +use std::io; +use std::os::fd::FromRawFd; use std::path::Path; use std::os::unix::{net::UnixStream, io::{AsRawFd, RawFd}}; +use vm_memory::{VolatileSlice, WriteVolatile}; -use crate::system::{FileDesc,ScmSocket}; +use crate::system::ScmSocket; use crate::devices::virtio_wl::{consts:: *, Error, Result, VfdObject, VfdRecv}; pub struct VfdSocket { @@ -29,14 +32,16 @@ impl VfdSocket { socket: Some(socket), }) } - fn socket_recv(socket: &mut UnixStream) -> Result<(Vec, Vec)> { + fn socket_recv(socket: &mut UnixStream) -> Result<(Vec, Vec)> { let mut buf = vec![0; IN_BUFFER_LEN]; let mut fd_buf = [0; VIRTWL_SEND_MAX_ALLOCS]; let (len, fd_len) = socket.recv_with_fds(&mut buf, &mut fd_buf) .map_err(Error::SocketReceive)?; buf.truncate(len); let files = fd_buf[..fd_len].iter() - .map(|&fd| FileDesc::new(fd)).collect(); + .map(|&fd| unsafe { + File::from_raw_fd(fd) + }).collect(); Ok((buf, files)) } } @@ -58,27 +63,29 @@ impl VfdObject for VfdSocket { let (buf,files) = Self::socket_recv(&mut sock)?; if !(buf.is_empty() && files.is_empty()) { self.socket.replace(sock); - if files.is_empty() { - return Ok(Some(VfdRecv::new(buf))); + return if files.is_empty() { + Ok(Some(VfdRecv::new(buf))) } else { - return Ok(Some(VfdRecv::new_with_fds(buf, files))); + Ok(Some(VfdRecv::new_with_fds(buf, files))) } } } Ok(None) } - fn send(&mut self, data: &[u8]) -> Result<()> { + fn send(&mut self, data: &VolatileSlice) -> Result<()> { if let Some(s) = self.socket.as_mut() { - s.write_all(data).map_err(Error::SendVfd) + s.write_all_volatile(data).map_err(Error::VolatileSendVfd) } else { Err(Error::InvalidSendVfd) } } - fn send_with_fds(&mut self, data: &[u8], fds: &[RawFd]) -> Result<()> { + fn send_with_fds(&mut self, data: &VolatileSlice, fds: &[RawFd]) -> Result<()> { if let Some(s) = self.socket.as_mut() { - s.send_with_fds(data, fds) + let mut buffer = vec![0u8; data.len()]; + data.copy_to(&mut buffer); + s.send_with_fds(&buffer, fds) .map_err(|_| Error::SendVfd(io::Error::last_os_error()))?; Ok(()) } else { @@ -97,6 +104,4 @@ impl VfdObject for VfdSocket { self.socket = None; Ok(()) } -} - - +} \ No newline at end of file diff --git a/src/devices/virtio_wl/vfd.rs b/src/devices/virtio_wl/vfd.rs index 4aa7634..f089f37 100644 --- a/src/devices/virtio_wl/vfd.rs +++ b/src/devices/virtio_wl/vfd.rs @@ -1,20 +1,24 @@ use std::collections::{HashMap, VecDeque}; -use std::io::{Write, SeekFrom}; +use std::fs::File; +use std::io; +use std::io::{Write, SeekFrom, Seek}; use std::os::unix::io::{AsRawFd,RawFd}; use std::path::PathBuf; use std::time::Duration; -use crate::memory::{MemoryManager, DrmDescriptor}; -use crate::system::{FileDesc, FileFlags,EPoll,MemoryFd}; +use crate::system::drm::DrmDescriptor; +use crate::system::EPoll; use crate::devices::virtio_wl::{ consts::*, Error, Result, shm::VfdSharedMemory, pipe::VfdPipe, socket::VfdSocket, VfdObject }; use crate::io::{Chain, VirtQueue}; +use crate::io::shm_mapper::DeviceSharedMemoryManager; +use crate::system::errno::cvt; pub struct VfdManager { wayland_path: PathBuf, - mm: MemoryManager, + dev_shm_manager: DeviceSharedMemoryManager, use_transition_flags: bool, vfd_map: HashMap>, next_vfd_id: u32, @@ -24,16 +28,12 @@ pub struct VfdManager { } impl VfdManager { - fn round_to_page_size(n: usize) -> usize { - let mask = 4096 - 1; - (n + mask) & !mask - } - - pub fn new>(mm: MemoryManager, use_transition_flags: bool, in_vq: VirtQueue, wayland_path: P) -> Result { + pub fn new>(dev_shm_manager: DeviceSharedMemoryManager, use_transition_flags: bool, in_vq: VirtQueue, wayland_path: P) -> Result { let poll_ctx = EPoll::new().map_err(Error::FailedPollContextCreate)?; Ok(VfdManager { wayland_path: wayland_path.into(), - mm, use_transition_flags, + dev_shm_manager, + use_transition_flags, vfd_map: HashMap::new(), next_vfd_id: NEXT_VFD_ID_BASE, poll_ctx, @@ -61,18 +61,18 @@ impl VfdManager { Ok(()) } - pub fn create_shm(&mut self, vfd_id: u32, size: u32) -> Result<(u64,u64)> { - let shm = VfdSharedMemory::create(vfd_id, self.use_transition_flags, size, &self.mm)?; - let (pfn,size) = shm.pfn_and_size().unwrap(); - self.vfd_map.insert(vfd_id, Box::new(shm)); - Ok((pfn,size)) + pub fn create_shm(&mut self, vfd_id: u32, size: u32) -> Result<(u64,usize)> { + let vfd = VfdSharedMemory::create(vfd_id, self.use_transition_flags, size, &self.dev_shm_manager)?; + let shm = vfd.shared_memory().unwrap(); + self.vfd_map.insert(vfd_id, Box::new(vfd)); + Ok((shm.pfn(),shm.size())) } - pub fn create_dmabuf(&mut self, vfd_id: u32, width: u32, height: u32, format: u32) -> Result<(u64, u64, DrmDescriptor)> { - let (vfd, desc) = VfdSharedMemory::create_dmabuf(vfd_id, self.use_transition_flags, width, height, format, &self.mm)?; - let (pfn, size) = vfd.pfn_and_size().unwrap(); + pub fn create_dmabuf(&mut self, vfd_id: u32, width: u32, height: u32, format: u32) -> Result<(u64, usize, DrmDescriptor)> { + let vfd = VfdSharedMemory::create_dmabuf(vfd_id, self.use_transition_flags, width, height, format, &self.dev_shm_manager)?; + let shm = vfd.shared_memory().unwrap(); self.vfd_map.insert(vfd_id, Box::new(vfd)); - Ok((pfn, size, desc)) + Ok((shm.pfn(), shm.size(), shm.drm_descriptor().unwrap())) } pub fn create_socket(&mut self, vfd_id: u32) -> Result { @@ -195,30 +195,35 @@ impl VfdManager { Ok(()) } - fn vfd_from_file(&self, vfd_id: u32, fd: FileDesc) -> Result> { - match fd.seek(SeekFrom::End(0)) { - Ok(size) => { - let size = Self::round_to_page_size(size as usize) as u64; - let (pfn,slot) = self.mm.register_device_memory(fd.as_raw_fd(), size as usize) - .map_err(Error::RegisterMemoryFailed)?; + fn vfd_from_file(&self, vfd_id: u32, fd: File) -> Result> { + fn has_size(mut fd: &File) -> bool { + fd.seek(SeekFrom::End(0)).is_ok() + } + + + if has_size(&fd) { + let shm = self.dev_shm_manager.allocate_buffer_from_file(fd) + .map_err(Error::ShmAllocFailed)?; + Ok(Box::new(VfdSharedMemory::new(vfd_id, self.use_transition_flags,shm))) + + } else { + let flags = match FileFlags::file_flags(fd.as_raw_fd()) { + Ok(FileFlags::Read) => VIRTIO_WL_VFD_READ, + Ok(FileFlags::Write) => VIRTIO_WL_VFD_WRITE, + Ok(FileFlags::ReadWrite) =>VIRTIO_WL_VFD_READ | VIRTIO_WL_VFD_WRITE, + _ => 0, + }; + Ok(Box::new(VfdPipe::local_only(vfd_id, fd, flags))) - let memfd = MemoryFd::from_filedesc(fd).map_err(Error::ShmAllocFailed)?; - return Ok(Box::new(VfdSharedMemory::new(vfd_id, self.use_transition_flags,self.mm.clone(), memfd, slot, pfn))); - } - _ => { - let flags = match fd.flags() { - Ok(FileFlags::Read) => VIRTIO_WL_VFD_READ, - Ok(FileFlags::Write) => VIRTIO_WL_VFD_WRITE, - Ok(FileFlags::ReadWrite) =>VIRTIO_WL_VFD_READ | VIRTIO_WL_VFD_WRITE, - _ => 0, - }; - return Ok(Box::new(VfdPipe::local_only(vfd_id, fd, flags))); - } } } pub fn close_vfd(&mut self, vfd_id: u32) -> Result<()> { if let Some(mut vfd) = self.vfd_map.remove(&vfd_id) { + if let Some(shm) = vfd.shared_memory() { + self.dev_shm_manager.free_buffer(shm.slot()) + .map_err(Error::ShmFreeFailed)?; + } vfd.close()?; } // XXX remove any matching fds from in_queue_pending @@ -290,9 +295,9 @@ impl PendingInput { chain.w32(0)?; chain.w32(vfd.id())?; chain.w32(vfd.flags())?; - let (pfn, size) = match vfd.pfn_and_size() { - Some(vals) => vals, - None => (0,0), + let (pfn, size) = match vfd.shared_memory() { + Some(shm) => (shm.pfn(), shm.size()), + None => (0, 0), }; chain.w64(pfn)?; chain.w32(size as u32)?; @@ -319,3 +324,23 @@ impl PendingInput { } } + +#[derive(Copy,Clone,Debug,Eq,PartialEq)] +pub enum FileFlags { + Read, + Write, + ReadWrite, +} + +impl FileFlags { + fn file_flags(fd: RawFd) -> io::Result { + let flags = unsafe { cvt(libc::fcntl(fd, libc::F_GETFL))? }; + match flags & libc::O_ACCMODE { + libc::O_RDONLY => Ok(FileFlags::Read), + libc::O_WRONLY => Ok(FileFlags::Write), + libc::O_RDWR => Ok(FileFlags::ReadWrite), + _ => Err(io::Error::from_raw_os_error(libc::EINVAL)), + } + } + +} diff --git a/src/disk/memory.rs b/src/disk/memory.rs index e87b021..1d4566d 100644 --- a/src/disk/memory.rs +++ b/src/disk/memory.rs @@ -1,32 +1,40 @@ -use crate::system::MemoryFd; +use std::fs::File; +use std::io; use crate::util::BitSet; use crate::disk::{Result, Error, SECTOR_SIZE, DiskImage}; -use std::io::SeekFrom; +use std::io::{Seek, SeekFrom}; +use memfd::MemfdOptions; +use vm_memory::{ReadVolatile, VolatileSlice, WriteVolatile}; pub struct MemoryOverlay { - memory: MemoryFd, + memory: File, written_sectors: BitSet, } impl MemoryOverlay { pub fn new() -> Result { - let memory = MemoryFd::new_memfd(0, false) + let memory = MemfdOptions::new() + .allow_sealing(true) + .create("disk-overlay-memfd") .map_err(Error::MemoryOverlayCreate)?; + let memory = memory.into_file(); let written_sectors = BitSet::new(); Ok(MemoryOverlay { memory, written_sectors }) } - pub fn write_sectors(&mut self, start: u64, buffer: &[u8]) -> Result<()> { + pub fn write_sectors(&mut self, start: u64, buffer: &VolatileSlice) -> Result<()> { let sector_count = buffer.len() / SECTOR_SIZE; let len = sector_count * SECTOR_SIZE; let seek_offset = SeekFrom::Start(start * SECTOR_SIZE as u64); - self.memory.fd_mut() - .seek(seek_offset) + self.memory.seek(seek_offset) .map_err(Error::DiskSeek)?; - self.memory.fd_mut() - .write_all(&buffer[..len]) + let slice = buffer.subslice(0, len) + .expect("Out of bounds in MemoryOverlay::write_sectors()"); + + self.memory.write_all_volatile(&slice) + .map_err(io::Error::other) .map_err(Error::DiskWrite)?; for n in 0..sector_count { @@ -36,7 +44,7 @@ impl MemoryOverlay { Ok(()) } - pub fn read_sectors(&mut self, disk: &mut D, start: u64, buffer: &mut [u8]) -> Result<()> { + pub fn read_sectors(&mut self, disk: &mut D, start: u64, buffer: &mut VolatileSlice) -> Result<()> { let sector_count = buffer.len() / SECTOR_SIZE; if (0..sector_count).all(|i| !self.written_sectors.get(i)) { return disk.read_sectors(start, buffer); @@ -45,22 +53,23 @@ impl MemoryOverlay { for n in 0..sector_count { let sector = start + n as u64; let offset = n * SECTOR_SIZE; - let sector_buffer = &mut buffer[offset..offset+SECTOR_SIZE]; + let mut sector_buffer = buffer.subslice(offset, SECTOR_SIZE) + .expect("Out of bounds in MemoryOverlay::read_sectors()"); if self.written_sectors.get(sector as usize) { - self.read_single_sector(sector, sector_buffer)?; + self.read_single_sector(sector, &mut sector_buffer)?; } else { - disk.read_sectors(sector, sector_buffer)?; + disk.read_sectors(sector, &mut sector_buffer)?; } } Ok(()) } - fn read_single_sector(&mut self, sector: u64, buffer: &mut [u8]) -> Result<()> { + fn read_single_sector(&mut self, sector: u64, buffer: &mut VolatileSlice) -> Result<()> { assert_eq!(buffer.len(), SECTOR_SIZE); let offset = SeekFrom::Start(sector * SECTOR_SIZE as u64); - self.memory.fd_mut().seek(offset) + self.memory.seek(offset) .map_err(Error::DiskSeek)?; - self.memory.fd_mut().read_exact(buffer) + self.memory.read_exact_volatile(buffer).map_err(io::Error::other) .map_err(Error::DiskRead)?; Ok(()) } diff --git a/src/disk/mod.rs b/src/disk/mod.rs index 59dcc0a..2b32ef8 100644 --- a/src/disk/mod.rs +++ b/src/disk/mod.rs @@ -3,8 +3,6 @@ use std::fs::File; use std::os::linux::fs::MetadataExt; use std::io::{SeekFrom, Seek}; -use crate::system; - mod realmfs; mod raw; mod memory; @@ -13,6 +11,7 @@ pub use raw::RawDiskImage; pub use realmfs::RealmFSImage; use std::path::PathBuf; use thiserror::Error; +use vm_memory::VolatileSlice; const SECTOR_SIZE: usize = 512; @@ -39,8 +38,8 @@ pub trait DiskImage: Sync+Send { .map_err(Error::DiskSeek)?; Ok(()) } - fn write_sectors(&mut self, start_sector: u64, buffer: &[u8]) -> Result<()>; - fn read_sectors(&mut self, start_sector: u64, buffer: &mut [u8]) -> Result<()>; + fn write_sectors(&mut self, start_sector: u64, buffer: &VolatileSlice) -> Result<()>; + fn read_sectors(&mut self, start_sector: u64, buffer: &mut VolatileSlice) -> Result<()>; fn flush(&mut self) -> Result<()> { Ok(()) } fn disk_image_id(&self) -> &[u8]; @@ -79,7 +78,7 @@ pub enum Error { #[error("attempt to access invalid sector offset {0}")] BadSectorOffset(u64), #[error("failed to create memory overlay: {0}")] - MemoryOverlayCreate(system::Error), + MemoryOverlayCreate(memfd::Error), #[error("disk not open")] NotOpen, } \ No newline at end of file diff --git a/src/disk/raw.rs b/src/disk/raw.rs index 415e176..55cfe40 100644 --- a/src/disk/raw.rs +++ b/src/disk/raw.rs @@ -1,10 +1,11 @@ use crate::disk::{Result, Error, DiskImage, SECTOR_SIZE, generate_disk_image_id, OpenType}; use std::fs::{File, OpenOptions}; -use std::io::{Write, Read, SeekFrom, Seek}; +use std::io; +use std::io::{SeekFrom, Seek}; use crate::disk::Error::DiskRead; use crate::disk::memory::MemoryOverlay; use std::path::{PathBuf, Path}; - +use vm_memory::{ReadVolatile, VolatileSlice, WriteVolatile}; pub struct RawDiskImage { path: PathBuf, @@ -94,7 +95,7 @@ impl DiskImage for RawDiskImage { Ok(()) } - fn write_sectors(&mut self, start_sector: u64, buffer: &[u8]) -> Result<()> { + fn write_sectors(&mut self, start_sector: u64, buffer: &VolatileSlice) -> Result<()> { if let Some(ref mut overlay) = self.overlay { return overlay.write_sectors(start_sector, buffer); } @@ -104,12 +105,15 @@ impl DiskImage for RawDiskImage { self.seek_to_sector(start_sector)?; let len = (buffer.len() / SECTOR_SIZE) * SECTOR_SIZE; let file = self.disk_file()?; - file.write_all(&buffer[..len]) + let buffer = buffer.subslice(0, len) + .expect("Out of bounds in RawDiskImage::write_sectors()"); + file.write_all_volatile(&buffer) + .map_err(io::Error::other) .map_err(Error::DiskWrite)?; Ok(()) } - fn read_sectors(&mut self, start_sector: u64, buffer: &mut [u8]) -> Result<()> { + fn read_sectors(&mut self, start_sector: u64, buffer: &mut VolatileSlice) -> Result<()> { if let Some(mut overlay) = self.overlay.take() { let ret = overlay.read_sectors(self, start_sector, buffer); self.overlay.replace(overlay); @@ -119,7 +123,10 @@ impl DiskImage for RawDiskImage { self.seek_to_sector(start_sector)?; let len = (buffer.len() / SECTOR_SIZE) * SECTOR_SIZE; let file = self.disk_file()?; - file.read_exact(&mut buffer[..len]) + let mut buffer = buffer.subslice(0, len) + .expect("Out of bounds in RawDiskImage::read_sectors()"); + file.read_exact_volatile(&mut buffer) + .map_err(io::Error::other) .map_err(DiskRead)?; Ok(()) } diff --git a/src/disk/realmfs.rs b/src/disk/realmfs.rs index 75debd1..3eb2c92 100644 --- a/src/disk/realmfs.rs +++ b/src/disk/realmfs.rs @@ -1,6 +1,7 @@ use crate::disk::{Result, DiskImage, SECTOR_SIZE, RawDiskImage, OpenType}; use std::fs::File; use std::path::PathBuf; +use vm_memory::VolatileSlice; // skip 4096 byte realmfs header const HEADER_SECTOR_COUNT: usize = 8; @@ -35,11 +36,11 @@ impl DiskImage for RealmFSImage { self.raw.disk_file() } - fn write_sectors(&mut self, start_sector: u64, buffer: &[u8]) -> Result<()> { + fn write_sectors(&mut self, start_sector: u64, buffer: &VolatileSlice) -> Result<()> { self.raw.write_sectors(start_sector, buffer) } - fn read_sectors(&mut self, start_sector: u64, buffer: &mut [u8]) -> Result<()> { + fn read_sectors(&mut self, start_sector: u64, buffer: &mut VolatileSlice) -> Result<()> { self.raw.read_sectors(start_sector, buffer) } diff --git a/src/memory/address.rs b/src/io/address.rs similarity index 100% rename from src/memory/address.rs rename to src/io/address.rs diff --git a/src/io/manager.rs b/src/io/manager.rs index 7778230..5e2a6d9 100644 --- a/src/io/manager.rs +++ b/src/io/manager.rs @@ -1,14 +1,16 @@ use std::sync::{Arc, Mutex, MutexGuard}; use vm_allocator::{AddressAllocator, AllocPolicy, IdAllocator, RangeInclusive}; +use vm_memory::GuestMemoryMmap; use vmm_sys_util::eventfd::EventFd; use crate::devices::rtc::Rtc; use crate::devices::serial::{SerialDevice, SerialPort}; use crate::io::bus::{Bus, BusDevice}; use crate::io::pci::{MmioHandler, PciBarAllocation, PciBus, PciDevice}; use crate::io::{PciIrq, virtio}; +use crate::io::address::AddressRange; +use crate::io::shm_mapper::DeviceSharedMemoryManager; use crate::io::virtio::{VirtioDeviceState,VirtioDevice}; -use crate::memory::{AddressRange, MemoryManager}; -use crate::vm::arch; +use crate::vm::{arch, KvmVm}; #[derive(Clone)] pub struct IoAllocator { @@ -41,7 +43,9 @@ impl IoAllocator { #[derive(Clone)] pub struct IoManager { - memory: MemoryManager, + kvm_vm: KvmVm, + memory: GuestMemoryMmap, + dev_shm_manager: DeviceSharedMemoryManager, pio_bus: Bus, mmio_bus: Bus, pci_bus: Arc>, @@ -49,13 +53,18 @@ pub struct IoManager { } impl IoManager { - pub fn new(memory: MemoryManager) -> IoManager { + pub fn new(kvm_vm: KvmVm, memory: GuestMemoryMmap) -> IoManager { let pci_bus = Arc::new(Mutex::new(PciBus::new())); let mut pio_bus = Bus::new(); pio_bus.insert(pci_bus.clone(), PciBus::PCI_CONFIG_ADDRESS as u64, 8) .expect("Failed to add PCI configuration to PIO"); + + let dev_shm_manager = DeviceSharedMemoryManager::new(&kvm_vm, &memory); + IoManager { + kvm_vm, memory, + dev_shm_manager, pio_bus, mmio_bus: Bus::new(), pci_bus, @@ -72,7 +81,7 @@ impl IoManager { } pub fn register_serial_port(&mut self, port: SerialPort) { - let serial = SerialDevice::new(self.memory.kvm_vm().clone(), port.irq()); + let serial = SerialDevice::new(self.kvm_vm.clone(), port.irq()); let serial = Arc::new(Mutex::new(serial)); self.pio_bus.insert(serial, port.io_port() as u64, 8).unwrap(); @@ -135,29 +144,15 @@ impl IoManager { } pub fn add_virtio_device(&mut self, dev: D) -> virtio::Result<()> { - //let devtype = dev.device_type(); - //let dev = Arc::new(Mutex::new(dev)); - //let devstate = VirtioDeviceState::new(dev.clone(), self.memory.clone(), self.allocator.clone())?; let irq = self.allocator.allocate_irq(); - //let devstate = VirtioDeviceState::new(dev, self.memory.clone(), self.allocator.clone())?; - let devstate = VirtioDeviceState::new(dev, self.memory.clone(), irq)?; + let devstate = VirtioDeviceState::new(dev, self.kvm_vm.clone(), self.memory.clone(), irq)?; self.add_pci_device(Arc::new(Mutex::new(devstate))); - - // let mmio_range = devstate.mmio_range(); - - //let mut pci = self.pci_bus.lock().unwrap(); - //pci.add_device(devstate); -// let mut pci_device = pci.new_device(devstate.irq() as u8, PCI_VENDOR_ID_REDHAT, devtype.device_id(), devtype.class_id()); - // XXX add mmio bar - //pci_device.set_mmio_bar(0, AddressRange::new(mmio_range.start(), mmio_range.len() as usize)); - -// devstate.add_pci_capabilities(&mut pci_device); - // XXX add devstate to mmio bus - //self.mmio_bus.insert(Arc::new(Mutex::new(devstate)), mmio_range.start(), mmio_range.len())?; - - //pci.add_device(pci_device); Ok(()) } + + pub fn dev_shm_manager(&self) -> &DeviceSharedMemoryManager { + &self.dev_shm_manager + } } pub struct I8042Device { diff --git a/src/io/mod.rs b/src/io/mod.rs index 2d09521..f228b1b 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -3,10 +3,14 @@ pub mod busdata; pub mod pci; pub mod manager; pub mod virtio; +mod address; +pub mod shm_mapper; + pub use virtio::{VirtioDevice,FeatureBits,VirtioDeviceType,VirtQueue,Chain,Queues}; pub use virtio::Error as VirtioError; -pub use busdata::{ReadableInt,WriteableInt}; +pub use busdata::ReadableInt; pub use pci::PciIrq; + // PCI Vendor id for Virtio devices pub const PCI_VENDOR_ID_REDHAT: u16 = 0x1af4; diff --git a/src/io/pci/config.rs b/src/io/pci/config.rs index db2fe5e..2fedf44 100644 --- a/src/io/pci/config.rs +++ b/src/io/pci/config.rs @@ -1,7 +1,7 @@ +use crate::io::address::AddressRange; use crate::io::pci::address::PciAddress; use crate::io::pci::consts::{PCI_BAR0, PCI_BAR5, PCI_CACHE_LINE_SIZE, PCI_CAP_BASE_OFFSET, PCI_CAP_ID_VENDOR, PCI_CAPABILITY_LIST, PCI_CLASS_DEVICE, PCI_CLASS_REVISION, PCI_COMMAND, PCI_COMMAND_IO, PCI_COMMAND_MEMORY, PCI_DEVICE_ID, PCI_INTERRUPT_LINE, PCI_INTERRUPT_PIN, PCI_STATUS, PCI_STATUS_CAP_LIST, PCI_SUBSYSTEM_ID, PCI_VENDOR_ID}; use crate::io::pci::device::PciBar; -use crate::memory::AddressRange; use crate::util::{ByteBuffer,Writeable}; const PCI_CONFIG_SPACE_SIZE: usize = 256; @@ -226,23 +226,4 @@ impl PciConfiguration { self.write_config(offset as usize, data); } } -} - -/* -impl BusDevice for PciConfiguration { - fn read(&mut self, offset: u64, data: &mut [u8]) { - if Self::is_valid_access(offset, data.len()) { - self.read_bytes(offset as usize, data) - } else { - data.fill(0xff) - } - } - - fn write(&mut self, offset: u64, data: &[u8]) { - if Self::is_valid_access(offset, data.len()) { - self.write_config(offset as usize, data); - } - } -} - - */ +} \ No newline at end of file diff --git a/src/io/pci/mod.rs b/src/io/pci/mod.rs index 71ca3cb..006cf55 100644 --- a/src/io/pci/mod.rs +++ b/src/io/pci/mod.rs @@ -5,6 +5,5 @@ mod config; mod consts; mod device; pub use bus::{PciBus,PciIrq}; -pub use config::{PciCapability,PciConfiguration}; -pub use address::PciAddress; +pub use config::PciConfiguration; pub use device::{PciDevice,PciBar,PciBarAllocation,MmioHandler}; diff --git a/src/io/shm_mapper.rs b/src/io/shm_mapper.rs new file mode 100644 index 0000000..65883fe --- /dev/null +++ b/src/io/shm_mapper.rs @@ -0,0 +1,341 @@ +use std::collections::HashMap; +use std::fs::File; +use std::os::fd::{AsRawFd, RawFd}; +use std::result; +use std::sync::{Arc, Mutex, MutexGuard}; +use vm_allocator::{AddressAllocator, AllocPolicy, RangeInclusive}; +use vm_memory::{Address, FileOffset, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, MmapRegion}; +use crate::system::drm::{DrmBufferAllocator, DrmDescriptor}; +use crate::system::drm; +use crate::util::BitSet; +use crate::vm::KvmVm; + +use thiserror::Error; +use std::io::{Seek, SeekFrom}; +use memfd::{FileSeal, MemfdOptions}; +use crate::system; + +pub type Result = result::Result; + +#[derive(Debug,Error)] +pub enum Error { + #[error("failed to create SharedMemory: {0}")] + SharedMemoryCreation(system::Error), + #[error("failed to allocate DRM buffer: {0}")] + DrmAllocateFailed(drm::Error), + #[error("no DRM memory allocator")] + NoDrmAllocator, + #[error("failed to register memory with hypervisor: {0}")] + RegisterMemoryFailed(kvm_ioctls::Error), + #[error("failed to unregister memory with hypervisor: {0}")] + UnregisterMemoryFailed(kvm_ioctls::Error), + #[error("failed to allocate memory for device")] + DeviceMemoryAllocFailed, + +} + +/// Tracks graphic buffer memory allocations shared between host and guest. +/// +/// The allocated buffers are opaque to the hypervisor and are referred to only +/// by file descriptor. These memory regions are passed to or received from the +/// wayland compositor (ie GNOME Shell) and are mapped into guest memory space so +/// that a device driver in the guest can make the shared memory available for +/// rendering application windows. +/// +#[derive(Clone)] +pub struct DeviceSharedMemoryManager { + device_memory: Arc>, +} + +impl DeviceSharedMemoryManager { + + pub fn new(kvm_vm: &KvmVm, memory: &GuestMemoryMmap) -> Self { + let device_memory = DeviceSharedMemory::new(kvm_vm.clone(), memory); + DeviceSharedMemoryManager { + device_memory: Arc::new(Mutex::new(device_memory)), + } + } + + pub fn free_buffer(&self, slot: u32) -> Result<()> { + self.dev_memory().unregister(slot) + } + + pub fn allocate_buffer_from_file(&self, fd: File) -> Result { + let memory = SharedMemoryMapping::from_file(fd) + .map_err(Error::SharedMemoryCreation)?; + + self.dev_memory().register(memory) + } + + pub fn allocate_buffer(&self, size: usize) -> Result { + let memory = SharedMemoryMapping::create_memfd(size, "ph-dev-shm") + .map_err(Error::SharedMemoryCreation)?; + + self.dev_memory().register(memory) + } + + pub fn allocate_drm_buffer(&self, width: u32, height: u32, format: u32) -> Result { + self.dev_memory().allocate_drm_buffer(width, height, format) + } + + fn dev_memory(&self) -> MutexGuard { + self.device_memory.lock().unwrap() + } +} + +#[derive(Copy,Clone)] +pub struct SharedMemoryAllocation { + pfn: u64, + size: usize, + slot: u32, + raw_fd: RawFd, + drm_descriptor: Option, +} + +impl SharedMemoryAllocation { + fn new(pfn: u64, size: usize, slot: u32, raw_fd: RawFd) -> Self { + SharedMemoryAllocation { + pfn, size, slot, raw_fd, + drm_descriptor: None, + } + } + + fn set_drm_descriptor(&mut self, drm_descriptor: DrmDescriptor) { + self.drm_descriptor.replace(drm_descriptor); + } + + pub fn pfn(&self) -> u64 { + self.pfn + } + + pub fn size(&self) -> usize { + self.size + } + + pub fn slot(&self) -> u32 { + self.slot + } + + pub fn raw_fd(&self) -> RawFd { + self.raw_fd + } + + pub fn drm_descriptor(&self) -> Option { + self.drm_descriptor + } +} + +struct DeviceSharedMemory { + kvm_vm: KvmVm, + slots: BitSet, + mappings: HashMap, + allocator: AddressAllocator, + drm_allocator: Option +} + +impl DeviceSharedMemory { + const WL_SHM_SIZE: u64 = 1 << 32; + + fn create_allocator(memory: &GuestMemoryMmap) -> AddressAllocator { + let device_memory_base = || -> GuestAddress { + // Put device memory at a 2MB boundary after physical memory or at 4GB, + // whichever is higher. + const ONE_MB: u64 = 1 << 20; + const FOUR_GB: u64 = 4 * 1024 * ONE_MB; + + memory.iter() + .map(|addr| addr.last_addr().unchecked_align_up(2 * ONE_MB)) + .max() + .map(|top| std::cmp::max(top, GuestAddress(FOUR_GB))) + .expect("Failed to compute device memory base") + }; + let base = device_memory_base(); + AddressAllocator::new(base.raw_value(), Self::WL_SHM_SIZE) + .expect("Failed to create wayland shared memory allocator") + + } + + fn new(kvm_vm: KvmVm, memory: &GuestMemoryMmap) -> Self { + let allocator = Self::create_allocator(memory); + let mut slots = BitSet::new(); + for idx in 0..memory.num_regions() { + slots.insert(idx); + } + + DeviceSharedMemory { + kvm_vm, + slots, + mappings: HashMap::new(), + allocator, + drm_allocator: None, + } + } + + fn is_drm_enabled(&self) -> bool { + self.drm_allocator.is_some() + } + + fn enable_drm_allocator(&mut self) { + if !self.is_drm_enabled() { + match DrmBufferAllocator::open() { + Ok(allocator) => { self.drm_allocator.replace(allocator); }, + Err(err) => { + warn!("failed to open DRM buffer allocator: {}", err); + }, + }; + } + } + + fn allocate_drm_buffer(&mut self, width: u32, height: u32, format: u32) -> Result { + if !self.is_drm_enabled() { + self.enable_drm_allocator(); + } + + if let Some(drm_allocator) = self.drm_allocator.as_ref() { + let (fd, desc) = drm_allocator.allocate(width, height, format) + .map_err(Error::DrmAllocateFailed)?; + let memory = SharedMemoryMapping::from_file(fd) + .map_err(Error::SharedMemoryCreation)?; + + let mut registration = self.register(memory)?; + registration.set_drm_descriptor(desc); + Ok(registration) + } else { + Err(Error::NoDrmAllocator) + } + } + + fn register(&mut self, mut memory: SharedMemoryMapping) -> Result { + + fn round_to_page_size(n: usize) -> usize { + let mask = 4096 - 1; + (n + mask) & !mask + } + + + let size = round_to_page_size(memory.size()); + let (range, slot) = self.allocate_addr_and_slot(size)?; + memory.set_guest_range(range.clone()); + + if let Err(e) = self.kvm_vm.add_memory_region(slot, range.start(), memory.mapping_host_address(), size) { + self.free_range_and_slot(&range, slot); + Err(Error::RegisterMemoryFailed(e)) + } else { + let pfn = range.start() >> 12; + let size = memory.size(); + let raw_fd = memory.raw_fd(); + self.mappings.insert(slot, memory); + Ok(SharedMemoryAllocation::new(pfn, size, slot, raw_fd)) + } + } + + fn unregister(&mut self, slot: u32) -> Result<()> { + if let Some(registration) = self.mappings.remove(&slot) { + self.kvm_vm.remove_memory_region(slot) + .map_err(Error::UnregisterMemoryFailed)?; + if let Some(range) = registration.guest_range() { + self.free_range_and_slot(range, slot); + } else { + // XXX + } + } + Ok(()) + } + + fn allocate_addr_and_slot(&mut self, size: usize) -> Result<(RangeInclusive, u32)> { + let range = self.allocator.allocate( + size as u64, + 4096, + AllocPolicy::FirstMatch + ).map_err(|_| Error::DeviceMemoryAllocFailed)?; + Ok((range, self.allocate_slot())) + } + + fn free_range_and_slot(&mut self, range: &RangeInclusive, slot: u32) { + if let Err(err) = self.allocator.free(range) { + warn!("Error freeing memory range from allocator: {}", err); + } + self.free_slot(slot); + } + + fn allocate_slot(&mut self) -> u32 { + for i in 0.. { + if !self.slots.get(i) { + self.slots.insert(i); + return i as u32; + } + } + unreachable!() + } + + fn free_slot(&mut self, slot: u32) { + self.slots.remove(slot as usize) + } +} + +struct SharedMemoryMapping { + mapping: MmapRegion, + guest_range: Option, +} + +impl SharedMemoryMapping { + fn from_file(fd: File) -> system::Result { + let size = (&fd).seek(SeekFrom::End(0))? as usize; + + let file_offset = FileOffset::new(fd, 0); + let mapping = MmapRegion::from_file(file_offset, size) + .map_err(system::Error::MmapRegionCreate)?; + Ok(SharedMemoryMapping { + mapping, + guest_range: None, + }) + } + + fn create_memfd(size: usize, name: &str) -> system::Result { + let memfd = MemfdOptions::default() + .allow_sealing(true) + .create(name) + .map_err(system::Error::ShmAllocFailed)?; + + memfd.as_file().set_len(size as u64)?; + memfd.add_seals(&[ + FileSeal::SealShrink, + FileSeal::SealGrow, + ]).map_err(system::Error::ShmAllocFailed)?; + memfd.add_seal(FileSeal::SealSeal) + .map_err(system::Error::ShmAllocFailed)?; + + let fd = memfd.into_file(); + let file_offset = FileOffset::new(fd, 0); + let mapping = MmapRegion::from_file(file_offset, size) + .map_err(system::Error::MmapRegionCreate)?; + + Ok(SharedMemoryMapping { + mapping, + guest_range: None, + }) + } + + fn size(&self) -> usize { + self.mapping.size() + } + + fn mapping_host_address(&self) -> u64 { + self.mapping.as_ptr() as u64 + } + + fn set_guest_range(&mut self, range: RangeInclusive) { + self.guest_range.replace(range); + } + + fn guest_range(&self) -> Option<&RangeInclusive> { + self.guest_range.as_ref() + } + + fn raw_fd(&self) -> RawFd { + self.mapping.file_offset() + .expect("SharedMemory mapping does not have a file!") + .file() + .as_raw_fd() + } +} diff --git a/src/io/virtio/device.rs b/src/io/virtio/device.rs index 0da9949..6160be0 100644 --- a/src/io/virtio/device.rs +++ b/src/io/virtio/device.rs @@ -1,6 +1,8 @@ use std::ops::Range; use std::sync::{Arc, Mutex, MutexGuard}; use byteorder::{ByteOrder, LittleEndian}; +use vm_memory::GuestMemoryMmap; +use crate::io::address::AddressRange; use crate::io::busdata::{ReadableInt, WriteableInt}; use crate::io::pci::{PciBar, PciBarAllocation, PciConfiguration, PciDevice}; @@ -9,7 +11,7 @@ use crate::io::virtio::features::FeatureBits; use crate::io::virtio::queues::Queues; use crate::io::virtio::Result; use crate::io::PCI_VENDOR_ID_REDHAT; -use crate::memory::{AddressRange, MemoryManager}; +use crate::vm::KvmVm; pub trait VirtioDevice: Send { @@ -41,12 +43,12 @@ pub struct VirtioDeviceState { impl VirtioDeviceState { - pub fn new(device: T, memory: MemoryManager, irq: u8) -> Result { + pub fn new(device: T, kvm_vm: KvmVm, guest_memory: GuestMemoryMmap, irq: u8) -> Result { let devtype = device.device_type(); let config_size = device.config_size(); let device = Arc::new(Mutex::new(device)); - let queues = Queues::new(memory, irq)?; + let queues = Queues::new(kvm_vm, guest_memory, irq)?; let mut pci_config = PciConfiguration::new(queues.irq(), PCI_VENDOR_ID_REDHAT, devtype.device_id(), devtype.class_id()); Self::add_pci_capabilities::(&mut pci_config, config_size); diff --git a/src/io/virtio/queues.rs b/src/io/virtio/queues.rs index 4fc61aa..133bf5d 100644 --- a/src/io/virtio/queues.rs +++ b/src/io/virtio/queues.rs @@ -1,8 +1,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use kvm_ioctls::{IoEventAddress, NoDatamatch}; +use vm_memory::GuestMemoryMmap; use vmm_sys_util::eventfd::EventFd; -use crate::memory::MemoryManager; use crate::io::virtio::{Error, Result}; use crate::io::virtio::consts::VIRTIO_MMIO_OFFSET_NOTIFY; use crate::io::VirtQueue; @@ -49,17 +49,19 @@ impl InterruptLine { } pub struct Queues { - memory: MemoryManager, + kvm_vm: KvmVm, + guest_memory: GuestMemoryMmap, selected_queue: u16, queues: Vec, interrupt: Arc, } impl Queues { - pub fn new(memory: MemoryManager, irq: u8) -> Result { - let interrupt = InterruptLine::new(memory.kvm_vm(), irq)?; + pub fn new(kvm_vm: KvmVm, guest_memory: GuestMemoryMmap, irq: u8) -> Result { + let interrupt = InterruptLine::new(&kvm_vm, irq)?; let queues = Queues { - memory, + kvm_vm, + guest_memory, selected_queue: 0, queues: Vec::new(), interrupt: Arc::new(interrupt), @@ -78,8 +80,12 @@ impl Queues { self.queues.clone() } - pub fn memory(&self) -> &MemoryManager { - &self.memory + pub fn guest_memory(&self) -> &GuestMemoryMmap{ + &self.guest_memory + } + + pub fn kvm_vm(&self) -> &KvmVm { + &self.kvm_vm } pub fn configure_queues(&self, features: u64) -> Result<()> { @@ -113,7 +119,7 @@ impl Queues { let mut idx = 0; for &sz in queue_sizes { let ioevent = self.create_ioevent(idx, mmio_base)?; - let vq = VirtQueue::new(self.memory.guest_ram().clone(), sz, self.interrupt.clone(), ioevent); + let vq = VirtQueue::new(self.guest_memory.clone(), sz, self.interrupt.clone(), ioevent); self.queues.push(vq); idx += 1; } @@ -130,7 +136,7 @@ impl Queues { let addr = IoEventAddress::Mmio(notify_address); - self.memory.kvm_vm().vm_fd().register_ioevent(&evt, &addr, NoDatamatch) + self.kvm_vm.vm_fd().register_ioevent(&evt, &addr, NoDatamatch) .map_err(Error::CreateIoEventFd)?; Ok(Arc::new(evt)) diff --git a/src/io/virtio/vq/chain.rs b/src/io/virtio/vq/chain.rs index 1e3b9e2..5668215 100644 --- a/src/io/virtio/vq/chain.rs +++ b/src/io/virtio/vq/chain.rs @@ -1,12 +1,12 @@ use std::{fmt, io}; use std::io::{Read, Write}; use std::sync::{Arc, Mutex}; +use vm_memory::{GuestAddress, GuestMemory, GuestMemoryMmap, ReadVolatile, VolatileSlice}; use crate::io::virtio::vq::descriptor::Descriptor; use crate::io::virtio::vq::virtqueue::QueueBackend; -use crate::memory::GuestRam; pub struct DescriptorList { - memory: GuestRam, + memory: GuestMemoryMmap, descriptors: Vec, offset: usize, total_size: usize, @@ -14,7 +14,7 @@ pub struct DescriptorList { } impl DescriptorList { - pub fn new(memory: GuestRam) -> Self { + pub fn new(memory: GuestMemoryMmap) -> Self { DescriptorList { memory, descriptors: Vec::new(), @@ -96,8 +96,8 @@ impl DescriptorList { 0 } - fn write_from_reader(&mut self, reader: R, size: usize) -> io::Result - where R: Read+Sized + fn write_from_reader(&mut self, reader: &mut R, size: usize) -> io::Result + where R: ReadVolatile+Sized { if let Some(d) = self.current() { let n = d.write_from_reader(&self.memory, self.offset, reader, size)?; @@ -108,23 +108,20 @@ impl DescriptorList { } } - fn current_slice(&self) -> &[u8] { - if let Some(d) = self.current() { - let size = d.remaining(self.offset); - let addr = d.address() + self.offset as u64; - self.memory.slice(addr, size).unwrap_or(&[]) - } else { - &[] + fn empty_slice() -> VolatileSlice<'static> { + unsafe { + VolatileSlice::new(0 as *mut u8, 0) } } - fn current_mut_slice(&self) -> &mut [u8] { + fn current_slice(&self) -> VolatileSlice { if let Some(d) = self.current() { let size = d.remaining(self.offset); let addr = d.address() + self.offset as u64; - self.memory.mut_slice(addr, size).unwrap_or(&mut []) + self.memory.get_slice(GuestAddress(addr), size) + .unwrap_or(Self::empty_slice()) } else { - &mut [] + Self::empty_slice() } } @@ -224,7 +221,7 @@ impl Chain { self.readable.is_empty() && self.writeable.is_empty() } - pub fn current_read_slice(&self) -> &[u8] { + pub fn current_read_slice(&self) -> VolatileSlice { self.readable.current_slice() } @@ -239,12 +236,12 @@ impl Chain { self.writeable.inc(sz); } - pub fn current_write_slice(&mut self) -> &mut [u8] { - self.writeable.current_mut_slice() + pub fn current_write_slice(&mut self) -> VolatileSlice { + self.writeable.current_slice() } - pub fn copy_from_reader(&mut self, r: R, size: usize) -> io::Result - where R: Read+Sized + pub fn copy_from_reader(&mut self, r: &mut R, size: usize) -> io::Result + where R: ReadVolatile+Sized { self.writeable.write_from_reader(r, size) } diff --git a/src/io/virtio/vq/descriptor.rs b/src/io/virtio/vq/descriptor.rs index 8c20f3a..504ac94 100644 --- a/src/io/virtio/vq/descriptor.rs +++ b/src/io/virtio/vq/descriptor.rs @@ -1,6 +1,5 @@ use std::{cmp, io}; -use std::io::Read; -use crate::memory::GuestRam; +use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, ReadVolatile}; #[repr(u16)] enum DescriptorFlag { @@ -83,29 +82,32 @@ impl Descriptor { (used != avail) && (avail == wrap_counter) } - pub fn read_from(&self, memory: &GuestRam, offset: usize, buf: &mut[u8]) -> usize { + pub fn read_from(&self, memory: &GuestMemoryMmap, offset: usize, buf: &mut[u8]) -> usize { let sz = cmp::min(buf.len(), self.remaining(offset)); if sz > 0 { - memory.read_bytes(self.address + offset as u64, &mut buf[..sz]).unwrap(); + let address = GuestAddress(self.address).checked_add(offset as u64).unwrap(); + memory.read_slice(&mut buf[..sz], address).unwrap(); } sz } - pub fn write_to(&self, memory: &GuestRam, offset: usize, buf: &[u8]) -> usize { + pub fn write_to(&self, memory: &GuestMemoryMmap, offset: usize, buf: &[u8]) -> usize { let sz = cmp::min(buf.len(), self.remaining(offset)); if sz > 0 { - memory.write_bytes(self.address + offset as u64, &buf[..sz]).unwrap(); + let address = GuestAddress(self.address).checked_add(offset as u64).unwrap(); + memory.write_slice(&buf[..sz], address).unwrap(); } sz } - pub fn write_from_reader(&self, memory: &GuestRam, offset: usize, mut r: R, size: usize) -> io::Result { + pub fn write_from_reader(&self, memory: &GuestMemoryMmap, offset: usize, r: &mut R, size: usize) -> io::Result { let sz = cmp::min(size, self.remaining(offset)); if sz > 0 { - let slice = memory.mut_slice(self.address + offset as u64, sz).unwrap(); - return r.read(slice); + let address = GuestAddress(self.address).checked_add(offset as u64).unwrap(); + let mut slice = memory.get_slice(address, sz).unwrap(); + let sz = r.read_volatile(&mut slice).unwrap(); + return Ok(sz) } Ok(0) } - } \ No newline at end of file diff --git a/src/io/virtio/vq/splitqueue.rs b/src/io/virtio/vq/splitqueue.rs index c21845a..dc79d9f 100644 --- a/src/io/virtio/vq/splitqueue.rs +++ b/src/io/virtio/vq/splitqueue.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, atomic}; use std::sync::atomic::Ordering; +use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; use crate::io::virtio::Error; use crate::io::virtio::features::ReservedFeatureBit; use crate::io::virtio::queues::InterruptLine; @@ -7,11 +8,10 @@ use crate::io::virtio::vq::chain::DescriptorList; use crate::io::virtio::vq::descriptor::Descriptor; use crate::io::virtio::vq::SharedIndex; use crate::io::virtio::vq::virtqueue::QueueBackend; -use crate::memory::GuestRam; pub struct SplitQueue { - memory: GuestRam, + memory: GuestMemoryMmap, interrupt: Arc, queue_size: u16, @@ -29,7 +29,7 @@ pub struct SplitQueue { } impl SplitQueue { - pub fn new(memory: GuestRam, interrupt: Arc) -> Self { + pub fn new(memory: GuestMemoryMmap, interrupt: Arc) -> Self { SplitQueue { memory, interrupt, @@ -54,12 +54,12 @@ impl SplitQueue { } let head = self.descriptor_base + (idx as u64 * 16); - let addr = self.memory.read_int::(head).unwrap(); - let len= self.memory.read_int::(head + 8).unwrap(); - let flags = self.memory.read_int::(head + 12).unwrap(); - let next = self.memory.read_int::(head + 14).unwrap(); + let addr = self.memory.read_obj::(GuestAddress(head)).ok()?; + let len= self.memory.read_obj::(GuestAddress(head + 8)).ok()?; + let flags = self.memory.read_obj::(GuestAddress(head + 12)).ok()?; + let next = self.memory.read_obj::(GuestAddress(head + 14)).ok()?; - if self.memory.is_valid_range(addr, len as usize) && next < self.queue_size { + if self.memory.check_range(GuestAddress(addr), len as usize) && next < self.queue_size { return Some(Descriptor::new(addr, len, flags, next)); } None @@ -101,7 +101,7 @@ impl SplitQueue { /// Load `avail_ring.idx` from guest memory and store it in `cached_avail_idx`. /// fn load_avail_idx(&self) -> u16 { - let avail_idx = self.memory.read_int::(self.avail_base + 2).unwrap(); + let avail_idx = self.memory.read_obj::(GuestAddress(self.avail_base + 2)).unwrap(); self.cached_avail_idx.set(avail_idx); avail_idx } @@ -112,21 +112,7 @@ impl SplitQueue { /// fn load_avail_entry(&self, ring_idx: u16) -> u16 { let offset = (4 + (ring_idx % self.queue_size) * 2) as u64; - self.memory.read_int(self.avail_base + offset).unwrap() - } - - /// Queue is empty if `next_avail` is same value as - /// `avail_ring.idx` value in guest memory If `cached_avail_idx` - /// currently matches `next_avail` it is reloaded from - /// memory in case guest has updated field since last - /// time it was loaded. - /// - fn is_empty(&self) -> bool { - let next_avail = self.next_avail.get(); - if self.cached_avail_idx.get() != next_avail { - return false; - } - next_avail == self.load_avail_idx() + self.memory.read_obj(GuestAddress(self.avail_base + offset)).unwrap() } /// @@ -147,7 +133,7 @@ impl SplitQueue { } fn read_avail_flags(&self) -> u16 { - self.memory.read_int::(self.avail_base).unwrap() + self.memory.read_obj::(GuestAddress(self.avail_base)).unwrap() } /// @@ -166,14 +152,14 @@ impl SplitQueue { let used_idx = (self.next_used_idx.get() % self.queue_size) as u64; let elem_addr = self.used_base + (4 + used_idx * 8); // write descriptor index to 'next used' slot in used ring - self.memory.write_int(elem_addr, idx as u32).unwrap(); + self.memory.write_obj(idx as u32, GuestAddress(elem_addr)).unwrap(); // write length to 'next used' slot in ring - self.memory.write_int(elem_addr + 4, len as u32).unwrap(); + self.memory.write_obj(len, GuestAddress(elem_addr + 4)).unwrap(); self.next_used_idx.inc(); atomic::fence(Ordering::Release); // write updated next_used - self.memory.write_int(self.used_base + 2, self.next_used_idx.get()).unwrap(); + self.memory.write_obj(self.next_used_idx.get(), GuestAddress(self.used_base + 2)).unwrap(); } /// @@ -187,7 +173,8 @@ impl SplitQueue { return; } let addr = self.used_base + 4 + (self.queue_size as u64 * 8); - self.memory.write_int::(addr, val).unwrap(); + self.memory.write_obj::( val, GuestAddress(addr)).unwrap(); + atomic::fence(Ordering::Release); } @@ -199,7 +186,7 @@ impl SplitQueue { /// Read and return the `used_event` field from the Avail ring fn read_used_event(&self) -> u16 { let addr = self.avail_base + 4 + (self.queue_size as u64 * 2); - self.memory.read_int::(addr).unwrap() + self.memory.read_obj::(GuestAddress(addr)).unwrap() } fn need_interrupt(&self, first_used: u16) -> bool { @@ -217,13 +204,13 @@ impl QueueBackend for SplitQueue { let avail_ring_sz = 6 + 2 * size as usize; let used_ring_sz = 6 + 8 * size as usize; - if !self.memory.is_valid_range(descriptor_area, desc_table_sz) { + if !self.memory.check_range(GuestAddress(descriptor_area), desc_table_sz) { return Err(Error::RangeInvalid(descriptor_area)); } - if !self.memory.is_valid_range(driver_area, avail_ring_sz) { + if !self.memory.check_range(GuestAddress(driver_area), avail_ring_sz) { return Err(Error::AvailInvalid(driver_area)); } - if !self.memory.is_valid_range(device_area, used_ring_sz) { + if !self.memory.check_range(GuestAddress(device_area), used_ring_sz) { return Err(Error::UsedInvalid(device_area)); } diff --git a/src/io/virtio/vq/virtqueue.rs b/src/io/virtio/vq/virtqueue.rs index 4a67619..cf2615f 100644 --- a/src/io/virtio/vq/virtqueue.rs +++ b/src/io/virtio/vq/virtqueue.rs @@ -1,4 +1,5 @@ use std::sync::{Arc, Mutex, MutexGuard}; +use vm_memory::GuestMemoryMmap; use vmm_sys_util::eventfd::EventFd; @@ -7,7 +8,6 @@ use crate::io::virtio::consts::MAX_QUEUE_SIZE; use crate::io::virtio::queues::InterruptLine; use crate::io::virtio::vq::chain::{Chain, DescriptorList}; use crate::io::virtio::vq::splitqueue::SplitQueue; -use crate::memory::GuestRam; pub trait QueueBackend: Send { @@ -44,7 +44,7 @@ pub struct VirtQueue { impl VirtQueue { pub const DEFAULT_QUEUE_SIZE: u16 = 128; - pub fn new(memory: GuestRam, default_size: u16, interrupt: Arc, ioeventfd: Arc) -> Self { + pub fn new(memory: GuestMemoryMmap, default_size: u16, interrupt: Arc, ioeventfd: Arc) -> Self { let backend = Arc::new(Mutex::new(SplitQueue::new(memory, interrupt))); VirtQueue { ioeventfd, diff --git a/src/lib.rs b/src/lib.rs index 2c06133..98c93cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ mod system; #[macro_use] pub mod util; mod vm; -mod memory; mod devices; mod disk; mod io; diff --git a/src/memory/allocator.rs b/src/memory/allocator.rs deleted file mode 100644 index 45a31ea..0000000 --- a/src/memory/allocator.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::memory::AddressRange; -use std::collections::BTreeMap; -use std::sync::{Mutex, Arc}; - -#[derive(Clone)] -pub struct SystemAllocator { - device_memory: AddressAllocator, -} - -impl SystemAllocator { - pub fn new(device_range: AddressRange) -> Self { - let device_memory = AddressAllocator::new(device_range, 4096); - SystemAllocator { device_memory } - } - - pub fn free_device_memory(&self, base: u64) -> bool { - self.device_memory.free(base) - } - - pub fn allocate_device_memory(&self, size: usize) -> Option { - self.device_memory.allocate(size) - } -} - -#[derive(Clone)] -struct AddressAllocator { - range: AddressRange, - default_alignment: usize, - allocations: Arc>>, -} - -impl AddressAllocator { - fn new(range: AddressRange, default_alignment: usize) -> Self { - let allocations = Arc::new(Mutex::new(BTreeMap::new())); - AddressAllocator { range, default_alignment, allocations } - } - - fn align_addr_to(addr: u64, alignment: u64) -> u64 { - let adjust = if addr % alignment != 0 { - alignment - (addr % alignment) - } else { - 0 - }; - addr + adjust - } - - fn allocate(&self, size: usize) -> Option { - self.allocate_aligned(size, self.default_alignment) - } - - fn allocate_aligned(&self, size: usize, alignment: usize) -> Option { - self.first_available(size, alignment) - } - - fn free(&self, base: u64) -> bool { - let mut map = self.allocations.lock().unwrap(); - map.remove(&base).is_some() - } - - fn first_available(&self, size: usize, alignment: usize) -> Option { - if size == 0 { - return None; - } - - let mut map = self.allocations.lock().unwrap(); - let base = self.first_available_base_addr(&map, size, alignment); - let size = size as usize; - if self.range.contains(base, size) { - map.insert(base, AddressRange::new(base, size)); - return Some(base); - } - None - } - - // Return lowest base address of requested alignment which does not - // conflict with any currently allocated range - fn first_available_base_addr(&self, map: &BTreeMap, size: usize, alignment: usize) -> u64 { - let mut base = Self::align_addr_to(self.range.base(), alignment as u64); - for alloc in map.values() { - // Alignment adjustment may have placed address beyond start of next - // allocation. - if let Some(gap_size) = alloc.base().checked_sub(base) { - if (gap_size as usize) >= size { - return base; - } - } - if base < alloc.end() { - base = Self::align_addr_to(alloc.end(), alignment as u64); - } - } - base - } -} diff --git a/src/memory/manager.rs b/src/memory/manager.rs deleted file mode 100644 index c5c4520..0000000 --- a/src/memory/manager.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::collections::HashMap; -use std::os::unix::io::{AsRawFd,RawFd}; -use std::sync::{Arc, RwLock}; - -use crate::memory::{GuestRam, SystemAllocator, Mapping, Error, Result}; -use crate::system::FileDesc; -use crate::util::BitSet; -use crate::memory::drm::{DrmBufferAllocator, DrmDescriptor}; -use std::io::SeekFrom; -use crate::memory::ram::MemoryRegion; -use crate::vm::KvmVm; - -#[derive(Clone)] -pub struct MemoryManager { - kvm_vm: KvmVm, - ram: GuestRam, - device_memory: Arc>, - drm_allocator: Option, -} - -impl MemoryManager { - - pub fn new(kvm_vm: KvmVm, ram: GuestRam, allocator: SystemAllocator, use_drm: bool) -> Result { - let device_memory = RwLock::new(DeviceMemory::new(ram.region_count(), allocator)).into(); - let drm_allocator = if use_drm { - DrmBufferAllocator::open().ok() - } else { - None - }; - Ok(MemoryManager { - kvm_vm, ram, device_memory, - drm_allocator, - }) - } - - pub fn guest_ram(&self) -> &GuestRam { - &self.ram - } - - pub fn kvm_vm(&self) -> &KvmVm { - &self.kvm_vm - } - - pub fn set_ram_regions(&mut self, regions: Vec) { - let mut devmem = self.device_memory.write().unwrap(); - devmem.set_slots_occupied(0, regions.len()); - self.ram.set_regions(regions); - } - - pub fn register_device_memory(&self, fd: RawFd, size: usize) -> Result<(u64, u32)> { - let mut devmem = self.device_memory.write().unwrap(); - devmem.register(self.kvm_vm(), fd, size) - } - - pub fn unregister_device_memory(&self, slot: u32) -> Result<()> { - let mut devmem = self.device_memory.write().unwrap(); - devmem.unregister(self.kvm_vm(), slot) - } - - pub fn drm_available(&self) -> bool { - self.drm_allocator.is_some() - } - - pub fn allocate_drm_buffer(&self, width: u32, height: u32, format: u32) -> Result<(u64, u32, FileDesc, DrmDescriptor)> { - if let Some(drm_allocator) = self.drm_allocator.as_ref() { - let (fd, desc) = drm_allocator.allocate(width, height, format)?; - let size = fd.seek(SeekFrom::End(0)).map_err(Error::CreateBuffer)?; - - let (pfn, slot) = self.register_device_memory(fd.as_raw_fd(), size as usize)?; - Ok((pfn, slot, fd, desc)) - } else { - Err(Error::NoDrmAllocator) - } - } -} - -pub struct MemoryRegistration { - guest_addr: u64, - _mapping: Mapping, -} - -impl MemoryRegistration { - fn new(guest_addr: u64, mapping: Mapping)-> Self { - MemoryRegistration { guest_addr, _mapping: mapping } - } -} - -struct DeviceMemory { - slots: BitSet, - mappings: HashMap, - allocator: SystemAllocator, -} - -impl DeviceMemory { - fn new(ram_region_count: usize, allocator: SystemAllocator) -> DeviceMemory { - let mut devmem = DeviceMemory { - slots: BitSet::new(), - mappings: HashMap::new(), - allocator - }; - devmem.set_slots_occupied(0, ram_region_count); - devmem - } - - fn set_slots_occupied(&mut self, first: usize, count: usize) { - for i in first..first+count { - self.slots.insert(i) - } - } - - fn register(&mut self, kvm_vm: &KvmVm, fd: RawFd, size: usize) -> Result<(u64, u32)> { - let mapping = Mapping::new_from_fd(fd, size) - .map_err(Error::MappingFailed)?; - - let (addr, slot) = self.allocate_addr_and_slot(size)?; - - if let Err(e) = kvm_vm.add_memory_region(slot, addr, mapping.address(), size) { - self.free_addr_and_slot(addr, slot); - Err(Error::RegisterMemoryFailed(e)) - } else { - self.mappings.insert(slot, MemoryRegistration::new(addr, mapping)); - Ok((addr >> 12, slot)) - } - } - - fn unregister(&mut self, kvm_vm: &KvmVm, slot: u32) -> Result<()> { - if let Some(registration) = self.mappings.remove(&slot) { - kvm_vm.remove_memory_region(slot) - .map_err(Error::UnregisterMemoryFailed)?; - self.free_addr_and_slot(registration.guest_addr, slot); - } - Ok(()) - } - - fn allocate_addr_and_slot(&mut self, size: usize) -> Result<(u64, u32)> { - let addr = self.allocator.allocate_device_memory(size) - .ok_or(Error::DeviceMemoryAllocFailed)?; - Ok((addr, self.allocate_slot())) - } - - fn free_addr_and_slot(&mut self, addr: u64, slot: u32) { - self.allocator.free_device_memory(addr); - self.free_slot(slot); - } - - fn allocate_slot(&mut self) -> u32 { - for i in 0.. { - if !self.slots.get(i) { - self.slots.insert(i); - return i as u32; - } - } - unreachable!() - } - - fn free_slot(&mut self, slot: u32) { - self.slots.remove(slot as usize) - } -} diff --git a/src/memory/mmap.rs b/src/memory/mmap.rs deleted file mode 100644 index a7661a4..0000000 --- a/src/memory/mmap.rs +++ /dev/null @@ -1,172 +0,0 @@ -use libc; -use std::ptr; -use std::slice; -use std::mem; -use std::io::Write; -use std::os::unix::io::RawFd; - -use crate::system::{Result,Error}; - -pub struct Mapping { - ptr: *mut u8, - size: usize, -} - -/// Marks types that can be passed to `write_int` and returned from `read_int` -pub unsafe trait Serializable: Copy+Send+Sync {} - -unsafe impl Serializable for u8 {} -unsafe impl Serializable for u16 {} -unsafe impl Serializable for u32 {} -unsafe impl Serializable for u64 {} - -unsafe impl Send for Mapping {} -unsafe impl Sync for Mapping {} - -/// A block of memory returned from the mmap() system call. Provides safe access to the raw -/// memory region. -impl Mapping { - - /// Creates a new anonymous mapping of `size` bytes. - /// - /// # Errors - /// Returns [`Err`] if the `mmap()` system call fails and returns an `Error` representing - /// the system error which occurred. - /// - pub fn new(size: usize) -> Result { - Mapping::_new(size,libc::MAP_ANONYMOUS | libc::MAP_SHARED | libc::MAP_NORESERVE, -1) - } - - /// Creates a new mapping of `size` bytes from the object referenced by file descriptor `fd` - /// - /// # Errors - /// Returns [`Err`] if the `mmap()` system call fails and returns an `Error` representing - /// the system error which occurred. - /// - pub fn new_from_fd(fd: RawFd, size: usize) -> Result { - Mapping::_new(size, libc::MAP_SHARED, fd) - } - - - fn _new(size: usize, flags: libc::c_int, fd: RawFd) -> Result { - let p = unsafe { mmap_allocate(size, flags, fd)? }; - Ok(Mapping { - ptr: p, - size - }) - } - - /// Ensure that `offset` is not larger than this allocation. - /// - /// # Errors - /// - /// Returns [`Err`] of kind `InvalidMappingOffset` if passed an - /// illegal `offset` - /// - fn check_offset(&self, offset: usize) -> Result<()> { - if offset > self.size { - Err(Error::InvalidOffset) - } else { - Ok(()) - } - } - - /// Return the pointer address of this allocation. - pub fn address(&self) -> u64 { - self.ptr as u64 - } - - /// Read and return an integer value in native byte order from `offset` into the memory allocation - /// - /// # Errors - /// Returns [`Err`] of kind `InvalidMappingOffset` if passed an - /// illegal `offset` - /// - pub fn read_int(&self, offset: usize) -> Result { - self.check_offset(offset + mem::size_of::())?; - unsafe { - Ok(ptr::read_volatile(&self.as_slice()[offset..] as *const _ as *const T)) - } - } - - /// Write the integer `val` in native byte order at `offset` into the memory allocation - /// - /// # Errors - /// Returns [`Err`] of kind `InvalidMappingOffset` if passed an - /// illegal `offset` - /// - pub fn write_int(&self, offset: usize, val: T) -> Result<()> { - self.check_offset(offset + mem::size_of::())?; - unsafe { ptr::write_volatile(&mut self.as_mut_slice()[offset..] as *mut _ as *mut T, val); } - Ok(()) - } - - pub fn write_bytes(&self, offset: usize, bytes: &[u8]) -> Result<()> { - self.check_offset(offset + bytes.len())?; - unsafe { - let mut slice: &mut [u8] = &mut self.as_mut_slice()[offset..]; - slice.write_all(bytes).map_err(|_| Error::InvalidOffset) - } - } - - pub fn read_bytes(&self, offset: usize, mut bytes: &mut [u8]) -> Result<()> { - self.check_offset(offset + bytes.len())?; - unsafe { - let slice: &[u8] = &self.as_slice()[offset..]; - bytes.write(slice).unwrap(); - Ok(()) - } - } - - pub fn slice(&self, offset: usize, size: usize) -> Result<&[u8]> { - self.check_offset(offset + size)?; - unsafe { - let x = &self.as_slice()[offset..offset+size]; - Ok(x) - } - } - - pub fn mut_slice(&self, offset: usize, size: usize) -> Result<&mut [u8]> { - self.check_offset(offset + size)?; - unsafe { - let x = &mut self.as_mut_slice()[offset..offset+size]; - Ok(x) - } - } - - #[allow(dead_code)] - pub fn set_mergeable(&self) -> Result<()> { - unsafe { - if libc::madvise(self.ptr as *mut libc::c_void, self.size, libc::MADV_MERGEABLE) == -1 { - return Err(Error::last_os_error()); - } - } - Ok(()) - } - - unsafe fn as_slice(&self) -> &[u8] { - slice::from_raw_parts(self.ptr, self.size) - } - unsafe fn as_mut_slice(&self) -> &mut [u8] { - slice::from_raw_parts_mut(self.ptr, self.size) - } -} - -impl Drop for Mapping { - fn drop(&mut self) { - unsafe { - libc::munmap(self.ptr as *mut libc::c_void, self.size); - } - } -} - -unsafe fn mmap_allocate(size: usize, flags: libc::c_int, fd: libc::c_int) -> Result<*mut u8> { - let p = libc::mmap(ptr::null_mut(), - size, libc::PROT_READ|libc::PROT_WRITE, - flags, fd, 0); - - if p.is_null() || p == libc::MAP_FAILED { - return Err(Error::last_os_error()); - } - Ok(p as *mut u8) -} \ No newline at end of file diff --git a/src/memory/mod.rs b/src/memory/mod.rs deleted file mode 100644 index a94fc43..0000000 --- a/src/memory/mod.rs +++ /dev/null @@ -1,47 +0,0 @@ -mod ram; -mod drm; -mod manager; -mod mmap; -mod address; -mod allocator; - -pub use self::allocator::SystemAllocator; -pub use self::address::AddressRange; -pub use self::mmap::Mapping; -pub use self::ram::{GuestRam,MemoryRegion}; -pub use manager::MemoryManager; - -pub use drm::{DrmDescriptor,DrmPlaneDescriptor}; - -use std::{result, io}; -use crate::system; - -use thiserror::Error; - -#[derive(Debug,Error)] -pub enum Error { - #[error("failed to allocate memory for device")] - DeviceMemoryAllocFailed, - #[error("failed to create memory mapping for device memory: {0}")] - MappingFailed(system::Error), - #[error("failed to register memory for device memory: {0}")] - RegisterMemoryFailed(kvm_ioctls::Error), - #[error("failed to unregister memory for device memory: {0}")] - UnregisterMemoryFailed(kvm_ioctls::Error), - #[error("failed to open device with libgbm: {0}")] - GbmCreateDevice(system::Error), - #[error("failed to allocate buffer with libgbm: {0}")] - GbmCreateBuffer(system::Error), - #[error("error opening render node: {0}")] - OpenRenderNode(io::Error), - #[error("exporting prime handle to fd failed: {0}")] - PrimeHandleToFD(system::ErrnoError), - #[error("failed to create buffer: {0}")] - CreateBuffer(io::Error), - #[error("no DRM allocator is available")] - NoDrmAllocator, -} - -pub type Result = result::Result; - - diff --git a/src/memory/ram.rs b/src/memory/ram.rs deleted file mode 100644 index 4a4750d..0000000 --- a/src/memory/ram.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::sync::Arc; -use std::mem; - -use crate::memory::{Mapping,AddressRange}; -use crate::memory::mmap::Serializable; -use crate::system::{Result, Error}; -use crate::util::ByteBuffer; - -#[derive(Clone)] -pub struct GuestRam { - ram_size: usize, - regions: Arc>, -} - -impl GuestRam { - pub fn new(ram_size: usize) -> GuestRam { - GuestRam { - ram_size, - regions: Vec::new().into(), - } - } - - pub fn ram_size(&self) -> usize { - self.ram_size - } - - pub fn region_count(&self) -> usize { - self.regions.len() - } - - pub fn write_bytes(&self, guest_address: u64, bytes: &[u8]) -> Result<()> { - let region = self.find_region(guest_address, bytes.len())?; - region.write_bytes(guest_address, bytes) - } - - pub fn read_bytes(&self, guest_address: u64, bytes: &mut [u8]) -> Result<()> { - let region = self.find_region(guest_address, bytes.len())?; - region.read_bytes(guest_address, bytes) - } - - pub fn slice(&self, guest_address: u64, size: usize) -> Result<&[u8]> { - let region = self.find_region(guest_address, size)?; - region.slice(guest_address, size) - } - - pub fn mut_buffer(&self, guest_address: u64, size: usize) -> Result> { - let bytes = self.mut_slice(guest_address, size)?; - Ok(ByteBuffer::from_bytes_mut(bytes)) - } - - pub fn mut_slice(&self, guest_address: u64, size: usize) -> Result<&mut[u8]> { - let region = self.find_region(guest_address, size)?; - region.mut_slice(guest_address, size) - } - - pub fn write_int(&self, guest_address: u64, val: T) -> Result<()> { - let region = self.find_region(guest_address, mem::size_of::())?; - region.write_int(guest_address, val) - } - - pub fn read_int(&self, guest_address: u64) -> Result { - let region = self.find_region(guest_address, mem::size_of::())?; - region.read_int(guest_address) - } - - pub fn set_regions(&mut self, regions: Vec) { - self.regions = regions.into(); - } - - #[allow(dead_code)] - pub fn end_addr(&self) -> u64 { - self.regions.iter() - .max_by_key(|r| r.guest_range.end()) - .map_or(0, |r| r.guest_range.end()) - } - - pub fn is_valid_range(&self, guest_address: u64, size: usize) -> bool { - self.find_region(guest_address, size).is_ok() - } - - fn find_region(&self, guest_address: u64, size: usize) -> Result<&MemoryRegion> { - self.regions.iter() - - .find(|r| r.contains(guest_address, size)) - .ok_or(Error::InvalidAddress(guest_address)) - } -} - -pub struct MemoryRegion { - guest_range: AddressRange, - mapping: Mapping, -} - -impl MemoryRegion { - pub fn new(guest_base: u64, size: usize) -> Result { - Ok(MemoryRegion{ - guest_range: AddressRange::new(guest_base, size), - mapping: Mapping::new(size)?, - }) - } - - pub fn base_address(&self) -> u64 { - self.mapping.address() - } - - fn contains(&self, guest_addr: u64, size: usize) -> bool { self.guest_range.contains(guest_addr, size) } - - fn checked_offset(&self, guest_addr: u64, size: usize) -> Result { - if self.contains(guest_addr, size) { - Ok(self.guest_range.offset_of(guest_addr)) - } else { - Err(Error::InvalidAddress(guest_addr)) - } - } - - pub fn write_bytes(&self, guest_address: u64, bytes: &[u8]) -> Result<()> { - let offset = self.checked_offset(guest_address, bytes.len())?; - self.mapping.write_bytes(offset, bytes) - } - - pub fn read_bytes(&self, guest_address: u64, bytes: &mut [u8]) -> Result<()> { - let offset = self.checked_offset(guest_address, bytes.len())?; - self.mapping.read_bytes(offset, bytes) - } - - pub fn slice(&self, guest_address: u64, size: usize) -> Result<&[u8]> { - let offset = self.checked_offset(guest_address, size)?; - self.mapping.slice(offset, size) - } - - pub fn mut_slice(&self, guest_address: u64, size: usize) -> Result<&mut [u8]> { - let offset = self.checked_offset(guest_address, size)?; - self.mapping.mut_slice(offset, size) - } - - pub fn write_int(&self, guest_address: u64, val: T) -> Result<()> { - let offset = self.checked_offset(guest_address, mem::size_of::())?; - self.mapping.write_int(offset, val) - } - - pub fn read_int(&self, guest_address: u64) -> Result { - let offset = self.checked_offset(guest_address, mem::size_of::())?; - self.mapping.read_int(offset) - } -} diff --git a/src/memory/drm.rs b/src/system/drm.rs similarity index 86% rename from src/memory/drm.rs rename to src/system/drm.rs index af62825..b1807ac 100644 --- a/src/memory/drm.rs +++ b/src/system/drm.rs @@ -1,19 +1,37 @@ use std::fs::{OpenOptions, File}; +use std::os::fd::FromRawFd; use std::os::raw::{c_ulong, c_int, c_uint}; use std::os::unix::io::{RawFd,AsRawFd}; use std::path::Path; +use std::{io, result}; use std::sync::Arc; -use crate::system::{self, ioctl::ioctl_with_mut_ref, FileDesc }; -use crate::memory::{Error,Result}; +use crate::system::{self, ioctl::ioctl_with_mut_ref}; -#[derive(Default,Debug)] +use thiserror::Error; + +pub type Result = result::Result; + +#[derive(Debug,Error)] +pub enum Error { + #[error("failed to open device with libgbm: {0}")] + GbmCreateDevice(system::Error), + #[error("failed to allocate buffer with libgbm: {0}")] + GbmCreateBuffer(system::Error), + #[error("error opening render node: {0}")] + OpenRenderNode(io::Error), + #[error("exporting prime handle to fd failed: {0}")] + PrimeHandleToFD(system::ErrnoError), +} + + +#[derive(Default,Debug,Copy,Clone)] pub struct DrmPlaneDescriptor { pub stride: u32, pub offset: u32, } -#[derive(Default,Debug)] +#[derive(Default,Debug,Copy,Clone)] pub struct DrmDescriptor { pub planes: [DrmPlaneDescriptor; 3] } @@ -32,7 +50,7 @@ impl DrmBufferAllocator { }) } - pub fn allocate(&self, width: u32, height: u32, format: u32) -> Result<(FileDesc, DrmDescriptor)> { + pub fn allocate(&self, width: u32, height: u32, format: u32) -> Result<(File, DrmDescriptor)> { const GBM_BO_USE_LINEAR: u32 = 16; let buffer = self.create_buffer(width, height, format, GBM_BO_USE_LINEAR)?; @@ -131,7 +149,7 @@ impl DrmBuffer { unsafe { gbm_bo_get_stride_for_plane(self.bo, plane) } } - fn buffer_fd(&self) -> Result { + fn buffer_fd(&self) -> Result { const DRM_CLOEXEC: u32 = libc::O_CLOEXEC as u32; const DRM_RDWR: u32 = libc::O_RDWR as u32; let mut prime = DrmPrimeHandle { @@ -143,8 +161,8 @@ impl DrmBuffer { unsafe { ioctl_with_mut_ref(self.dev.as_raw_fd(), DRM_IOCTL_PRIME_HANDLE_TO_FD, &mut prime) .map_err(Error::PrimeHandleToFD)?; + Ok(File::from_raw_fd(prime.fd)) } - Ok(FileDesc::new(prime.fd)) } } diff --git a/src/system/filedesc.rs b/src/system/filedesc.rs deleted file mode 100644 index f6b43f1..0000000 --- a/src/system/filedesc.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::os::unix::io::{IntoRawFd,AsRawFd,RawFd}; -use std::{mem, io}; -use crate::system::errno::cvt; -use std::os::raw::c_void; -use libc::c_int; -use std::io::SeekFrom; - -#[derive(Debug)] -pub struct FileDesc { - fd: RawFd, -} - -#[derive(Copy,Clone,Debug,Eq,PartialEq)] -pub enum FileFlags { - Read, - Write, - ReadWrite, -} - -#[allow(dead_code)] -impl FileDesc { - pub fn new(fd: RawFd) -> Self { - FileDesc { fd } - } - - pub fn read(&self, buf: &mut [u8]) -> io::Result { - let ret = cvt(unsafe { - libc::read(self.fd, buf.as_mut_ptr() as *mut c_void, buf.len()) - })?; - Ok(ret as usize) - } - - pub fn read_exact(&mut self, mut buf: &mut [u8]) -> io::Result<()> { - while !buf.is_empty() { - match self.read(buf) { - Ok(0) => break, - Ok(n) => { let tmp = buf; buf = &mut tmp[n..]; } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), - } - } - if !buf.is_empty() { - Err(io::Error::new(io::ErrorKind::UnexpectedEof, - "failed to fill whole buffer")) - } else { - Ok(()) - } - } - pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { - unsafe { - let v = nonblocking as c_int; - cvt(libc::ioctl(self.fd, libc::FIONBIO, &v))?; - Ok(()) - } - } - pub fn set_cloexec(&self) -> io::Result<()> { - unsafe { - cvt(libc::ioctl(self.fd, libc::FIOCLEX))?; - Ok(()) - } - } - - pub fn flags(&self) -> io::Result { - let flags = unsafe { cvt(libc::fcntl(self.fd, libc::F_GETFL))? }; - match flags & libc::O_ACCMODE { - libc::O_RDONLY => Ok(FileFlags::Read), - libc::O_WRONLY => Ok(FileFlags::Write), - libc::O_RDWR => Ok(FileFlags::ReadWrite), - _ => Err(io::Error::from_raw_os_error(libc::EINVAL)), - } - } - - pub fn seek(&self, pos: SeekFrom) -> io::Result { - let (whence, pos) = match pos { - // Casting to `i64` is fine, too large values will end up as - // negative which will cause an error in `lseek64`. - SeekFrom::Start(off) => (libc::SEEK_SET, off as i64), - SeekFrom::End(off) => (libc::SEEK_END, off), - SeekFrom::Current(off) => (libc::SEEK_CUR, off), - }; - let n = cvt(unsafe { libc::lseek64(self.fd, pos, whence) })?; - Ok(n as u64) - } - pub fn write(&self, buf: &[u8]) -> io::Result { - let ret = cvt(unsafe { - libc::write(self.fd, - buf.as_ptr() as *const c_void, - buf.len()) - })?; - Ok(ret as usize) - } - - pub fn write_all(&self, buf: &[u8]) -> io::Result<()> { - let mut buf = buf; - while !buf.is_empty() { - match self.write(buf) { - Ok(0) => return Err(io::Error::new(io::ErrorKind::WriteZero, - "failed to write whole buffer")), - Ok(n) => { - buf = &buf[n..]; - } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), - } - } - Ok(()) - - - } - - pub fn set_size(&self, size: usize) -> io::Result<()> { - unsafe { - if libc::ftruncate64(self.fd, size as libc::off64_t) < 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } - } - } - - pub fn flush(&self) -> io::Result<()> { - Ok(()) - } -} - -impl Drop for FileDesc { - fn drop(&mut self) { - let _ = unsafe { libc::close(self.fd) }; - } -} -impl AsRawFd for FileDesc { - fn as_raw_fd(&self) -> RawFd { - self.fd - } -} - -impl IntoRawFd for FileDesc { - fn into_raw_fd(self) -> RawFd { - let fd = self.fd; - mem::forget(self); - fd - } -} \ No newline at end of file diff --git a/src/system/memfd.rs b/src/system/memfd.rs deleted file mode 100644 index b545c8d..0000000 --- a/src/system/memfd.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::ffi::CString; -use std::io::SeekFrom; -use std::os::unix::io::{RawFd,AsRawFd}; - -use crate::system::{Error,Result, FileDesc}; - -use libc::{ - self, c_char, c_uint, c_int, c_long,SYS_memfd_create, - MFD_CLOEXEC, MFD_ALLOW_SEALING, F_SEAL_GROW,F_SEAL_SHRINK, F_SEAL_SEAL -}; - - -#[derive(Debug)] -pub struct MemoryFd { - fd: FileDesc, - size: usize, -} - -impl MemoryFd { - - pub fn from_filedesc(fd: FileDesc) -> Result { - let size = fd.seek(SeekFrom::End(0))? as usize; - Ok(MemoryFd { fd, size }) - } - - pub fn new_memfd(size: usize, sealed: bool) -> Result { - Self::new_memfd_with_name(size, sealed, "pH-memfd") - } - - pub fn new_memfd_with_name(size: usize, sealed: bool, name: &str) -> Result { - let fd = Self::memfd_create(name, MFD_CLOEXEC | MFD_ALLOW_SEALING)?; - fd.set_size(size)?; - - let memfd = MemoryFd { fd, size }; - if sealed { - memfd.add_seals(F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL)?; - } - - Ok(memfd) - } - - pub fn size(&self) -> usize { - self.size - } - - pub fn fd_mut(&mut self) -> &mut FileDesc { - &mut self.fd - } - - fn memfd_create(name: &str, flags: c_uint) -> Result { - let name = CString::new(name).expect("Cstring from &str"); - let name = name.as_ptr() as *const c_char; - let fd = unsafe { libc::syscall(SYS_memfd_create as c_long, name, flags) } as c_int; - if fd < 0 { - Err(Error::last_os_error()) - } else { - Ok(FileDesc::new(fd)) - } - } - - fn add_seals(&self, flags: c_int) -> Result<()> { - let ret = unsafe { libc::fcntl(self.fd.as_raw_fd(), libc::F_ADD_SEALS, flags) }; - if ret < 0 { - Err(Error::last_os_error()) - } else { - Ok(()) - } - } - -} - -impl AsRawFd for MemoryFd { - fn as_raw_fd(&self) -> RawFd { - self.fd.as_raw_fd() - } -} diff --git a/src/system/mod.rs b/src/system/mod.rs index 5ffd4fd..3be7441 100644 --- a/src/system/mod.rs +++ b/src/system/mod.rs @@ -1,15 +1,11 @@ #[macro_use]pub mod ioctl; mod epoll; -mod errno; +pub mod errno; mod socket; -mod filedesc; -mod memfd; mod tap; -//pub mod priority; pub mod netlink; +pub mod drm; -pub use filedesc::{FileDesc, FileFlags}; -pub use memfd::MemoryFd; pub use epoll::{EPoll,Event}; pub use socket::ScmSocket; pub use netlink::NetlinkSocket; @@ -19,6 +15,8 @@ use std::{result, io}; pub use errno::Error as ErrnoError; use thiserror::Error; +use vm_memory::guest_memory; +use vm_memory::mmap::MmapRegionError; pub type Result = result::Result; @@ -38,6 +36,12 @@ pub enum Error { EventFdWrite, #[error("failed reading from eventfd")] EventFdRead, + #[error("guest memory error: {0}")] + GuestMemory(guest_memory::Error), + #[error("failed to allocate shared memory: {0}")] + ShmAllocFailed(memfd::Error), + #[error("failed to create mmap region: {0}")] + MmapRegionCreate(MmapRegionError) } @@ -70,6 +74,12 @@ impl Error { } } +impl From for Error { + fn from(err: guest_memory::Error) -> Error { + Error::GuestMemory(err) + } +} + impl From for Error { fn from(err: errno::Error) -> Error { Error::Errno(err) diff --git a/src/vm/arch/error.rs b/src/vm/arch/error.rs index a25a658..4f1da5e 100644 --- a/src/vm/arch/error.rs +++ b/src/vm/arch/error.rs @@ -1,13 +1,14 @@ -use crate::{system, memory}; +use crate::system; use crate::system::ErrnoError; use std::result; use kvm_ioctls::Cap; use thiserror::Error; +use vm_memory::guest_memory; #[derive(Debug,Error)] pub enum Error { #[error("failed to create memory manager: {0}")] - MemoryManagerCreate(memory::Error), + MemoryManagerCreate(vm_memory::Error), #[error("failed to register memory region: {0}")] MemoryRegister(kvm_ioctls::Error), #[error("failed to create memory region: {0}")] @@ -24,6 +25,8 @@ pub enum Error { IoctlError(&'static str, ErrnoError), #[error("error setting up vm: {0}")] SetupError(kvm_ioctls::Error), + #[error("guest memory error: {0}")] + GuestMemory(guest_memory::Error), } pub type Result = result::Result; diff --git a/src/vm/arch/mod.rs b/src/vm/arch/mod.rs index 07df74c..fb61ac2 100644 --- a/src/vm/arch/mod.rs +++ b/src/vm/arch/mod.rs @@ -1,7 +1,7 @@ use kvm_bindings::CpuId; use kvm_ioctls::VcpuFd; +use vm_memory::GuestMemoryMmap; pub use crate::vm::arch::x86::X86ArchSetup; -use crate::memory::MemoryManager; mod error; mod x86; @@ -20,7 +20,7 @@ pub fn create_setup(config: &VmConfig) -> X86ArchSetup { } pub trait ArchSetup { - fn create_memory(&mut self, kvm_vm: KvmVm) -> Result; + fn create_memory(&mut self, kvm_vm: KvmVm) -> Result; fn setup_memory(&mut self, cmdline: &KernelCmdLine, pci_irqs: &[PciIrq]) -> Result<()>; fn setup_vcpu(&self, vcpu: &VcpuFd, cpuid: CpuId) -> Result<()>; } diff --git a/src/vm/arch/x86/kernel.rs b/src/vm/arch/x86/kernel.rs index 45a93f8..33458e9 100644 --- a/src/vm/arch/x86/kernel.rs +++ b/src/vm/arch/x86/kernel.rs @@ -1,6 +1,6 @@ use std::io; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; -use crate::memory::GuestRam; use crate::system; use crate::util::ByteBuffer; use crate::vm::arch::PCI_MMIO_RESERVED_BASE; @@ -33,8 +33,8 @@ const KERNEL_MIN_ALIGNMENT_BYTES: u32 = 0x1000000; const E820_RAM: u32 = 1; -fn setup_e820(memory: &GuestRam, mut zero: ByteBuffer<&mut [u8]>) -> system::Result<()> { - let ram_size = memory.ram_size() as u64; +fn setup_e820(ram_size: usize, zero: &mut ByteBuffer>) -> system::Result<()> { + let ram_size = ram_size as u64; let mut e820_ranges = Vec::new(); e820_ranges.push((0u64, EBDA_START)); @@ -56,8 +56,8 @@ fn setup_e820(memory: &GuestRam, mut zero: ByteBuffer<&mut [u8]>) -> system::Res Ok(()) } -fn setup_zero_page(memory: &GuestRam, cmdline_addr: u64, cmdline_size: usize) -> system::Result<()> { - let mut zero = memory.mut_buffer(KERNEL_ZERO_PAGE, 4096)?; +fn setup_zero_page(ram_size: usize, memory: &GuestMemoryMmap, cmdline_addr: u64, cmdline_size: usize) -> system::Result<()> { + let mut zero = ByteBuffer::new(4096); zero.write_at(HDR_BOOT_FLAG, KERNEL_BOOT_FLAG_MAGIC) .write_at(HDR_HEADER, KERNEL_HDR_MAGIC) .write_at(HDR_TYPE_LOADER, KERNEL_LOADER_OTHER) @@ -65,24 +65,27 @@ fn setup_zero_page(memory: &GuestRam, cmdline_addr: u64, cmdline_size: usize) -> .write_at(HDR_CMDLINE_SIZE, cmdline_size as u32) .write_at(HDR_KERNEL_ALIGNMENT, KERNEL_MIN_ALIGNMENT_BYTES); - setup_e820(memory, zero) + setup_e820(ram_size, &mut zero)?; + memory.write_slice(zero.as_ref(), GuestAddress(KERNEL_ZERO_PAGE))?; + Ok(()) + } -pub fn load_pm_kernel(memory: &GuestRam, cmdline_addr: u64, cmdline_size: usize) -> system::Result<()> { +pub fn load_pm_kernel(ram_size: usize, memory: &GuestMemoryMmap, cmdline_addr: u64, cmdline_size: usize) -> system::Result<()> { load_elf_kernel(memory)?; - setup_zero_page(memory, cmdline_addr, cmdline_size) + setup_zero_page(ram_size, memory, cmdline_addr, cmdline_size) } -fn load_elf_segment(memory: &GuestRam, hdr: ElfPhdr) { +fn load_elf_segment(memory: &GuestMemoryMmap, hdr: ElfPhdr) { let addr = hdr.p_paddr + KVM_KERNEL_LOAD_ADDRESS; let size = hdr.p_filesz as usize; let off = hdr.p_offset as usize; - let dst = memory.mut_slice(addr, size).unwrap(); + let src = &KERNEL[off..off+size]; - dst.copy_from_slice(src); + memory.write_slice(src, GuestAddress(addr)).unwrap(); } -pub fn load_elf_kernel(memory: &GuestRam) -> io::Result<()> { +pub fn load_elf_kernel(memory: &GuestMemoryMmap) -> io::Result<()> { let mut k = ByteBuffer::from_bytes(KERNEL); let phoff = k.read_at::(32); let phnum = k.read_at::(56); diff --git a/src/vm/arch/x86/memory.rs b/src/vm/arch/x86/memory.rs index fb20f1c..5245999 100644 --- a/src/vm/arch/x86/memory.rs +++ b/src/vm/arch/x86/memory.rs @@ -1,12 +1,10 @@ -use crate::memory::{MemoryManager, MemoryRegion, GuestRam}; use crate::vm::arch::{Error, Result}; -use std::cmp; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; use crate::io::PciIrq; use crate::vm::kernel_cmdline::KernelCmdLine; use crate::vm::arch::x86::kernel::{load_pm_kernel, KERNEL_CMDLINE_ADDRESS}; use crate::system; use crate::vm::arch::x86::mptable::setup_mptable; -use crate::vm::KvmVm; pub const HIMEM_BASE: u64 = 1 << 32; pub const PCI_MMIO_RESERVED_SIZE: usize = 512 << 20; @@ -14,28 +12,6 @@ pub const PCI_MMIO_RESERVED_BASE: u64 = HIMEM_BASE - PCI_MMIO_RESERVED_SIZE as u pub const IRQ_BASE: u32 = 5; pub const IRQ_MAX: u32 = 23; - -pub fn x86_setup_memory_regions(memory: &mut MemoryManager, ram_size: usize) -> Result<()> { - let mut regions = Vec::new(); - let lowmem_sz = cmp::min(ram_size, PCI_MMIO_RESERVED_BASE as usize); - regions.push(create_region(memory.kvm_vm(), 0, lowmem_sz, 0)?); - - if lowmem_sz < ram_size { - let himem_sz = ram_size - lowmem_sz; - regions.push(create_region(memory.kvm_vm(), HIMEM_BASE, himem_sz, 1)?); - } - memory.set_ram_regions(regions); - Ok(()) -} - -fn create_region(kvm_vm: &KvmVm, base: u64, size: usize, slot: u32) -> Result { - let mr = MemoryRegion::new(base, size) - .map_err(Error::MemoryRegionCreate)?; - kvm_vm.add_memory_region(slot, base, mr.base_address(), size) - .map_err(Error::MemoryRegister)?; - Ok(mr) -} - const BOOT_GDT_OFFSET: usize = 0x500; const BOOT_IDT_OFFSET: usize = 0x520; @@ -43,29 +19,29 @@ const BOOT_PML4: u64 = 0x9000; const BOOT_PDPTE: u64 = 0xA000; const BOOT_PDE: u64 = 0xB000; -pub fn x86_setup_memory(memory: &mut MemoryManager, cmdline: &KernelCmdLine, ncpus: usize, pci_irqs: &[PciIrq]) -> Result<()> { - load_pm_kernel(memory.guest_ram(), KERNEL_CMDLINE_ADDRESS, cmdline.size()) +pub fn x86_setup_memory(ram_size: usize, memory: &GuestMemoryMmap, cmdline: &KernelCmdLine, ncpus: usize, pci_irqs: &[PciIrq]) -> Result<()> { + load_pm_kernel(ram_size, memory, KERNEL_CMDLINE_ADDRESS, cmdline.size()) .map_err(Error::LoadKernel)?; - setup_gdt(memory.guest_ram())?; - setup_boot_pagetables(memory.guest_ram()).map_err(Error::SystemError)?; - setup_mptable(memory.guest_ram(), ncpus, pci_irqs).map_err(Error::SystemError)?; - write_cmdline(memory.guest_ram(), cmdline).map_err(Error::SystemError)?; + setup_gdt(memory)?; + setup_boot_pagetables(memory).map_err(Error::SystemError)?; + setup_mptable(memory, ncpus, pci_irqs).map_err(Error::SystemError)?; + write_cmdline(memory, cmdline).map_err(Error::SystemError)?; Ok(()) } -fn setup_boot_pagetables(memory: &GuestRam) -> system::Result<()> { - memory.write_int::(BOOT_PML4, BOOT_PDPTE | 0x3)?; - memory.write_int::(BOOT_PDPTE, BOOT_PDE | 0x3)?; +fn setup_boot_pagetables(memory: &GuestMemoryMmap) -> system::Result<()> { + memory.write_obj(BOOT_PDPTE | 0x3, GuestAddress(BOOT_PML4))?; + memory.write_obj(BOOT_PDE | 0x3, GuestAddress(BOOT_PDPTE))?; for i in 0..512_u64 { let entry = (i << 21) | 0x83; - memory.write_int::(BOOT_PDE + (i * 8), entry)?; + memory.write_obj(entry, GuestAddress(BOOT_PDE + (i * 8)))?; } Ok(()) } -fn write_gdt_table(table: &[u64], memory: &GuestRam) -> system::Result<()> { +fn write_gdt_table(table: &[u64], memory: &GuestMemoryMmap) -> system::Result<()> { for i in 0..table.len() { - memory.write_int((BOOT_GDT_OFFSET + i * 8) as u64, table[i])?; + memory.write_obj(table[i], GuestAddress((BOOT_GDT_OFFSET + i * 8) as u64))?; } Ok(()) } @@ -76,7 +52,7 @@ pub fn gdt_entry(flags: u16, base: u32, limit: u32) -> u64 { (((base as u64) & 0x00ffffffu64) << 16) | ((limit as u64) & 0x0000ffffu64) } -pub fn setup_gdt(memory: &GuestRam) -> Result<()> { +pub fn setup_gdt(memory: &GuestMemoryMmap) -> Result<()> { let table = [ gdt_entry(0,0,0), gdt_entry(0xa09b,0,0xfffff), @@ -86,16 +62,16 @@ pub fn setup_gdt(memory: &GuestRam) -> Result<()> { write_gdt_table(&table, memory) .map_err(Error::SystemError)?; - memory.write_int::(BOOT_IDT_OFFSET as u64, 0u64) - .map_err(Error::SystemError)?; + memory.write_obj(0u64, GuestAddress(BOOT_IDT_OFFSET as u64)) + .map_err(Error::GuestMemory)?; Ok(()) } -fn write_cmdline(memory: &GuestRam, cmdline: &KernelCmdLine) -> system::Result<()> { +fn write_cmdline(memory: &GuestMemoryMmap, cmdline: &KernelCmdLine) -> system::Result<()> { let bytes = cmdline.as_bytes(); let len = bytes.len() as u64; - memory.write_bytes(KERNEL_CMDLINE_ADDRESS, bytes)?; - memory.write_int(KERNEL_CMDLINE_ADDRESS + len, 0u8)?; + memory.write_slice(bytes, GuestAddress(KERNEL_CMDLINE_ADDRESS))?; + memory.write_obj(0u8, GuestAddress(KERNEL_CMDLINE_ADDRESS + len))?; Ok(()) } diff --git a/src/vm/arch/x86/mptable.rs b/src/vm/arch/x86/mptable.rs index 81f55d5..a2bfe37 100644 --- a/src/vm/arch/x86/mptable.rs +++ b/src/vm/arch/x86/mptable.rs @@ -1,7 +1,7 @@ use std::iter; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; use crate::io::PciIrq; -use crate::memory::GuestRam; use crate::system::Result; use crate::util::ByteBuffer; @@ -201,7 +201,7 @@ fn align(sz: usize, n: usize) -> usize { (sz + (n - 1)) & !(n - 1) } -pub fn setup_mptable(memory: &GuestRam, ncpus: usize, pci_irqs: &[PciIrq]) -> Result<()> { +pub fn setup_mptable(memory: &GuestMemoryMmap, ncpus: usize, pci_irqs: &[PciIrq]) -> Result<()> { let ioapicid = (ncpus + 1) as u8; let mut buffer = Buffer::new(); let address = MPTABLE_START; @@ -217,5 +217,6 @@ pub fn setup_mptable(memory: &GuestRam, ncpus: usize, pci_irqs: &[PciIrq]) -> Re .write_mpc_lintsrc(MP_IRQ_SRC_NMI, 1) .write_mpc_table(MPF_INTEL_SIZE); - memory.write_bytes(address, buffer.buffer.as_ref()) + memory.write_slice(buffer.buffer.as_ref(), GuestAddress(address))?; + Ok(()) } \ No newline at end of file diff --git a/src/vm/arch/x86/setup.rs b/src/vm/arch/x86/setup.rs index da0c355..cb91e5e 100644 --- a/src/vm/arch/x86/setup.rs +++ b/src/vm/arch/x86/setup.rs @@ -1,11 +1,11 @@ use kvm_bindings::CpuId; use kvm_ioctls::VcpuFd; +use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion}; use crate::io::PciIrq; -use crate::memory::{MemoryManager, GuestRam, SystemAllocator, AddressRange}; use crate::vm::VmConfig; use crate::vm::arch::{ArchSetup, Error, PCI_MMIO_RESERVED_BASE, Result}; use crate::vm::kernel_cmdline::KernelCmdLine; -use crate::vm::arch::x86::memory::{x86_setup_memory_regions, x86_setup_memory, HIMEM_BASE}; +use crate::vm::arch::x86::memory::{x86_setup_memory, HIMEM_BASE}; use crate::vm::arch::x86::cpuid::setup_cpuid; use crate::vm::arch::x86::registers::{setup_pm_sregs, setup_pm_regs, setup_fpu, setup_msrs}; use crate::vm::arch::x86::interrupts::setup_lapic; @@ -14,65 +14,52 @@ use crate::vm::kvm_vm::KvmVm; pub struct X86ArchSetup { ram_size: usize, - use_drm: bool, ncpus: usize, - memory: Option, + memory: Option, } impl X86ArchSetup { pub fn create(config: &VmConfig) -> Self { let ram_size = config.ram_size(); - let use_drm = config.is_wayland_enabled() && config.is_dmabuf_enabled(); X86ArchSetup { ram_size, - use_drm, ncpus: config.ncpus(), memory: None, } } } -fn arch_memory_regions(mem_size: usize) -> Vec<(u64, usize)> { +fn x86_memory_ranges(mem_size: usize) -> Vec<(GuestAddress, usize)> { match mem_size.checked_sub(PCI_MMIO_RESERVED_BASE as usize) { - None | Some(0) => vec![(0, mem_size)], + None | Some(0) => vec![(GuestAddress(0), mem_size)], Some(remaining) => vec![ - (0, PCI_MMIO_RESERVED_BASE as usize), - (HIMEM_BASE, remaining), + (GuestAddress(0), PCI_MMIO_RESERVED_BASE as usize), + (GuestAddress(HIMEM_BASE), remaining), ], } } -fn device_memory_start(regions: &[(u64, usize)]) -> u64 { - let top = regions.last().map(|&(base, size)| { - base + size as u64 - }).unwrap(); - // Put device memory at a 2MB boundary after physical memory or 4gb, whichever is greater. - const MB: u64 = 1 << 20; - const TWO_MB: u64 = 2 * MB; - const FOUR_GB: u64 = 4 * 1024 * MB; - let dev_base_round_2mb = (top + TWO_MB - 1) & !(TWO_MB - 1); - std::cmp::max(dev_base_round_2mb, FOUR_GB) -} - impl ArchSetup for X86ArchSetup { - fn create_memory(&mut self, kvm_vm: KvmVm) -> Result { - let ram = GuestRam::new(self.ram_size); - let regions = arch_memory_regions(self.ram_size); - let dev_addr_start = device_memory_start(®ions); - - let dev_addr_size = u64::MAX - dev_addr_start; - let allocator = SystemAllocator::new(AddressRange::new(dev_addr_start,dev_addr_size as usize)); - let mut mm = MemoryManager::new(kvm_vm, ram, allocator, self.use_drm) + fn create_memory(&mut self, kvm_vm: KvmVm) -> Result { + let ranges = x86_memory_ranges(self.ram_size); + let guest_memory = GuestMemoryMmap::from_ranges(&ranges) .map_err(Error::MemoryManagerCreate)?; - x86_setup_memory_regions(&mut mm, self.ram_size)?; - self.memory = Some(mm.clone()); - Ok(mm) + + for (i, r) in guest_memory.iter().enumerate() { + let slot = i as u32; + let guest_address = r.start_addr().raw_value(); + let size = r.len() as usize; + let host_address = guest_memory.get_host_address(r.start_addr()).unwrap() as u64; + kvm_vm.add_memory_region(slot, guest_address, host_address, size).map_err(Error::MemoryRegister)?; + } + self.memory = Some(guest_memory.clone()); + Ok(guest_memory) } fn setup_memory(&mut self, cmdline: &KernelCmdLine, pci_irqs: &[PciIrq]) -> Result<()> { let memory = self.memory.as_mut().expect("No memory created"); - x86_setup_memory(memory, cmdline, self.ncpus, pci_irqs)?; + x86_setup_memory(self.ram_size, memory, cmdline, self.ncpus, pci_irqs)?; Ok(()) } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index f71b845..c81076b 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -15,6 +15,6 @@ pub use setup::VmSetup; pub use kvm_vm::KvmVm; pub use self::error::{Result,Error}; -pub use arch::{ArchSetup,create_setup}; +pub use arch::ArchSetup; diff --git a/src/vm/setup.rs b/src/vm/setup.rs index 987c8b7..9498352 100644 --- a/src/vm/setup.rs +++ b/src/vm/setup.rs @@ -7,9 +7,9 @@ use std::{env, fs, thread}; use crate::system::{Tap, NetlinkSocket}; use crate::disk::DiskImage; use std::sync::{Arc, Barrier, Mutex}; -use crate::memory::MemoryManager; use std::sync::atomic::AtomicBool; use kvm_ioctls::VmFd; +use vm_memory::GuestMemoryMmap; use vmm_sys_util::eventfd::EventFd; use crate::devices::ac97::Ac97Dev; use crate::devices::serial::SerialPort; @@ -21,7 +21,7 @@ use crate::vm::vcpu::Vcpu; pub struct Vm { kvm_vm: KvmVm, vcpus: Vec, - memory: MemoryManager, + memory: GuestMemoryMmap, io_manager: IoManager, termios: Option, } @@ -36,7 +36,7 @@ impl Vm { let memory = arch.create_memory(kvm_vm.clone()) .map_err(Error::ArchError)?; - let io_manager = IoManager::new(memory.clone()); + let io_manager = IoManager::new(kvm_vm.clone(), memory.clone()); Ok(Vm { kvm_vm, @@ -75,6 +75,10 @@ impl Vm { self.kvm_vm.vm_fd() } + pub fn guest_memory(&self) -> &GuestMemoryMmap { + &self.memory + } + } pub struct VmSetup { @@ -111,7 +115,7 @@ impl VmSetup { if self.config.rootshell() { self.cmdline.push("phinit.rootshell"); } - if vm.memory.drm_available() && self.config.is_dmabuf_enabled() { + if self.config.is_wayland_enabled() && self.config.is_dmabuf_enabled() { self.cmdline.push("phinit.virtwl_dmabuf"); } @@ -134,9 +138,8 @@ impl VmSetup { env::set_var("HOME", "/home/citadel"); env::set_var("XDG_RUNTIME_DIR", "/run/user/1000"); let irq = vm.io_manager.allocator().allocate_irq(); - let mem = vm.memory.guest_ram().clone(); // XXX expect() - let ac97 = Ac97Dev::try_new(&vm.kvm_vm, irq, mem).expect("audio initialize error"); + let ac97 = Ac97Dev::try_new(&vm.kvm_vm, irq, vm.guest_memory()).expect("audio initialize error"); vm.io_manager.add_pci_device(Arc::new(Mutex::new(ac97))); } @@ -162,7 +165,8 @@ impl VmSetup { io_manager.add_virtio_device(VirtioRandom::new())?; if self.config.is_wayland_enabled() { - io_manager.add_virtio_device(VirtioWayland::new(self.config.is_dmabuf_enabled()))?; + let dev_shm_manager = io_manager.dev_shm_manager().clone(); + io_manager.add_virtio_device(VirtioWayland::new(self.config.is_dmabuf_enabled(), dev_shm_manager))?; } let homedir = self.config.homedir();