fault_inject_preload/
lib.rs

1//! LD_PRELOAD shim that faults writes aimed at configured path prefixes, used
2//! by integration tests to reproduce storage faults (a full disk, a failing or
3//! slow device) that are otherwise hard to provoke deterministically without
4//! root.
5//!
6//! A test scopes injection by directory, filename, and time, letting it fault
7//! one subsystem while the rest of the process keeps working. The directory
8//! scope is a colon-separated list of absolute prefixes in
9//! `KUMO_FAULT_PATH_PREFIXES`. The filename scope is an optional
10//! colon-separated list of suffixes in `KUMO_FAULT_PATH_SUFFIXES` (e.g. `.sst`
11//! to hit only flushed table files and leave the write-ahead log untouched).
12//! The time scope is a sentinel file at `KUMO_FAULT_ACTIVE_FILE`: faults apply
13//! only while that file exists. A test lets startup complete and creates the
14//! sentinel to begin the fault, then removes it to restore service.
15//!
16//! Two fault kinds are available. By default a faulted write fails with an
17//! errno (`ENOSPC`, overridable via `KUMO_FAULT_ERRNO`), modelling a full or
18//! failing disk. If `KUMO_FAULT_DELAY_MS` is non-zero a faulted write instead
19//! sleeps that long and then succeeds, modelling slow storage -- which, applied
20//! to flush writes, stalls RocksDB into rejecting foreground writes with
21//! `Incomplete` and exercises the backpressure paths.
22
23#[cfg(target_os = "linux")]
24mod linux {
25    use libc::{c_char, c_int, c_void, iovec, mode_t, off_t, size_t, ssize_t};
26    use std::ffi::CStr;
27    use std::sync::atomic::{AtomicBool, Ordering};
28    use std::sync::OnceLock;
29    use std::time::Duration;
30
31    // Upper bound on the fd numbers `TRACKED` covers. rocksdb keeps its
32    // descriptors well below this bound. Anything above it is treated as
33    // untracked.
34    const MAX_FD: usize = 1 << 16;
35
36    // Whether each fd falls under a faulted prefix, indexed by fd number.
37    //
38    // Only open/openat/close update this table. A duplicate of a tracked fd via
39    // dup/dup2/fcntl(F_DUPFD) would come up untracked and bypass injection
40    // through the duplicate. The file layer of RocksDB does not duplicate fds
41    // today, and the current test is unaffected. A future use of this shim
42    // against code that duplicates fds would need those interposed too.
43    static TRACKED: [AtomicBool; MAX_FD] = [const { AtomicBool::new(false) }; MAX_FD];
44
45    struct Config {
46        prefixes: Vec<Vec<u8>>,
47        suffixes: Vec<Vec<u8>>,
48        sentinel: Option<Vec<u8>>,
49        errno: c_int,
50        delay: Duration,
51    }
52
53    fn split_env(name: &str) -> Vec<Vec<u8>> {
54        std::env::var(name)
55            .ok()
56            .map(|v| {
57                v.split(':')
58                    .filter(|s| !s.is_empty())
59                    .map(|s| s.as_bytes().to_vec())
60                    .collect()
61            })
62            .unwrap_or_default()
63    }
64
65    fn config() -> &'static Config {
66        static CONFIG: OnceLock<Config> = OnceLock::new();
67        CONFIG.get_or_init(|| {
68            let prefixes = split_env("KUMO_FAULT_PATH_PREFIXES");
69            let suffixes = split_env("KUMO_FAULT_PATH_SUFFIXES");
70            let sentinel = std::env::var("KUMO_FAULT_ACTIVE_FILE").ok().map(|mut v| {
71                v.push('\0');
72                v.into_bytes()
73            });
74            let errno = std::env::var("KUMO_FAULT_ERRNO")
75                .ok()
76                .and_then(|v| v.parse().ok())
77                .unwrap_or(libc::ENOSPC);
78            let delay = std::env::var("KUMO_FAULT_DELAY_MS")
79                .ok()
80                .and_then(|v| v.parse().ok())
81                .map(Duration::from_millis)
82                .unwrap_or_default();
83            Config {
84                prefixes,
85                suffixes,
86                sentinel,
87                errno,
88                delay,
89            }
90        })
91    }
92
93    // True while the sentinel file exists, meaning faults should be applied.
94    fn faults_active() -> bool {
95        match &config().sentinel {
96            Some(path) => unsafe { libc::access(path.as_ptr() as *const c_char, libc::F_OK) == 0 },
97            None => false,
98        }
99    }
100
101    fn path_is_faulted(path: *const c_char) -> bool {
102        if path.is_null() {
103            return false;
104        }
105        let bytes = unsafe { CStr::from_ptr(path) }.to_bytes();
106        let cfg = config();
107        let under_prefix = cfg.prefixes.iter().any(|prefix| bytes.starts_with(prefix));
108        let suffix_ok =
109            cfg.suffixes.is_empty() || cfg.suffixes.iter().any(|suffix| bytes.ends_with(suffix));
110        under_prefix && suffix_ok
111    }
112
113    fn set_tracked(fd: c_int, faulted: bool) {
114        if (0..MAX_FD as c_int).contains(&fd) {
115            TRACKED[fd as usize].store(faulted, Ordering::Relaxed);
116        }
117    }
118
119    fn is_tracked(fd: c_int) -> bool {
120        (0..MAX_FD as c_int).contains(&fd) && TRACKED[fd as usize].load(Ordering::Relaxed)
121    }
122
123    // Decides how to fault a write to `fd`. Returns true when the caller should
124    // inject the error now, which it signals by skipping the real write and
125    // returning that error instead. In delay mode a faulted write sleeps here
126    // and returns false, so the caller falls through to the real (now slowed)
127    // write. An unfaulted fd returns false immediately, with no delay.
128    //
129    // `close` clears the tracked bit for its fd. A later `open` that the kernel
130    // assigns the same fd number is tracked from the path it opens, independent
131    // of what that fd number faulted on before.
132    fn should_fail(fd: c_int) -> bool {
133        if !(is_tracked(fd) && faults_active()) {
134            return false;
135        }
136        let delay = config().delay;
137        if delay.is_zero() {
138            return true;
139        }
140        std::thread::sleep(delay);
141        false
142    }
143
144    unsafe fn set_errno() {
145        *libc::__errno_location() = config().errno;
146    }
147
148    // Resolves the libc implementation that our interposed symbol shadows.
149    macro_rules! real {
150        ($cell:ident, $ty:ty, $sym:literal) => {{
151            static $cell: OnceLock<$ty> = OnceLock::new();
152            *$cell.get_or_init(|| unsafe {
153                let ptr = libc::dlsym(
154                    libc::RTLD_NEXT,
155                    concat!($sym, "\0").as_ptr() as *const c_char,
156                );
157                assert!(!ptr.is_null(), concat!("dlsym failed for ", $sym));
158                std::mem::transmute::<*mut c_void, $ty>(ptr)
159            })
160        }};
161    }
162
163    type OpenFn = unsafe extern "C" fn(*const c_char, c_int, mode_t) -> c_int;
164    type OpenatFn = unsafe extern "C" fn(c_int, *const c_char, c_int, mode_t) -> c_int;
165    type WriteFn = unsafe extern "C" fn(c_int, *const c_void, size_t) -> ssize_t;
166    type PwriteFn = unsafe extern "C" fn(c_int, *const c_void, size_t, off_t) -> ssize_t;
167    type WritevFn = unsafe extern "C" fn(c_int, *const iovec, c_int) -> ssize_t;
168    type PwritevFn = unsafe extern "C" fn(c_int, *const iovec, c_int, off_t) -> ssize_t;
169    type FsyncFn = unsafe extern "C" fn(c_int) -> c_int;
170    type FallocateFn = unsafe extern "C" fn(c_int, c_int, off_t, off_t) -> c_int;
171    type FtruncateFn = unsafe extern "C" fn(c_int, off_t) -> c_int;
172    type CloseFn = unsafe extern "C" fn(c_int) -> c_int;
173
174    // The `mode` argument of `open`/`openat` is variadic in C and only read
175    // when `O_CREAT`/`O_TMPFILE` is set. Declaring it as a fixed parameter
176    // and forwarding it is sound because the real call ignores it otherwise.
177    unsafe extern "C" fn open_impl(fd: c_int, path: *const c_char) {
178        set_tracked(fd, fd >= 0 && path_is_faulted(path));
179    }
180
181    #[no_mangle]
182    pub unsafe extern "C" fn open(path: *const c_char, flags: c_int, mode: mode_t) -> c_int {
183        let fd = real!(REAL_OPEN, OpenFn, "open")(path, flags, mode);
184        open_impl(fd, path);
185        fd
186    }
187
188    #[no_mangle]
189    pub unsafe extern "C" fn open64(path: *const c_char, flags: c_int, mode: mode_t) -> c_int {
190        let fd = real!(REAL_OPEN64, OpenFn, "open64")(path, flags, mode);
191        open_impl(fd, path);
192        fd
193    }
194
195    #[no_mangle]
196    pub unsafe extern "C" fn openat(
197        dirfd: c_int,
198        path: *const c_char,
199        flags: c_int,
200        mode: mode_t,
201    ) -> c_int {
202        let fd = real!(REAL_OPENAT, OpenatFn, "openat")(dirfd, path, flags, mode);
203        open_impl(fd, path);
204        fd
205    }
206
207    #[no_mangle]
208    pub unsafe extern "C" fn openat64(
209        dirfd: c_int,
210        path: *const c_char,
211        flags: c_int,
212        mode: mode_t,
213    ) -> c_int {
214        let fd = real!(REAL_OPENAT64, OpenatFn, "openat64")(dirfd, path, flags, mode);
215        open_impl(fd, path);
216        fd
217    }
218
219    #[no_mangle]
220    pub unsafe extern "C" fn write(fd: c_int, buf: *const c_void, count: size_t) -> ssize_t {
221        if should_fail(fd) {
222            set_errno();
223            return -1;
224        }
225        real!(REAL_WRITE, WriteFn, "write")(fd, buf, count)
226    }
227
228    #[no_mangle]
229    pub unsafe extern "C" fn pwrite(
230        fd: c_int,
231        buf: *const c_void,
232        count: size_t,
233        offset: off_t,
234    ) -> ssize_t {
235        if should_fail(fd) {
236            set_errno();
237            return -1;
238        }
239        real!(REAL_PWRITE, PwriteFn, "pwrite")(fd, buf, count, offset)
240    }
241
242    #[no_mangle]
243    pub unsafe extern "C" fn writev(fd: c_int, iov: *const iovec, iovcnt: c_int) -> ssize_t {
244        if should_fail(fd) {
245            set_errno();
246            return -1;
247        }
248        real!(REAL_WRITEV, WritevFn, "writev")(fd, iov, iovcnt)
249    }
250
251    #[no_mangle]
252    pub unsafe extern "C" fn pwritev(
253        fd: c_int,
254        iov: *const iovec,
255        iovcnt: c_int,
256        offset: off_t,
257    ) -> ssize_t {
258        if should_fail(fd) {
259            set_errno();
260            return -1;
261        }
262        real!(REAL_PWRITEV, PwritevFn, "pwritev")(fd, iov, iovcnt, offset)
263    }
264
265    #[no_mangle]
266    pub unsafe extern "C" fn fsync(fd: c_int) -> c_int {
267        if should_fail(fd) {
268            set_errno();
269            return -1;
270        }
271        real!(REAL_FSYNC, FsyncFn, "fsync")(fd)
272    }
273
274    #[no_mangle]
275    pub unsafe extern "C" fn fdatasync(fd: c_int) -> c_int {
276        if should_fail(fd) {
277            set_errno();
278            return -1;
279        }
280        real!(REAL_FDATASYNC, FsyncFn, "fdatasync")(fd)
281    }
282
283    #[no_mangle]
284    pub unsafe extern "C" fn fallocate(fd: c_int, mode: c_int, offset: off_t, len: off_t) -> c_int {
285        if should_fail(fd) {
286            set_errno();
287            return -1;
288        }
289        real!(REAL_FALLOCATE, FallocateFn, "fallocate")(fd, mode, offset, len)
290    }
291
292    #[no_mangle]
293    pub unsafe extern "C" fn ftruncate(fd: c_int, length: off_t) -> c_int {
294        if should_fail(fd) {
295            set_errno();
296            return -1;
297        }
298        real!(REAL_FTRUNCATE, FtruncateFn, "ftruncate")(fd, length)
299    }
300
301    #[no_mangle]
302    pub unsafe extern "C" fn close(fd: c_int) -> c_int {
303        set_tracked(fd, false);
304        real!(REAL_CLOSE, CloseFn, "close")(fd)
305    }
306}
fault_inject_preload/lib.rs

fault_inject_preload/
lib.rs
