proptest/test_runner/failure_persistence/
file.rs

1//-
2// Copyright 2017, 2018, 2019 The proptest developers
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use core::any::Any;
11use core::fmt::Debug;
12use std::borrow::{Cow, ToOwned};
13use std::boxed::Box;
14use std::env;
15use std::fs;
16use std::io::{self, BufRead, Write};
17use std::path::{Path, PathBuf};
18use std::string::{String, ToString};
19use std::sync::RwLock;
20use std::vec::Vec;
21
22use self::FileFailurePersistence::*;
23use crate::test_runner::failure_persistence::{
24    FailurePersistence, PersistedSeed,
25};
26
27/// Describes how failing test cases are persisted.
28///
29/// Note that file names in this enum are `&str` rather than `&Path` since
30/// constant functions are not yet in Rust stable as of 2017-12-16.
31///
32/// In all cases, if a derived path references a directory which does not yet
33/// exist, proptest will attempt to create all necessary parent directories.
34#[derive(Clone, Copy, Debug, PartialEq)]
35pub enum FileFailurePersistence {
36    /// Completely disables persistence of failing test cases.
37    ///
38    /// This is semantically equivalent to `Direct("/dev/null")` on Unix and
39    /// `Direct("NUL")` on Windows (though it is internally handled by simply
40    /// not doing any I/O).
41    Off,
42    /// The path given to `TestRunner::set_source_file()` is parsed. The path
43    /// is traversed up the directory tree until a directory containing a file
44    /// named `lib.rs` or `main.rs` is found. A sibling to that directory with
45    /// the name given by the string in this configuration is created, and a
46    /// file with the same name and path relative to the source directory, but
47    /// with the extension changed to `.txt`, is used.
48    ///
49    /// For example, given a source path of
50    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
51    /// `SourceParallel("proptest-regressions")` (the default), assuming the
52    /// `src` directory has a `lib.rs` or `main.rs`, the resulting file would
53    /// be `/home/jsmith/code/project/proptest-regressions/foo/bar.txt`.
54    ///
55    /// If no `lib.rs` or `main.rs` can be found, a warning is printed and this
56    /// behaves like `WithSource`.
57    ///
58    /// If no source file has been configured, a warning is printed and this
59    /// behaves like `Off`.
60    SourceParallel(&'static str),
61    /// The path given to `TestRunner::set_source_file()` is parsed. The
62    /// extension of the path is changed to the string given in this
63    /// configuration, and that filename is used.
64    ///
65    /// For example, given a source path of
66    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
67    /// `WithSource("regressions")`, the resulting path would be
68    /// `/home/jsmith/code/project/src/foo/bar.regressions`.
69    WithSource(&'static str),
70    /// The string given in this option is directly used as a file path without
71    /// any further processing.
72    Direct(&'static str),
73    #[doc(hidden)]
74    #[allow(missing_docs)]
75    _NonExhaustive,
76}
77
78impl Default for FileFailurePersistence {
79    fn default() -> Self {
80        SourceParallel("proptest-regressions")
81    }
82}
83
84impl FailurePersistence for FileFailurePersistence {
85    fn load_persisted_failures2(
86        &self,
87        source_file: Option<&'static str>,
88    ) -> Vec<PersistedSeed> {
89        let p = self.resolve(
90            source_file
91                .and_then(|s| absolutize_source_file(Path::new(s)))
92                .as_ref()
93                .map(|cow| &**cow),
94        );
95
96        let path: Option<&PathBuf> = p.as_ref();
97        let result: io::Result<Vec<PersistedSeed>> = path.map_or_else(
98            || Ok(vec![]),
99            |path| {
100                // .ok() instead of .unwrap() so we don't propagate panics here
101                let _lock = PERSISTENCE_LOCK.read().ok();
102                io::BufReader::new(fs::File::open(path)?)
103                    .lines()
104                    .enumerate()
105                    .filter_map(|(lineno, line)| match line {
106                        Err(err) => Some(Err(err)),
107                        Ok(line) => parse_seed_line(line, path, lineno).map(Ok),
108                    })
109                    .collect()
110            },
111        );
112
113        unwrap_or!(result, err => {
114            if io::ErrorKind::NotFound != err.kind() {
115                eprintln!(
116                    "proptest: failed to open {}: {}",
117                    &path.map(|x| &**x)
118                        .unwrap_or_else(|| Path::new("??"))
119                        .display(),
120                    err
121                );
122            }
123            vec![]
124        })
125    }
126
127    fn save_persisted_failure2(
128        &mut self,
129        source_file: Option<&'static str>,
130        seed: PersistedSeed,
131        shrunken_value: &dyn Debug,
132    ) {
133        let path = self.resolve(source_file.map(Path::new));
134        if let Some(path) = path {
135            // .ok() instead of .unwrap() so we don't propagate panics here
136            let _lock = PERSISTENCE_LOCK.write().ok();
137            let is_new = !path.is_file();
138
139            let mut to_write = Vec::<u8>::new();
140            if is_new {
141                write_header(&mut to_write)
142                    .expect("proptest: couldn't write header.");
143            }
144
145            write_seed_line(&mut to_write, &seed, shrunken_value)
146                .expect("proptest: couldn't write seed line.");
147
148            if let Err(e) = write_seed_data_to_file(&path, &to_write) {
149                eprintln!(
150                    "proptest: failed to append to {}: {}",
151                    path.display(),
152                    e
153                );
154            } else if is_new {
155                eprintln!(
156                    "proptest: Saving this and future failures in {}\n\
157                     proptest: If this test was run on a CI system, you may \
158                     wish to add the following line to your copy of the file.{}\n\
159                     {}",
160                    path.display(),
161                    if is_new { " (You may need to create it.)" } else { "" },
162                    seed);
163            }
164        }
165    }
166
167    fn box_clone(&self) -> Box<dyn FailurePersistence> {
168        Box::new(*self)
169    }
170
171    fn eq(&self, other: &dyn FailurePersistence) -> bool {
172        other
173            .as_any()
174            .downcast_ref::<Self>()
175            .map_or(false, |x| x == self)
176    }
177
178    fn as_any(&self) -> &dyn Any {
179        self
180    }
181}
182
183/// Ensure that the source file to use for resolving the location of the persisted
184/// failing cases file is absolute.
185///
186/// The source location can only be used if it is absolute. If `source` is
187/// not an absolute path, an attempt will be made to determine the absolute
188/// path based on the current working directory and its parents. If no
189/// absolute path can be determined, a warning will be printed and proptest
190/// will continue as if this function had never been called.
191///
192/// See [`FileFailurePersistence`](enum.FileFailurePersistence.html) for details on
193/// how this value is used once it is made absolute.
194///
195/// This is normally called automatically by the `proptest!` macro, which
196/// passes `file!()`.
197///
198fn absolutize_source_file<'a>(source: &'a Path) -> Option<Cow<'a, Path>> {
199    absolutize_source_file_with_cwd(env::current_dir, source)
200}
201
202fn absolutize_source_file_with_cwd<'a>(
203    getcwd: impl FnOnce() -> io::Result<PathBuf>,
204    source: &'a Path,
205) -> Option<Cow<'a, Path>> {
206    if source.is_absolute() {
207        // On Unix, `file!()` is absolute. In these cases, we can use
208        // that path directly.
209        Some(Cow::Borrowed(source))
210    } else {
211        // On Windows, `file!()` is relative to the crate root, but the
212        // test is not generally run with the crate root as the working
213        // directory, so the path is not directly usable. However, the
214        // working directory is almost always a subdirectory of the crate
215        // root, so pop directories off until pushing the source onto the
216        // directory results in a path that refers to an existing file.
217        // Once we find such a path, we can use that.
218        //
219        // If we can't figure out an absolute path, print a warning and act
220        // as if no source had been given.
221        match getcwd() {
222            Ok(mut cwd) => loop {
223                let joined = cwd.join(source);
224                if joined.is_file() {
225                    break Some(Cow::Owned(joined));
226                }
227
228                if !cwd.pop() {
229                    eprintln!(
230                        "proptest: Failed to find absolute path of \
231                         source file '{:?}'. Ensure the test is \
232                         being run from somewhere within the crate \
233                         directory hierarchy.",
234                        source
235                    );
236                    break None;
237                }
238            },
239
240            Err(e) => {
241                eprintln!(
242                    "proptest: Failed to determine current \
243                     directory, so the relative source path \
244                     '{:?}' cannot be resolved: {}",
245                    source, e
246                );
247                None
248            }
249        }
250    }
251}
252
253fn parse_seed_line(
254    mut line: String,
255    path: &Path,
256    lineno: usize,
257) -> Option<PersistedSeed> {
258    // Remove anything after and including '#':
259    if let Some(comment_start) = line.find('#') {
260        line.truncate(comment_start);
261    }
262
263    if line.len() > 0 {
264        let ret = line.parse::<PersistedSeed>().ok();
265        if !ret.is_some() {
266            eprintln!(
267                "proptest: {}:{}: unparsable line, ignoring",
268                path.display(),
269                lineno + 1
270            );
271        }
272        return ret;
273    }
274
275    None
276}
277
278fn write_seed_line(
279    buf: &mut Vec<u8>,
280    seed: &PersistedSeed,
281    shrunken_value: &dyn Debug,
282) -> io::Result<()> {
283    // Write the seed itself
284    write!(buf, "{}", seed.to_string())?;
285
286    // Write out comment:
287    let debug_start = buf.len();
288    write!(buf, " # shrinks to {:?}", shrunken_value)?;
289
290    // Ensure there are no newlines in the debug output
291    for byte in &mut buf[debug_start..] {
292        if b'\n' == *byte || b'\r' == *byte {
293            *byte = b' ';
294        }
295    }
296
297    buf.push(b'\n');
298
299    Ok(())
300}
301
302fn write_header(buf: &mut Vec<u8>) -> io::Result<()> {
303    writeln!(
304        buf,
305        "\
306# Seeds for failure cases proptest has generated in the past. It is
307# automatically read and these particular cases re-run before any
308# novel cases are generated.
309#
310# It is recommended to check this file in to source control so that
311# everyone who runs the test benefits from these saved cases."
312    )
313}
314
315fn write_seed_data_to_file(dst: &Path, data: &[u8]) -> io::Result<()> {
316    if let Some(parent) = dst.parent() {
317        fs::create_dir_all(parent)?;
318    }
319
320    let mut options = fs::OpenOptions::new();
321    options.append(true).create(true);
322    let mut out = options.open(dst)?;
323    out.write_all(data)?;
324
325    Ok(())
326}
327
328impl FileFailurePersistence {
329    /// Given the nominal source path, determine the location of the failure
330    /// persistence file, if any.
331    pub(super) fn resolve(&self, source: Option<&Path>) -> Option<PathBuf> {
332        let source = source.and_then(absolutize_source_file);
333
334        match *self {
335            Off => None,
336
337            SourceParallel(sibling) => match source {
338                Some(source_path) => {
339                    let mut dir = Cow::into_owned(source_path.clone());
340                    let mut found = false;
341                    while dir.pop() {
342                        if dir.join("lib.rs").is_file()
343                            || dir.join("main.rs").is_file()
344                        {
345                            found = true;
346                            break;
347                        }
348                    }
349
350                    if !found {
351                        eprintln!(
352                            "proptest: FileFailurePersistence::SourceParallel set, \
353                             but failed to find lib.rs or main.rs"
354                        );
355                        WithSource(sibling).resolve(Some(&*source_path))
356                    } else {
357                        let suffix = source_path
358                            .strip_prefix(&dir)
359                            .expect("parent of source is not a prefix of it?")
360                            .to_owned();
361                        let mut result = dir;
362                        // If we've somehow reached the root, or someone gave
363                        // us a relative path that we've exhausted, just accept
364                        // creating a subdirectory instead.
365                        let _ = result.pop();
366                        result.push(sibling);
367                        result.push(&suffix);
368                        result.set_extension("txt");
369                        Some(result)
370                    }
371                }
372                None => {
373                    eprintln!(
374                        "proptest: FileFailurePersistence::SourceParallel set, \
375                         but no source file known"
376                    );
377                    None
378                }
379            },
380
381            WithSource(extension) => match source {
382                Some(source_path) => {
383                    let mut result = Cow::into_owned(source_path);
384                    result.set_extension(extension);
385                    Some(result)
386                }
387
388                None => {
389                    eprintln!(
390                        "proptest: FileFailurePersistence::WithSource set, \
391                         but no source file known"
392                    );
393                    None
394                }
395            },
396
397            Direct(path) => Some(Path::new(path).to_owned()),
398
399            _NonExhaustive => {
400                panic!("FailurePersistence set to _NonExhaustive")
401            }
402        }
403    }
404}
405
406lazy_static! {
407    /// Used to guard access to the persistence file(s) so that a single
408    /// process will not step on its own toes.
409    ///
410    /// We don't have much protecting us should two separate process try to
411    /// write to the same file at once (depending on how atomic append mode is
412    /// on the OS), but this should be extremely rare.
413    static ref PERSISTENCE_LOCK: RwLock<()> = RwLock::new(());
414}
415
416#[cfg(test)]
417mod tests {
418    use super::*;
419
420    struct TestPaths {
421        crate_root: &'static Path,
422        src_file: PathBuf,
423        subdir_file: PathBuf,
424        misplaced_file: PathBuf,
425    }
426
427    lazy_static! {
428        static ref TEST_PATHS: TestPaths = {
429            let crate_root = Path::new(env!("CARGO_MANIFEST_DIR"));
430            let lib_root = crate_root.join("src");
431            let src_subdir = lib_root.join("strategy");
432            let src_file = lib_root.join("foo.rs");
433            let subdir_file = src_subdir.join("foo.rs");
434            let misplaced_file = crate_root.join("foo.rs");
435            TestPaths {
436                crate_root,
437                src_file,
438                subdir_file,
439                misplaced_file,
440            }
441        };
442    }
443
444    #[test]
445    fn persistence_file_location_resolved_correctly() {
446        // If off, there is never a file
447        assert_eq!(None, Off.resolve(None));
448        assert_eq!(None, Off.resolve(Some(&TEST_PATHS.subdir_file)));
449
450        // For direct, we don't care about the source file, and instead always
451        // use whatever is in the config.
452        assert_eq!(
453            Some(Path::new("bar.txt").to_owned()),
454            Direct("bar.txt").resolve(None)
455        );
456        assert_eq!(
457            Some(Path::new("bar.txt").to_owned()),
458            Direct("bar.txt").resolve(Some(&TEST_PATHS.subdir_file))
459        );
460
461        // For WithSource, only the extension changes, but we get nothing if no
462        // source file was configured.
463        // Accounting for the way absolute paths work on Windows would be more
464        // complex, so for now don't test that case.
465        #[cfg(unix)]
466        fn absolute_path_case() {
467            assert_eq!(
468                Some(Path::new("/foo/bar.ext").to_owned()),
469                WithSource("ext").resolve(Some(Path::new("/foo/bar.rs")))
470            );
471        }
472        #[cfg(not(unix))]
473        fn absolute_path_case() {}
474        absolute_path_case();
475        assert_eq!(None, WithSource("ext").resolve(None));
476
477        // For SourceParallel, we make a sibling directory tree and change the
478        // extensions to .txt ...
479        assert_eq!(
480            Some(TEST_PATHS.crate_root.join("sib").join("foo.txt")),
481            SourceParallel("sib").resolve(Some(&TEST_PATHS.src_file))
482        );
483        assert_eq!(
484            Some(
485                TEST_PATHS
486                    .crate_root
487                    .join("sib")
488                    .join("strategy")
489                    .join("foo.txt")
490            ),
491            SourceParallel("sib").resolve(Some(&TEST_PATHS.subdir_file))
492        );
493        // ... but if we can't find lib.rs / main.rs, give up and set the
494        // extension instead ...
495        assert_eq!(
496            Some(TEST_PATHS.crate_root.join("foo.sib")),
497            SourceParallel("sib").resolve(Some(&TEST_PATHS.misplaced_file))
498        );
499        // ... and if no source is configured, we do nothing
500        assert_eq!(None, SourceParallel("ext").resolve(None));
501    }
502
503    #[test]
504    fn relative_source_files_absolutified() {
505        const TEST_RUNNER_PATH: &[&str] = &["src", "test_runner", "mod.rs"];
506        lazy_static! {
507            static ref TEST_RUNNER_RELATIVE: PathBuf =
508                TEST_RUNNER_PATH.iter().collect();
509        }
510        const CARGO_DIR: &str = env!("CARGO_MANIFEST_DIR");
511
512        let expected = ::std::iter::once(CARGO_DIR)
513            .chain(TEST_RUNNER_PATH.iter().map(|s| *s))
514            .collect::<PathBuf>();
515
516        // Running from crate root
517        assert_eq!(
518            &*expected,
519            absolutize_source_file_with_cwd(
520                || Ok(Path::new(CARGO_DIR).to_owned()),
521                &TEST_RUNNER_RELATIVE
522            )
523            .unwrap()
524        );
525
526        // Running from test subdirectory
527        assert_eq!(
528            &*expected,
529            absolutize_source_file_with_cwd(
530                || Ok(Path::new(CARGO_DIR).join("target")),
531                &TEST_RUNNER_RELATIVE
532            )
533            .unwrap()
534        );
535    }
536}
proptest/test_runner/failure_persistence/file.rs

proptest/test_runner/failure_persistence/
file.rs
