ruzstd/tests/
dict_test.rs

1#[test]
2fn test_dict_parsing() {
3    use crate::decoding::dictionary::Dictionary;
4    use alloc::vec;
5    let mut raw = vec![0u8; 8];
6
7    // correct magic num
8    raw[0] = 0x37;
9    raw[1] = 0xA4;
10    raw[2] = 0x30;
11    raw[3] = 0xEC;
12
13    //dict-id
14    let dict_id = 0x47232101;
15    raw[4] = 0x01;
16    raw[5] = 0x21;
17    raw[6] = 0x23;
18    raw[7] = 0x47;
19
20    // tables copied from ./dict_tests/dictionary
21    let raw_tables = &[
22        54, 16, 192, 155, 4, 0, 207, 59, 239, 121, 158, 116, 220, 93, 114, 229, 110, 41, 249, 95,
23        165, 255, 83, 202, 254, 68, 74, 159, 63, 161, 100, 151, 137, 21, 184, 183, 189, 100, 235,
24        209, 251, 174, 91, 75, 91, 185, 19, 39, 75, 146, 98, 177, 249, 14, 4, 35, 0, 0, 0, 40, 40,
25        20, 10, 12, 204, 37, 196, 1, 173, 122, 0, 4, 0, 128, 1, 2, 2, 25, 32, 27, 27, 22, 24, 26,
26        18, 12, 12, 15, 16, 11, 69, 37, 225, 48, 20, 12, 6, 2, 161, 80, 40, 20, 44, 137, 145, 204,
27        46, 0, 0, 0, 0, 0, 116, 253, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
28    ];
29    raw.extend(&raw_tables[..]);
30
31    //offset history 3,10,0x00ABCDEF
32    raw.extend(vec![3, 0, 0, 0]);
33    raw.extend(vec![10, 0, 0, 0]);
34    raw.extend(vec![0xEF, 0xCD, 0xAB, 0]);
35
36    //just some random bytes
37    let raw_content = vec![
38        1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 123, 3, 234, 23, 234, 34, 23, 234, 34, 34, 234, 234,
39    ];
40    raw.extend(&raw_content);
41
42    let dict = Dictionary::decode_dict(&raw).unwrap();
43
44    if dict.id != dict_id {
45        panic!(
46            "Dict-id did not get parsed correctly. Is: {}, Should be: {}",
47            dict.id, dict_id
48        );
49    }
50
51    if !dict.dict_content.eq(&raw_content) {
52        panic!(
53            "dict content did not get parsed correctly. Is: {:?}, Should be: {:?}",
54            dict.dict_content, raw_content
55        );
56    }
57
58    if !dict.offset_hist.eq(&[3, 10, 0x00ABCDEF]) {
59        panic!(
60            "offset history did not get parsed correctly. Is: {:?}, Should be: {:?}",
61            dict.offset_hist,
62            [3, 10, 0x00ABCDEF]
63        );
64    }
65
66    // test magic num checking
67    raw[0] = 1;
68    raw[1] = 1;
69    raw[2] = 1;
70    raw[3] = 1;
71    match Dictionary::decode_dict(&raw) {
72        Ok(_) => panic!("The dict got decoded but the magic num was incorrect!"),
73        Err(_) => { /* This is what should happen*/ }
74    }
75}
76
77#[test]
78fn test_dict_decoding() {
79    extern crate std;
80    use crate::frame_decoder;
81    use alloc::borrow::ToOwned;
82    use alloc::string::{String, ToString};
83    use alloc::vec::Vec;
84    use std::fs;
85    use std::io::Read;
86    use std::println;
87
88    let mut success_counter = 0;
89    let mut fail_counter_diff = 0;
90    let mut fail_counter_size = 0;
91    let mut fail_counter_bytes_read = 0;
92    let mut total_counter = 0;
93    let mut failed: Vec<String> = Vec::new();
94
95    let mut speeds = Vec::new();
96    let mut speeds_read = Vec::new();
97
98    let mut files: Vec<_> = fs::read_dir("./dict_tests/files").unwrap().collect();
99    let dict = fs::File::open("./dict_tests/dictionary").unwrap();
100    let dict: Vec<u8> = dict.bytes().map(|x| x.unwrap()).collect();
101
102    files.sort_by_key(|x| match x {
103        Err(_) => "".to_owned(),
104        Ok(entry) => entry.path().to_str().unwrap().to_owned(),
105    });
106
107    let mut frame_dec = frame_decoder::FrameDecoder::new();
108    let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap();
109    frame_dec.add_dict(dict).unwrap();
110
111    for file in files {
112        let f = file.unwrap();
113        let metadata = f.metadata().unwrap();
114        let file_size = metadata.len();
115
116        let p = String::from(f.path().to_str().unwrap());
117        if !p.ends_with(".zst") {
118            continue;
119        }
120        println!("Trying file: {}", p);
121
122        let mut content = fs::File::open(f.path()).unwrap();
123
124        frame_dec.reset(&mut content).unwrap();
125
126        let start_time = std::time::Instant::now();
127        /////DECODING
128        frame_dec
129            .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
130            .unwrap();
131        let result = frame_dec.collect().unwrap();
132        let end_time = start_time.elapsed();
133
134        match frame_dec.get_checksum_from_data() {
135            Some(chksum) => {
136                #[cfg(feature = "hash")]
137                if frame_dec.get_calculated_checksum().unwrap() != chksum {
138                    println!(
139                        "Checksum did not match! From data: {}, calculated while decoding: {}\n",
140                        chksum,
141                        frame_dec.get_calculated_checksum().unwrap()
142                    );
143                } else {
144                    println!("Checksums are ok!\n");
145                }
146                #[cfg(not(feature = "hash"))]
147                println!(
148                    "Checksum feature not enabled, skipping. From data: {}\n",
149                    chksum
150                );
151            }
152            None => println!("No checksums to test\n"),
153        }
154
155        let mut original_p = p.clone();
156        original_p.truncate(original_p.len() - 4);
157        let original_f = fs::File::open(original_p).unwrap();
158        let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
159
160        println!("Results for file: {}", p.clone());
161        let mut success = true;
162
163        if original.len() != result.len() {
164            println!(
165                "Result has wrong length: {}, should be: {}",
166                result.len(),
167                original.len()
168            );
169            success = false;
170            fail_counter_size += 1;
171        }
172
173        if frame_dec.bytes_read_from_source() != file_size {
174            println!(
175                "Framedecoder counted wrong amount of bytes: {}, should be: {}",
176                frame_dec.bytes_read_from_source(),
177                file_size
178            );
179            success = false;
180            fail_counter_bytes_read += 1;
181        }
182
183        let mut counter = 0;
184        let min = if original.len() < result.len() {
185            original.len()
186        } else {
187            result.len()
188        };
189        for idx in 0..min {
190            if original[idx] != result[idx] {
191                counter += 1;
192                //println!(
193                //    "Original {} not equal to result {} at byte: {}",
194                //    original[idx], result[idx], idx,
195                //);
196            }
197        }
198
199        if counter > 0 {
200            println!("Result differs in at least {} bytes from original", counter);
201            success = false;
202            fail_counter_diff += 1;
203        }
204
205        if success {
206            success_counter += 1;
207        } else {
208            failed.push(p.clone().to_string());
209        }
210        total_counter += 1;
211
212        let dur = end_time.as_micros() as usize;
213        let speed = result.len() / if dur == 0 { 1 } else { dur };
214        let speed_read = file_size as usize / if dur == 0 { 1 } else { dur };
215        println!("SPEED: {}", speed);
216        println!("SPEED_read: {}", speed_read);
217        speeds.push(speed);
218        speeds_read.push(speed_read);
219    }
220
221    println!("###################");
222    println!("Summary:");
223    println!("###################");
224    println!(
225        "Total: {}, Success: {}, WrongSize: {}, WrongBytecount: {}, Diffs: {}",
226        total_counter,
227        success_counter,
228        fail_counter_size,
229        fail_counter_bytes_read,
230        fail_counter_diff
231    );
232    println!("Failed files: ");
233    for f in &failed {
234        println!("{}", f);
235    }
236
237    let speed_len = speeds.len();
238    let sum_speed: usize = speeds.into_iter().sum();
239    let avg_speed = sum_speed / speed_len;
240    let avg_speed_bps = avg_speed * 1_000_000;
241    if avg_speed_bps < 1000 {
242        println!("Average speed: {} B/s", avg_speed_bps);
243    } else if avg_speed_bps < 1_000_000 {
244        println!("Average speed: {} KB/s", avg_speed_bps / 1000);
245    } else {
246        println!("Average speed: {} MB/s", avg_speed_bps / 1_000_000);
247    }
248
249    let speed_read_len = speeds_read.len();
250    let sum_speed_read: usize = speeds_read.into_iter().sum();
251    let avg_speed_read = sum_speed_read / speed_read_len;
252    let avg_speed_read_bps = avg_speed_read * 1_000_000;
253    if avg_speed_read_bps < 1000 {
254        println!("Average speed reading: {} B/s", avg_speed_read_bps);
255    } else if avg_speed_bps < 1_000_000 {
256        println!("Average speed reading: {} KB/s", avg_speed_read_bps / 1000);
257    } else {
258        println!(
259            "Average speed reading: {} MB/s",
260            avg_speed_read_bps / 1_000_000
261        );
262    }
263
264    assert!(failed.is_empty());
265}
ruzstd/tests/dict_test.rs

ruzstd/tests/
dict_test.rs
