Add Multithreading

schemsearch-cli/Cargo.toml

@@ -14,6 +14,7 @@ schemsearch-sql = { path = "../schemsearch-sql", optional = true }
 clap = { version = "4.1.8", features = ["cargo"] }
 futures = { version = "0.3", optional = true }
 sqlx = { version = "0.6", features = [ "runtime-async-std-native-tls" , "mysql" ], optional = true }
+rayon = "1.7.0"
 
 [features]
-sql = [ "dep:schemsearch-sql", "dep:futures", "dep:sqlx" ]
+sql = ["dep:schemsearch-sql", "dep:futures", "dep:sqlx"]

schemsearch-cli/src/main.rs

@@ -18,15 +18,18 @@
 mod types;
 
 use std::fs::File;
-use std::io::{BufWriter, StdoutLock, Write};
+use std::io;
-use clap::{command, Arg, ArgAction, Command, ValueHint};
+use std::io::{BufWriter, Write};
+use clap::{command, Arg, ArgAction, ValueHint};
 use schemsearch_files::Schematic;
-use std::path::Path;
+use std::path::PathBuf;
 use clap::error::ErrorKind;
 use schemsearch_lib::{search, SearchBehavior};
 use crate::types::{PathSchematicSupplier, SchematicSupplierType};
 #[cfg(feature = "sql")]
 use futures::executor::block_on;
+use rayon::prelude::*;
+use rayon::ThreadPoolBuilder;
 use schemsearch_sql::filter::SchematicFilter;
 #[cfg(feature = "sql")]
 use schemsearch_sql::load_all_schematics;
@@ -110,6 +113,15 @@ fn main() {
                 .default_value("0.9")
                 .value_parser(|s: &str| s.parse::<f32>().map_err(|e| e.to_string())),
         )
+        .arg(
+            Arg::new("threads")
+                .help("The number of threads to use [0 = Available Threads]")
+                .short('T')
+                .long("threads")
+                .action(ArgAction::Set)
+                .default_value("0")
+                .value_parser(|s: &str| s.parse::<usize>().map_err(|e| e.to_string())),
+        )
         .about("Searches for a pattern in a schematic")
         .bin_name("schemsearch");
 
@@ -155,16 +167,35 @@ fn main() {
         threshold: *matches.get_one::<f32>("threshold").expect("Couldn't get threshold"),
     };
 
-    let pattern = match Schematic::load(Path::new(matches.get_one::<String>("pattern").unwrap())) {
+    let pattern = match Schematic::load(&PathBuf::from(matches.get_one::<String>("pattern").unwrap())) {
         Ok(x) => x,
         Err(e) => {
             cmd.error(ErrorKind::Io, format!("Error while loading Pattern: {}", e.to_string())).exit();
         }
     };
 
-    let mut schematics: Vec<SchematicSupplierType> = match matches.get_many::<String>("schematic") {
+    let mut schematics: Vec<SchematicSupplierType> = Vec::new();
-        None => vec![],
+    match matches.get_many::<String>("schematic") {
-        Some(x) => x.map(|x| SchematicSupplierType::PATH(Box::new(PathSchematicSupplier{path: Path::new(x) }))).collect()
+        None => {},
+        Some(x) => {
+            let paths = x.map(|x| PathBuf::from(x));
+            for path in paths {
+                if path.is_dir() {
+                    path.read_dir()
+                        .expect("Couldn't read directory")
+                        .filter_map(|x| x.ok())
+                        .filter(|x| x.path().is_file())
+                        .filter(|x| x.path().extension().unwrap().to_str().unwrap() == "schem")
+                        .for_each(|x| {
+                            schematics.push(SchematicSupplierType::PATH(Box::new(PathSchematicSupplier {
+                                path: x.path(),
+                            })))
+                        });
+                } else if path.extension().unwrap().to_str().unwrap() == "schem" {
+                    schematics.push(SchematicSupplierType::PATH(Box::new(PathSchematicSupplier { path })));
+                }
+            }
+        }
     };
 
     #[cfg(feature = "sql")]
@@ -201,10 +232,6 @@ fn main() {
             _ => {}
         }
     };
-
-    let stdout = std::io::stdout();
-    let mut lock = stdout.lock();
-
     let file: Option<File>;
     let mut file_out: Option<BufWriter<File>> = None;
 
@@ -224,81 +251,82 @@ fn main() {
         };
         file_out = Some(BufWriter::new(file.unwrap()));
     }
+    ThreadPoolBuilder::new().num_threads(*matches.get_one::<usize>("threads").expect("Could not get threads")).build_global().unwrap();
 
-    for schem in schematics {
+    let matches: Vec<Result> = schematics.par_iter().map(|schem| {
         match schem {
             SchematicSupplierType::PATH(schem) => {
-                let path = schem.path;
+                let schematic = match load_schem(&schem.path) {
-                if path.is_dir() {
+                    Some(x) => x,
-                    match path.read_dir() {
+                    None => return Result {
-                        Ok(x) => {
+                        name: schem.get_name(),
-                            for path in x {
+                        matches: vec![]
-                                match path {
-                                    Ok(x) => {
-                                        if x.path().extension().unwrap_or_default() == "schem" {
-                                            let schematic = load_schem(&mut cmd, &x.path());
-                                            search_schempath(search_behavior, &pattern, &mut output_std, &mut output_std_csv, &mut output_file_csv, &mut output_file, &mut lock, &mut file_out, schematic, x.path().file_name().unwrap().to_str().unwrap().to_string());
-                                        }
-                                    }
-                                    Err(e) => cmd.error(ErrorKind::Io, format!("Error while reading schem: {}", e.to_string())).exit()
-                                }
-                            }
-                        }
-                        Err(e) => cmd.error(ErrorKind::Io, format!("Expected to be a dir: {}", e.to_string())).exit()
                     }
-                } else {
+                };
-                    let schematic = load_schem(&mut cmd, &path);
+                Result {
-                    search_schempath(search_behavior, &pattern, &mut output_std, &mut output_std_csv, &mut output_file_csv, &mut output_file, &mut lock, &mut file_out, schematic, schem.get_name());
+                    name: schem.get_name(),
+                    matches: search(schematic, &pattern, search_behavior)
                 }
             }
             #[cfg(feature = "sql")]
             SchematicSupplierType::SQL(schem) => {
                 match schem.get_schematic() {
                     Ok(schematic) => {
-                        search_schempath(search_behavior, &pattern, &mut output_std, &mut output_std_csv, &mut output_file_csv, &mut output_file, &mut lock, &mut file_out, schematic, schem.get_name());
+                        Result {
+                            name: schem.get_name(),
+                            matches: search(schematic, &pattern, search_behavior)
+                        }
                     }
                     Err(e) => {
                         if !output_std && !output_std_csv {
                             println!("Error while loading schematic ({}): {}", schem.get_name(), e.to_string());
                         }
+                        Result {
+                            name: schem.get_name(),
+                            matches: vec![]
+                        }
                     }
                 }
             }
         }
+    }).collect();
+
+    let stdout = io::stdout();
+    let mut lock = stdout.lock();
+
+    for matching in matches {
+        let schem_name = matching.name;
+        let matching = matching.matches;
+        for x in matching {
+            if output_std {
+                writeln!(lock, "Found match in '{}' at x: {}, y: {}, z: {}, % = {}", schem_name, x.0, x.1, x.2, x.3).unwrap();
+            }
+            if output_std_csv {
+                writeln!(lock, "{},{},{},{},{}", schem_name, x.0, x.1, x.2, x.3).unwrap();
+            }
+            if output_file {
+                writeln!(file_out.as_mut().unwrap(), "Found match in '{}' at x: {}, y: {}, z: {}, % = {}", schem_name, x.0, x.1, x.2, x.3).unwrap();
+            }
+            if output_file_csv {
+                writeln!(file_out.as_mut().unwrap(), "{},{},{},{},{}", schem_name, x.0, x.1, x.2, x.3).unwrap();
+            }
+        }
     }
 }
 
-fn load_schem(cmd: &mut Command, schem_path: &Path) -> Schematic {
+fn load_schem(schem_path: &PathBuf) -> Option<Schematic> {
     match Schematic::load(schem_path) {
-        Ok(x) => x,
+        Ok(x) => Some(x),
         Err(e) => {
-            cmd.error(ErrorKind::Io, format!("Error while loading Schematic ({}): {}", schem_path.file_name().unwrap().to_str().unwrap(), e.to_string())).exit();
+            println!("Error while loading schematic ({}): {}", schem_path.to_str().unwrap(), e.to_string());
+            None
         }
     }
 }
 
-fn search_schempath(search_behavior: SearchBehavior, pattern: &Schematic, output_std: &mut bool, output_std_csv: &mut bool, output_file_csv: &mut bool, output_file: &mut bool, stdout: &mut StdoutLock, file_out: &mut Option<BufWriter<File>>, schematic: Schematic, schem_name: String) {
+#[derive(Debug, Clone)]
-    if *output_std {
+struct Result {
-        writeln!(stdout, "Searching in schematic: {}", schem_name).unwrap();
+    name: String,
-    }
+    matches: Vec<(u16, u16, u16, f32)>,
-    if *output_file {
-        writeln!(file_out.as_mut().unwrap(), "Searching in schematic: {}", schem_name).unwrap();
-    }
-
-    let matches = search(schematic, pattern, search_behavior);
-
-    for x in matches {
-        if *output_std {
-            writeln!(stdout, "Found match at x: {}, y: {}, z: {}, % = {}", x.0, x.1, x.2, x.3).unwrap();
-        }
-        if *output_std_csv {
-            writeln!(stdout, "{},{},{},{},{}", schem_name, x.0, x.1, x.2, x.3).unwrap();
-        }
-        if *output_file {
-            writeln!(file_out.as_mut().unwrap(), "Found match at x: {}, y: {}, z: {}, % = {}", x.0, x.1, x.2, x.3).unwrap();
-        }
-        if *output_file_csv {
-            writeln!(file_out.as_mut().unwrap(), "{},{},{},{},{}", schem_name, x.0, x.1, x.2, x.3).unwrap();
-        }
-    }
 }
+

schemsearch-cli/src/types.rs

@@ -15,26 +15,26 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-use std::path::Path;
+use std::path::PathBuf;
 #[cfg(feature = "sql")]
 use futures::executor::block_on;
 use schemsearch_files::Schematic;
 #[cfg(feature = "sql")]
 use schemsearch_sql::{load_schemdata, SchematicNode};
 
-pub enum SchematicSupplierType<'local> {
+pub enum SchematicSupplierType {
-    PATH(Box<PathSchematicSupplier<'local>>),
+    PATH(Box<PathSchematicSupplier>),
     #[cfg(feature = "sql")]
     SQL(SqlSchematicSupplier),
 }
 
-pub struct PathSchematicSupplier<'local> {
+pub struct PathSchematicSupplier {
-    pub path: &'local Path,
+    pub path: PathBuf,
 }
 
-impl PathSchematicSupplier<'_> {
+impl PathSchematicSupplier {
     pub fn get_name(&self) -> String {
-        self.path.file_name().unwrap().to_str().unwrap().to_string()
+        self.path.file_stem().unwrap().to_str().unwrap().to_string()
     }
 }
 

schemsearch-files/src/lib.rs

@@ -16,7 +16,7 @@
  */
 
 use std::io::Read;
-use std::path::Path;
+use std::path::PathBuf;
 use nbt::{Map, Value};
 use serde::{Deserialize, Deserializer, Serialize};
 
@@ -81,10 +81,10 @@ impl Schematic {
         Ok(schematic)
     }
 
-    pub fn load(path: &Path) -> Result<Schematic, String> {
+    pub fn load(path: &PathBuf) -> Result<Schematic, String> {
         let file = match std::fs::File::open(path) {
             Ok(x) => x,
-            Err(_) => return Err(format!("Failed to open file: {}", path.display()))
+            Err(_) => return Err(format!("Failed to open file: {}", path.to_str().unwrap()))
         };
         Schematic::load_data(file)
     }

schemsearch-java/src/lib.rs

@@ -15,7 +15,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-use std::path::Path;
+use std::path::PathBuf;
 use jni::JNIEnv;
 
 use jni::objects::{JClass, JString};
@@ -32,8 +32,8 @@ pub extern "system" fn Java_SchemSearch_search<'local>(mut env: JNIEnv<'local>,
     pattern_path: JString<'local>) -> jstring {
     let schematic_path: String = env.get_string(&schematic_path).expect("Couldn't get java string!").into();
     let pattern_path: String = env.get_string(&pattern_path).expect("Couldn't get java string!").into();
-    let schematic = Schematic::load(Path::new(&schematic_path)).unwrap();
+    let schematic = Schematic::load(&PathBuf::from(&schematic_path)).unwrap();
-    let pattern = Schematic::load(Path::new(&pattern_path)).unwrap();
+    let pattern = Schematic::load(&PathBuf::from(&pattern_path)).unwrap();
 
     let matches = search(schematic, &pattern, SearchBehavior {
         ignore_block_data: true,