Add configurable thread pool size for MySQL connections and update schematics loading logic

🔧 Add invalid_nbt flag.

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "schemsearch-sql",
     "schemsearch-java"
 ]
+resolver = "2"
 
 [profile.small]
 inherits = "release"

schemsearch-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "schemsearch-cli"
-version = "0.1.5"
+version = "0.1.7"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 
@@ -13,7 +13,7 @@ schemsearch-sql = { path = "../schemsearch-sql", optional = true }
 
 clap = { version = "4.1.8", features = ["cargo"] }
 futures = { version = "0.3", optional = true }
-sqlx = { version = "0.6", features = [ "runtime-async-std-native-tls" , "mysql" ], optional = true }
+sqlx = { version = "0.7", features = [ "runtime-async-std-native-tls" , "mysql" ], optional = true }
 rayon = "1.7.0"
 indicatif = { version = "0.17.3", features = ["rayon"] }
 serde = "1.0.157"

schemsearch-cli/src/main.rs

@@ -26,8 +26,8 @@ use clap::{command, Arg, ArgAction, ValueHint};
 use std::path::PathBuf;
 use std::str::FromStr;
 use clap::error::ErrorKind;
-use schemsearch_lib::{Match, search, SearchBehavior};
+use schemsearch_lib::{Match, SearchBehavior};
-use crate::types::{PathSchematicSupplier, SchematicSupplierType};
+use crate::types::{PathSchematicSupplier, SchematicSupplier, SchematicSupplierType};
 #[cfg(feature = "sql")]
 use futures::executor::block_on;
 use rayon::prelude::*;
@@ -42,6 +42,8 @@ use indicatif::*;
 use schemsearch_files::SpongeSchematic;
 use crate::sinks::{OutputFormat, OutputSink};
 use crate::stderr::MaschineStdErr;
+use schemsearch_lib::nbt_search::has_invalid_nbt;
+use schemsearch_lib::search::search;
 
 fn main() {
     #[allow(unused_mut)]
@@ -49,8 +51,8 @@ fn main() {
         .arg(
             Arg::new("pattern")
                 .help("The pattern to search for")
-                .required(true)
                 .value_hint(ValueHint::FilePath)
+                .required_unless_present("invalid-nbt")
                 .action(ArgAction::Set),
         )
         .arg(
@@ -94,6 +96,13 @@ fn main() {
                 .long("air-as-any")
                 .action(ArgAction::SetTrue),
         )
+        .arg(
+            Arg::new("invalid-nbt")
+                .help("Search for Schematics with Invalid or missing NBT data")
+                .short('I')
+                .long("invalid-nbt")
+                .action(ArgAction::SetTrue),
+        )
         .arg(
             Arg::new("output")
                 .help("The output format and path [Format:Path] available formats: text, json, csv; available paths: std, err, (file path)")
@@ -134,7 +143,7 @@ fn main() {
         )
         .arg(
             Arg::new("threads")
-                .help("The number of threads to use [0 = Available Threads]")
+                .help("The number of threads to use [0 = all Available Threads]")
                 .short('T')
                 .long("threads")
                 .action(ArgAction::Set)
@@ -204,18 +213,22 @@ fn main() {
         air_as_any: matches.get_flag("air-as-any"),
         ignore_entities: matches.get_flag("ignore-entities"),
         threshold: *matches.get_one::<f32>("threshold").expect("Couldn't get threshold"),
+        invalid_nbt: matches.get_flag("invalid-nbt"),
     };
 
-    let pattern = match SpongeSchematic::load(&PathBuf::from(matches.get_one::<String>("pattern").unwrap())) {
+    let pattern = match matches.get_one::<String>("pattern") {
-        Ok(x) => x,
+        Some(p) => match SpongeSchematic::load(&PathBuf::from(p)) {
+            Ok(x) => Some(x),
             Err(e) => {
                 cmd.error(ErrorKind::Io, format!("Error while loading Pattern: {}", e.to_string())).exit();
             }
+        },
+        None => None,
     };
 
     let mut schematics: Vec<SchematicSupplierType> = Vec::new();
     match matches.get_many::<String>("schematic") {
-        None => {},
+        None => {}
         Some(x) => {
             let paths = x.map(|x| PathBuf::from(x));
             for path in paths {
@@ -226,12 +239,12 @@ fn main() {
                         .filter(|x| x.path().is_file())
                         .filter(|x| x.path().extension().unwrap().to_str().unwrap() == "schem")
                         .for_each(|x| {
-                            schematics.push(SchematicSupplierType::PATH(Box::new(PathSchematicSupplier {
+                            schematics.push(SchematicSupplierType::PATH(PathSchematicSupplier {
                                 path: x.path(),
-                            })))
+                            }))
                         });
                 } else if path.extension().unwrap().to_str().unwrap() == "schem" {
-                    schematics.push(SchematicSupplierType::PATH(Box::new(PathSchematicSupplier { path })));
+                    schematics.push(SchematicSupplierType::PATH(PathSchematicSupplier { path }));
                 }
             }
         }
@@ -246,8 +259,8 @@ fn main() {
         if let Some(x) = matches.get_many::<String>("sql-filter-name") {
             filter = filter.name(x.collect());
         }
-        for schem in block_on(load_all_schematics(filter)) {
+        for schem in block_on(load_all_schematics(filter, *matches.get_one::<usize>("threads").expect("Could not get Threads") as u32)) {
-            schematics.push(SchematicSupplierType::SQL(SqlSchematicSupplier{
+            schematics.push(SchematicSupplierType::SQL(SqlSchematicSupplier {
                 node: schem
             }))
         };
@@ -282,28 +295,20 @@ fn main() {
                     Some(x) => x,
                     None => return SearchResult {
                         name: schem.get_name(),
-                        matches: Vec::default()
+                        matches: Vec::default(),
                     }
                 };
-                SearchResult {
+                search_in_schem(schematic, pattern.as_ref(), search_behavior, schem)
-                    name: schem.get_name(),
-                    matches: search(schematic, &pattern, search_behavior)
-                }
             }
             #[cfg(feature = "sql")]
             SchematicSupplierType::SQL(schem) => {
                 match schem.get_schematic() {
-                    Ok(schematic) => {
+                    Ok(schematic) => search_in_schem(schematic, pattern.as_ref(), search_behavior, schem),
-                        SearchResult {
-                            name: schem.get_name(),
-                            matches: search(schematic, &pattern, search_behavior)
-                        }
-                    }
                     Err(e) => {
                         eprintln!("Error while loading schematic ({}): {}", schem.get_name(), e.to_string());
                         SearchResult {
                             name: schem.get_name(),
-                            matches: Vec::default()
+                            matches: Vec::default(),
                         }
                     }
                 }
@@ -334,6 +339,32 @@ fn main() {
     }
 }
 
+fn search_in_schem(schematic: SpongeSchematic, pattern: Option<&SpongeSchematic>, search_behavior: SearchBehavior, schem: &impl SchematicSupplier) -> SearchResult {
+    if search_behavior.invalid_nbt {
+        if has_invalid_nbt(schematic) {
+            SearchResult {
+                name: schem.get_name(),
+                matches: vec![Match {
+                    x: 0,
+                    y: 0,
+                    z: 0,
+                    percent: 1.0,
+                }],
+            }
+        } else {
+            SearchResult {
+                name: schem.get_name(),
+                matches: vec![],
+            }
+        }
+    } else {
+        SearchResult {
+            name: schem.get_name(),
+            matches: search(schematic, pattern.unwrap(), search_behavior),
+        }
+    }
+}
+
 fn load_schem(schem_path: &PathBuf) -> Option<SpongeSchematic> {
     match SpongeSchematic::load(schem_path) {
         Ok(x) => Some(x),

schemsearch-cli/src/sinks.rs

@@ -71,7 +71,7 @@ impl OutputFormat {
     pub fn start(&self, total: u32, search_behavior: &SearchBehavior, start_time: u128) -> String {
         match self {
             OutputFormat::Text => format!("Starting search in {} schematics\n", total),
-            OutputFormat::CSV => format!("Name,X,Y,Z,Percent\n"),
+            OutputFormat::CSV => "Name,X,Y,Z,Percent\n".to_owned(),
             OutputFormat::JSON => format!("{}\n", serde_json::to_string(&JsonEvent::Init(InitEvent {
                 total,
                 search_behavior: search_behavior.clone(),

schemsearch-cli/src/types.rs

@@ -26,17 +26,21 @@ use schemsearch_files::SpongeSchematic;
 use schemsearch_sql::{load_schemdata, SchematicNode};
 
 pub enum SchematicSupplierType {
-    PATH(Box<PathSchematicSupplier>),
+    PATH(PathSchematicSupplier),
     #[cfg(feature = "sql")]
     SQL(SqlSchematicSupplier),
 }
 
+pub trait SchematicSupplier {
+    fn get_name(&self) -> String;
+}
+
 pub struct PathSchematicSupplier {
     pub path: PathBuf,
 }
 
-impl PathSchematicSupplier {
+impl SchematicSupplier for PathSchematicSupplier {
-    pub fn get_name(&self) -> String {
+    fn get_name(&self) -> String {
         self.path.file_stem().unwrap().to_str().unwrap().to_string()
     }
 }
@@ -52,8 +56,13 @@ impl SqlSchematicSupplier {
         let mut schemdata = block_on(load_schemdata(self.node.id));
         SpongeSchematic::load_data(&mut Cursor::new(schemdata.as_mut_slice()))
     }
+}
 
-    pub fn get_name(&self) -> String {
+#[cfg(feature = "sql")]
+impl SchematicSupplier for SqlSchematicSupplier {
+    fn get_name(&self) -> String {
         format!("{} ({})", self.node.name, self.node.id)
     }
 }
+
+

schemsearch-files/src/lib.rs

@@ -15,10 +15,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+use nbt::{CompoundTag, Tag};
 use std::collections::hash_map::HashMap;
 use std::io::Read;
 use std::path::PathBuf;
-use nbt::{CompoundTag, Tag};
 
 #[derive(Clone, Debug)]
 pub struct SpongeSchematic {
@@ -62,8 +62,12 @@ pub struct Entity {
 }
 
 impl SpongeSchematic {
-    pub fn load_data<R>(data: &mut R) -> Result<SpongeSchematic, String> where R: Read {
+    pub fn load_data<R>(data: &mut R) -> Result<SpongeSchematic, String>
-        let nbt: CompoundTag = nbt::decode::read_gzip_compound_tag(data).map_err(|e| e.to_string())?;
+    where
+        R: Read,
+    {
+        let nbt: CompoundTag =
+            nbt::decode::read_gzip_compound_tag(data).map_err(|e| e.to_string())?;
         let version = nbt.get_i32("Version").unwrap_or_else(|_| {
             return if nbt.contains_key("Blocks") {
                 3
@@ -79,7 +83,10 @@ impl SpongeSchematic {
         match version {
             1 => SpongeSchematic::from_nbt_1(nbt),
             2 => SpongeSchematic::from_nbt_2(nbt),
-            3 => SpongeSchematic::from_nbt_3(nbt),
+            3 => SpongeSchematic::from_nbt_3(
+                nbt.get_compound_tag("Schematic")
+                    .map_err(|e| e.to_string())?,
+            ),
             _ => Err("Invalid schematic: Unknown Version".to_string()),
         }
     }
@@ -92,7 +99,10 @@ impl SpongeSchematic {
     pub fn from_nbt_1(nbt: CompoundTag) -> Result<Self, String> {
         Ok(Self {
             data_version: 0,
-            metadata: nbt.get_compound_tag("Metadata").map_err(|e| e.to_string())?.clone(),
+            metadata: nbt
+                .get_compound_tag("Metadata")
+                .map(|v| v.clone())
+                .unwrap_or_else(|_| CompoundTag::new()),
             width: nbt.get_i16("Width").map_err(|e| e.to_string())? as u16,
             height: nbt.get_i16("Height").map_err(|e| e.to_string())? as u16,
             length: nbt.get_i16("Length").map_err(|e| e.to_string())? as u16,
@@ -100,15 +110,21 @@ impl SpongeSchematic {
             palette_max: nbt.get_i32("PaletteMax").map_err(|e| e.to_string())?,
             palette: read_palette(nbt.get_compound_tag("Palette").map_err(|e| e.to_string())?),
             block_data: read_blocks(nbt.get_i8_vec("BlockData").map_err(|e| e.to_string())?),
-            block_entities: read_tile_entities(nbt.get_compound_tag_vec("TileEntities").unwrap_or_else(|_| vec![]))?,
+            block_entities: read_tile_entities(
+                nbt.get_compound_tag_vec("TileEntities")
+                    .unwrap_or_else(|_| vec![]),
+            )?,
             entities: None,
         })
     }
 
     pub fn from_nbt_2(nbt: CompoundTag) -> Result<Self, String> {
-        Ok(Self{
+        Ok(Self {
             data_version: nbt.get_i32("DataVersion").map_err(|e| e.to_string())?,
-            metadata: nbt.get_compound_tag("Metadata").map_err(|e| e.to_string())?.clone(),
+            metadata: nbt
+                .get_compound_tag("Metadata")
+                .map(|v| v.clone())
+                .unwrap_or_else(|_| CompoundTag::new()),
             width: nbt.get_i16("Width").map_err(|e| e.to_string())? as u16,
             height: nbt.get_i16("Height").map_err(|e| e.to_string())? as u16,
             length: nbt.get_i16("Length").map_err(|e| e.to_string())? as u16,
@@ -116,32 +132,37 @@ impl SpongeSchematic {
             palette_max: nbt.get_i32("PaletteMax").map_err(|e| e.to_string())?,
             palette: read_palette(nbt.get_compound_tag("Palette").map_err(|e| e.to_string())?),
             block_data: read_blocks(nbt.get_i8_vec("BlockData").map_err(|e| e.to_string())?),
-            block_entities: read_tile_entities(nbt.get_compound_tag_vec("BlockEntities").unwrap_or_else(|_| vec![]))?,
+            block_entities: read_tile_entities(
+                nbt.get_compound_tag_vec("BlockEntities")
+                    .unwrap_or_else(|_| vec![]),
+            )?,
             entities: None,
         })
     }
 
-    pub fn from_nbt_3(nbt: CompoundTag) -> Result<Self, String> {
+    pub fn from_nbt_3(nbt: &CompoundTag) -> Result<Self, String> {
         let blocks = nbt.get_compound_tag("Blocks").map_err(|e| e.to_string())?;
-        Ok(Self{
+        Ok(Self {
             data_version: nbt.get_i32("DataVersion").map_err(|e| e.to_string())?,
-            metadata: nbt.get_compound_tag("Metadata").map_err(|e| e.to_string())?.clone(),
+            metadata: nbt
+                .get_compound_tag("Metadata")
+                .map(|v| v.clone())
+                .unwrap_or_else(|_| CompoundTag::new()),
             width: nbt.get_i16("Width").map_err(|e| e.to_string())? as u16,
             height: nbt.get_i16("Height").map_err(|e| e.to_string())? as u16,
             length: nbt.get_i16("Length").map_err(|e| e.to_string())? as u16,
             offset: read_offset(nbt.get_i32_vec("Offset").map_err(|e| e.to_string())?)?,
-            palette_max: compute_palette_max(blocks.get_compound_tag("Palette").map_err(|e| e.to_string())?),
+            palette_max: compute_palette_max(blocks.get_compound_tag("Palette").map_err(|e| e.to_string())?) + 1,
             palette: read_palette(blocks.get_compound_tag("Palette").map_err(|e| e.to_string())?),
-            block_data: read_blocks(blocks.get_i8_vec("BlockData").map_err(|e| e.to_string())?),
+            block_data: read_blocks(blocks.get_i8_vec("Data").map_err(|e| e.to_string())?),
             block_entities: read_tile_entities(blocks.get_compound_tag_vec("BlockEntities").unwrap_or_else(|_| vec![]))?,
             entities: None,
         })
     }
-
 }
 
 fn read_tile_entities(tag: Vec<&CompoundTag>) -> Result<Vec<BlockEntity>, String> {
-let mut tile_entities = Vec::new();
+    let mut tile_entities = Vec::new();
     for t in tag {
         tile_entities.push(BlockEntity {
             id: t.get_str("Id").map_err(|e| e.to_string())?.to_string(),
@@ -164,8 +185,10 @@ fn read_palette(p: &CompoundTag) -> HashMap<String, i32> {
     let mut palette = HashMap::new();
     for (key, value) in p.iter() {
         match value {
-            Tag::Int(n) => { palette.insert(key.clone(), *n); },
+            Tag::Int(n) => {
-            _ => {},
+                palette.insert(key.clone(), *n);
+            }
+            _ => {}
         };
     }
     palette
@@ -173,10 +196,15 @@ fn read_palette(p: &CompoundTag) -> HashMap<String, i32> {
 
 #[inline]
 fn compute_palette_max(palette: &CompoundTag) -> i32 {
-    palette.iter().map(|(_, v)| v).filter_map(|v| match v {
+    palette
+        .iter()
+        .map(|(_, v)| v)
+        .filter_map(|v| match v {
             Tag::Int(n) => Some(*n),
             _ => None,
-    }).max().unwrap_or(0)
+        })
+        .max()
+        .unwrap_or(0)
 }
 
 #[inline]
@@ -193,13 +221,16 @@ pub fn read_varint_array(read: &Vec<i8>) -> Vec<i32> {
     let mut cursor = 0;
     loop {
         match read.get(cursor) {
-            Some(byte) => { current_byte = *byte as u8; cursor += 1; },
+            Some(byte) => {
+                current_byte = *byte as u8;
+                cursor += 1;
+            }
             None => break,
         };
 
         value |= (((current_byte & 0x7F) as u32) << position) as i32;
 
-        if(current_byte & 0x80) == 0 {
+        if (current_byte & 0x80) == 0 {
             data.push(value);
             value = 0;
             position = 0;

schemsearch-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "schemsearch-lib"
-version = "0.1.5"
+version = "0.1.7"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 
@@ -10,3 +10,5 @@ license = "AGPL-3.0-or-later"
 serde = { version = "1.0.160", features = ["derive"] }
 schemsearch-files = { path = "../schemsearch-files" }
 named-binary-tag = "0.6"
+libmath = "0.2.1"
+lazy_static = "1.4.0"

schemsearch-lib/src/blocks.txt

@@ -0,0 +1,163 @@
+oak_sign
+oak_wall_sign
+oak_hanging_sign
+oak_wall_hanging_sign
+birch_sign
+birch_wall_sign
+birch_hanging_sign
+birch_wall_hanging_sign
+spruce_sign
+spruce_wall_sign
+spruce_hanging_sign
+spruce_wall_hanging_sign
+jungle_sign
+jungle_wall_sign
+jungle_hanging_sign
+jungle_wall_hanging_sign
+dark_oak_sign
+dark_oak_wall_sign
+dark_oak_hanging_sign
+dark_oak_wall_hanging_sign
+acacia_sign
+acacia_wall_sign
+acacia_hanging_sign
+acacia_wall_hanging_sign
+mangrove_sign
+mangrove_wall_sign
+mangrove_hanging_sign
+mangrove_wall_hanging_sign
+cherry_sign
+cherry_wall_sign
+cherry_hanging_sign
+cherry_wall_hanging_sign
+bamboo_sign
+bamboo_wall_sign
+bamboo_hanging_sign
+bamboo_wall_hanging_sign
+warped_sign
+warped_wall_sign
+warped_hanging_sign
+warped_wall_hanging_sign
+crimson_sign
+crimson_wall_sign
+crimson_hanging_sign
+crimson_wall_hanging_sign
+suspicious_gravel
+suspicious_sand
+white_banner
+light_gray_banner
+gray_banner
+black_banner
+brown_banner
+red_banner
+orange_banner
+yellow_banner
+lime_banner
+green_banner
+cyan_banner
+light_blue_banner
+blue_banner
+purple_banner
+magenta_banner
+pink_banner
+white_wall_banner
+light_gray_wall_banner
+gray_wall_banner
+black_wall_banner
+brown_wall_banner
+red_wall_banner
+orange_wall_banner
+yellow_wall_banner
+lime_wall_banner
+green_wall_banner
+cyan_wall_banner
+light_blue_wall_banner
+blue_wall_banner
+purple_wall_banner
+magenta_wall_banner
+pink_wall_banner
+white_bed
+light_gray_bed
+gray_bed
+black_bed
+brown_bed
+red_bed
+orange_bed
+yellow_bed
+lime_bed
+green_bed
+cyan_bed
+light_blue_bed
+blue_bed
+purple_bed
+magenta_bed
+pink_bed
+shulker_box
+white_shulker_box
+light_gray_shulker_box
+gray_shulker_box
+black_shulker_box
+brown_shulker_box
+red_shulker_box
+orange_shulker_box
+yellow_shulker_box
+lime_shulker_box
+green_shulker_box
+cyan_shulker_box
+light_blue_shulker_box
+blue_shulker_box
+purple_shulker_box
+magenta_shulker_box
+pink_shulker_box
+furnace
+blast_furnace
+smoker
+chest
+trapped_chest
+ender_chest
+enchanting_table
+barrel
+lectern
+jukebox
+bell
+brewing_stand
+bee_nest
+beehive
+decorated_pot
+beacon
+conduit
+campfire
+soul_campfire
+redstone_comparator
+hopper
+dispenser
+dropper
+moving_piston
+daylight_detector
+sculk_sensor
+calibrated_sculk_sensor
+sculk_catalyst
+sculk_shrieker
+player_head
+player_wall_head
+wither_skeleton_skull
+wither_skeleton_wall_skull
+zombie_head
+zombie_wall_head
+skeleton_skull
+skeleton_wall_skull
+creeper_head
+creeper_wall_head
+piglin_head
+piglin_wall_head
+dragon_head
+dragon_wall_head
+chiseled_bookshelf
+command_block
+chain_command_block
+repeating_command_block
+structure_block
+jigsaw_block
+end_portal
+end_gateway
+monster_spawner

schemsearch-lib/src/lib.rs

@@ -16,11 +16,10 @@
  */
 
 pub mod pattern_mapper;
+pub mod search;
+pub mod nbt_search;
 
 use serde::{Serialize, Deserialize};
-use pattern_mapper::match_palette;
-use schemsearch_files::SpongeSchematic;
-use crate::pattern_mapper::match_palette_adapt;
 
 #[derive(Debug, Clone, Copy, Deserialize, Serialize)]
 pub struct SearchBehavior {
@@ -30,82 +29,10 @@ pub struct SearchBehavior {
     pub air_as_any: bool,
     pub ignore_entities: bool,
     pub threshold: f32,
+    pub invalid_nbt: bool,
 }
 
-pub fn search(
+#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)]
-    schem: SpongeSchematic,
-    pattern_schem: &SpongeSchematic,
-    search_behavior: SearchBehavior,
-) -> Vec<Match> {
-    if schem.width < pattern_schem.width || schem.height < pattern_schem.height || schem.length < pattern_schem.length {
-        return vec![];
-    }
-
-    if pattern_schem.palette.len() > schem.palette.len() {
-        return vec![];
-    }
-
-    let pattern_schem = match_palette(&schem, &pattern_schem, search_behavior.ignore_block_data);
-
-    let mut matches: Vec<Match> = Vec::new();
-
-    let pattern_data = pattern_schem.block_data.as_slice();
-
-    let schem_data = if search_behavior.ignore_block_data {
-        match_palette_adapt(&schem, &pattern_schem.palette, search_behavior.ignore_block_data)
-    } else {
-        schem.block_data.clone()
-    };
-
-    let schem_data = schem_data.as_slice();
-
-    let air_id = if search_behavior.ignore_air || search_behavior.air_as_any { pattern_schem.palette.get("minecraft:air").unwrap_or(&-1) } else { &-1};
-
-    let pattern_blocks = pattern_data.len() as f32;
-
-    let pattern_width = pattern_schem.width as usize;
-    let pattern_height = pattern_schem.height as usize;
-    let pattern_length = pattern_schem.length as usize;
-
-    let schem_width = schem.width as usize;
-    let schem_height = schem.height as usize;
-    let schem_length = schem.length as usize;
-
-    let matching_needed = (pattern_blocks * search_behavior.threshold) as i32;
-
-    for y in 0..=schem_height - pattern_height {
-        for z in 0..=schem_length - pattern_length {
-            for x in 0..=schem_width - pattern_width {
-                let mut matching = 0;
-                for j in 0..pattern_height {
-                    for k in 0..pattern_length {
-                        for i in 0..pattern_width {
-                            let index = (x + i) + schem_width * ((z + k) + (y + j) * schem_length);
-                            let pattern_index = i + pattern_width * (k + j * pattern_length);
-                            let data = unsafe {schem_data.get_unchecked(index) };
-                            let pattern_data = unsafe { pattern_data.get_unchecked(pattern_index) };
-                            if *data == *pattern_data || (search_behavior.ignore_air && *data == *air_id) || (search_behavior.air_as_any && *pattern_data == *air_id) {
-                                matching += 1;
-                            }
-                        }
-                    }
-                }
-                if matching >= matching_needed {
-                    matches.push(Match {
-                        x: x as u16,
-                        y: y as u16,
-                        z: z as u16,
-                        percent: matching as f32 / pattern_blocks,
-                    });
-                }
-            }
-        }
-    }
-
-    return matches;
-}
-
-#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
 pub struct Match {
     pub x: u16,
     pub y: u16,
@@ -113,17 +40,6 @@ pub struct Match {
     pub percent: f32,
 }
 
-impl Default for Match {
-    fn default() -> Self {
-        Self {
-            x: 0,
-            y: 0,
-            z: 0,
-            percent: 0.0,
-        }
-    }
-}
-
 #[inline]
 pub fn normalize_data(data: &str, ignore_data: bool) -> &str {
     if ignore_data {
@@ -137,7 +53,9 @@ pub fn normalize_data(data: &str, ignore_data: bool) -> &str {
 #[cfg(test)]
 mod tests {
     use std::path::{Path, PathBuf};
-    use crate::pattern_mapper::strip_data;
+    use schemsearch_files::SpongeSchematic;
+    use crate::pattern_mapper::{match_palette, strip_data};
+    use crate::search::search;
     use super::*;
 
     #[test]
@@ -191,7 +109,8 @@ mod tests {
             ignore_entities: true,
             ignore_air: false,
             air_as_any: false,
-            threshold: 0.9
+            threshold: 0.9,
+            invalid_nbt: false
         });
     }
 
@@ -206,10 +125,10 @@ mod tests {
             ignore_entities: true,
             ignore_air: false,
             air_as_any: false,
-            threshold: 0.9
+            threshold: 0.9,
+            invalid_nbt: false
         });
 
-        println!("{:?}", matches);
         assert_eq!(matches.len(), 1);
         assert_eq!(matches[0].x, 1);
         assert_eq!(matches[0].y, 0);
@@ -228,10 +147,10 @@ mod tests {
             ignore_entities: false,
             ignore_air: false,
             air_as_any: false,
-            threshold: 0.9
+            threshold: 0.9,
+            invalid_nbt: false
         });
 
-        println!("{:?}", matches);
         assert_eq!(matches.len(), 1);
     }
 }

schemsearch-lib/src/nbt_search.rs

@@ -0,0 +1,110 @@
+use std::borrow::ToOwned;
+use std::collections::HashSet;
+use std::iter::Iterator;
+use lazy_static::lazy_static;
+use schemsearch_files::SpongeSchematic;
+
+const NBT_BLOCKS: &str = include_str!("blocks.txt");
+
+lazy_static! {
+    static ref NBT_BLOCKS_SET: HashSet<String> = {
+        NBT_BLOCKS.lines().map(|x| format!("minecraft:{}", x)).collect()
+    };
+}
+
+pub fn has_invalid_nbt(schem: SpongeSchematic) -> bool {
+    if schem.block_entities.is_empty() && schem.palette.keys().any(|v| NBT_BLOCKS_SET.contains(v)) {
+        return true;
+    }
+
+    let nbt_blocks = schem.palette.iter().filter(|(k, _)| NBT_BLOCKS_SET.contains(k.to_owned())).map(|(_, v)| *v).collect::<HashSet<i32>>();
+
+    for (i, block_entity) in schem.block_data.iter().enumerate() {
+        if nbt_blocks.contains(&*block_entity) {
+            // i = x + z * Width + y * Width * Length
+            let x = i % schem.width as usize;
+            let z = (i / schem.width as usize) % schem.length as usize;
+            let y = i / (schem.width as usize * schem.length as usize);
+            if schem.block_entities.iter().any(|e| !e.pos.eq(&[x as i32, y as i32, z as i32])) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+#[allow(unused_imports)]
+#[cfg(test)]
+mod tests {
+    use nbt::CompoundTag;
+    use schemsearch_files::{BlockEntity, SpongeSchematic};
+    use super::*;
+
+    #[test]
+    fn test_has_invalid_nbt() {
+        let schem = SpongeSchematic {
+            data_version: 1,
+            metadata: CompoundTag::new(),
+            width: 0,
+            height: 0,
+            length: 0,
+            offset: [0, 0, 0],
+            palette_max: 1,
+            palette: vec![("minecraft:chest".to_owned(), 1)].into_iter().collect(),
+            block_data: vec![1],
+            block_entities: vec![],
+            entities: None,
+        };
+
+        assert_eq!(has_invalid_nbt(schem), true);
+    }
+
+    #[test]
+    fn test_has_invalid_nbt_2() {
+        let schem = SpongeSchematic {
+            data_version: 1,
+            metadata: CompoundTag::new(),
+            width: 1,
+            height: 1,
+            length: 1,
+            offset: [0, 0, 0],
+            palette_max: 1,
+            palette: vec![("minecraft:chest".to_owned(), 1)].into_iter().collect(),
+            block_data: vec![1],
+            block_entities: vec![
+                BlockEntity {
+                    id: "minecraft:chest".to_owned(),
+                    pos: [0, 0, 0],
+                }
+            ],
+            entities: None,
+        };
+
+        assert_eq!(has_invalid_nbt(schem), false);
+    }
+
+    #[test]
+    fn test_has_invalid_nbt_3() {
+        let schem = SpongeSchematic {
+            data_version: 1,
+            metadata: CompoundTag::new(),
+            width: 2,
+            height: 1,
+            length: 1,
+            offset: [0, 0, 0],
+            palette_max: 1,
+            palette: vec![("minecraft:chest".to_owned(), 1), ("minecraft:stone".to_owned(), 2)].into_iter().collect(),
+            block_data: vec![1, 2],
+            block_entities: vec![
+                BlockEntity {
+                    id: "minecraft:chest".to_owned(),
+                    pos: [1, 0, 0],
+                }
+            ],
+            entities: None,
+        };
+
+        assert_eq!(has_invalid_nbt(schem), true);
+    }
+}

schemsearch-lib/src/pattern_mapper.rs

@@ -64,10 +64,10 @@ pub fn strip_data(schem: &SpongeSchematic) -> SpongeSchematic {
 }
 
 pub fn match_palette_adapt(schem: &SpongeSchematic, matching_palette: &HashMap<String, i32>, ignore_data: bool) -> Vec<i32> {
-    let mut data: Vec<i32> = Vec::new();
+    let mut data = Vec::with_capacity(schem.block_data.len());
     let reverse_palette = create_reverse_palette(schem);
 
-    for x in schem.block_data.iter() {
+    for x in schem.block_data.as_slice().iter() {
         let blockname = reverse_palette[*x as usize];
         let blockname = if ignore_data { normalize_data(blockname, ignore_data) } else { blockname };
         let block_id = match matching_palette.get(&*blockname) {

schemsearch-lib/src/search.rs

@@ -0,0 +1,87 @@
+use math::round::ceil;
+use schemsearch_files::SpongeSchematic;
+use crate::{Match, SearchBehavior};
+use crate::pattern_mapper::{match_palette, match_palette_adapt};
+
+pub fn search(
+    schem: SpongeSchematic,
+    pattern_schem: &SpongeSchematic,
+    search_behavior: SearchBehavior,
+) -> Vec<Match> {
+    if schem.width < pattern_schem.width || schem.height < pattern_schem.height || schem.length < pattern_schem.length {
+        return Vec::new();
+    }
+
+    if pattern_schem.palette.len() > schem.palette.len() {
+        return Vec::new();
+    }
+
+    let pattern_schem = match_palette(&schem, &pattern_schem, search_behavior.ignore_block_data);
+
+    let mut matches: Vec<Match> = Vec::with_capacity(4);
+
+    let pattern_data = pattern_schem.block_data.as_ptr();
+
+    let schem_data = if search_behavior.ignore_block_data {
+        match_palette_adapt(&schem, &pattern_schem.palette, search_behavior.ignore_block_data)
+    } else {
+        schem.block_data
+    };
+
+    let schem_data = schem_data.as_ptr();
+
+    let air_id = if search_behavior.ignore_air || search_behavior.air_as_any { pattern_schem.palette.get("minecraft:air").unwrap_or(&-1) } else { &-1};
+
+    let pattern_blocks = pattern_schem.block_data.len() as f32;
+    let i_pattern_blocks = pattern_blocks as i32;
+
+    let pattern_width = pattern_schem.width as usize;
+    let pattern_height = pattern_schem.height as usize;
+    let pattern_length = pattern_schem.length as usize;
+
+    let schem_width = schem.width as usize;
+    let schem_height = schem.height as usize;
+    let schem_length = schem.length as usize;
+
+    let skip_amount = ceil((pattern_blocks * (1.0 - search_behavior.threshold)) as f64, 0) as i32;
+
+    for y in 0..=schem_height - pattern_height {
+        for z in 0..=schem_length - pattern_length {
+            for x in 0..=schem_width - pattern_width {
+                let mut not_matching = 0;
+                'outer:
+                for j in 0..pattern_height {
+                    for k in 0..pattern_length {
+                        'inner:
+                        for i in 0..pattern_width {
+                            let index = (x + i) + schem_width * ((z + k) + (y + j) * schem_length);
+                            let pattern_index = i + pattern_width * (k + j * pattern_length);
+                            let data = unsafe { *schem_data.add(index) };
+                            let pattern_data = unsafe { *pattern_data.add(pattern_index) };
+                            if (search_behavior.ignore_air && data != *air_id) || (search_behavior.air_as_any && pattern_data != *air_id) {
+                                continue 'inner;
+                            }
+                            if data != pattern_data {
+                                not_matching += 1;
+                                if not_matching >= skip_amount {
+                                    break 'outer;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if not_matching < skip_amount {
+                    matches.push(Match {
+                        x: x as u16,
+                        y: y as u16,
+                        z: z as u16,
+                        percent: (i_pattern_blocks - not_matching) as f32 / pattern_blocks,
+                    });
+                }
+            }
+        }
+    }
+
+    return matches;
+}

schemsearch-sql/Cargo.toml

@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sqlx = { version = "0.6", features = [ "runtime-async-std-native-tls" , "mysql" ] }
+sqlx = { version = "0.7", features = [ "runtime-async-std-native-tls" , "mysql" ] }
 
 schemsearch-lib = { path = "../schemsearch-lib" }
 schemsearch-files = { path = "../schemsearch-files" }

schemsearch-sql/src/lib.rs

@@ -30,12 +30,12 @@ pub struct SchematicNode {
     pub name: String
 }
 
-pub async unsafe fn get_connection() {
+pub async unsafe fn get_connection(connections: u32) {
     let mut conn = CONN.lock().unwrap();
     if conn.is_none() {
         let properties = properties::load_mysql_properties();
         let _ = conn.insert(MySqlPoolOptions::new()
-            .max_connections(5)
+            .max_connections(connections)
             .connect_with(MySqlConnectOptions::new()
                 .host(properties.host.as_str())
                 .port(3306)
@@ -46,10 +46,10 @@ pub async unsafe fn get_connection() {
     }
 }
 
-pub async fn load_all_schematics(filter: SchematicFilter) -> Vec<SchematicNode> {
+pub async fn load_all_schematics(filter: SchematicFilter, connections: u32) -> Vec<SchematicNode> {
-    unsafe { get_connection().await; }
+    unsafe { get_connection(connections).await; }
     let mut schematics = Vec::new();
-    let rows = unsafe { &CONN }.lock().unwrap().as_mut().unwrap().fetch_all(format!("SELECT SN.NodeId, SN.NodeName FROM NodeData ND INNER JOIN SchematicNode SN ON SN.NodeId = ND.NodeId WHERE NodeFormat = true {}", filter.build()).as_str()).await.expect("Failed to fetch schematics");
+    let rows = unsafe { &CONN }.lock().unwrap().as_mut().unwrap().fetch_all(format!("SELECT SN.NodeId, SN.NodeName FROM NodeData ND INNER JOIN SchematicNode SN ON SN.NodeId = ND.NodeId WHERE NodeFormat >= 2 {}", filter.build()).as_str()).await.expect("Failed to fetch schematics");
     for row in rows {
         schematics.push(SchematicNode {
             id: row.get(0),
@@ -60,7 +60,7 @@ pub async fn load_all_schematics(filter: SchematicFilter) -> Vec<SchematicNode>
 }
 
 pub async fn load_schemdata(id: i32) -> Vec<u8> {
-    unsafe { get_connection().await; }
+    unsafe { get_connection(5).await; }
     let rows = unsafe { &CONN }.lock().unwrap().as_mut().unwrap().fetch_one(format!("SELECT SchemData FROM NodeData WHERE NodeId = {}", id).as_str()).await.expect("Failed to fetch schematics");
     rows.get(0)
 }