build: Add SIMD optimizations to compare.c

Add SIMD (Single Instruction, Multiple Data) optimizations to the
compare.c file to improve performance. This includes vectorization
flags and architecture-specific optimizations to enhance the
processing of data in the comparison function. Additionally, the
implementation now dynamically selects an optimized code path based
on the size of the pattern data, ensuring efficient execution for
various scenarios.

By Lixfel

.gitignore

@@ -3,3 +3,5 @@
 /.idea/
 /SchemSearch.class
 /SchemSearch.h
+/schemsearch-lib/src/.idea/
+/schemsearch-lib/src/cmake-build-debug/

Cargo.toml

@@ -16,4 +16,5 @@ opt-level = "z"
 codegen-units = 1
 
 [profile.release]
-lto = true
+lto = true
+opt-level = 3

Makefile

@@ -1,10 +1,10 @@
 default:
 	@echo "Building (Release)...";
-	cargo rustc --release --color=always -p schemsearch-cli -- -C target-feature=+avx2
+	cargo rustc --release --color=always -p schemsearch-cli -- -C target-cpu=native
 
 sql:
 	@echo "Building (Release)...";
-	cargo rustc --release --color=always -p schemsearch-cli --features sql -- -C target-feature=+avx2
+	cargo rustc --release --color=always -p schemsearch-cli --features sql -- -C target-cpu=native
 
 debug:
 	@echo "Building (Debug)...";

schemsearch-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "schemsearch-cli"
-version = "0.1.5"
+version = "0.1.6"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 
@@ -13,7 +13,7 @@ schemsearch-sql = { path = "../schemsearch-sql", optional = true }
 
 clap = { version = "4.1.8", features = ["cargo"] }
 futures = { version = "0.3", optional = true }
-sqlx = { version = "0.6", features = [ "runtime-async-std-native-tls" , "mysql" ], optional = true }
+sqlx = { version = "0.7", features = [ "runtime-async-std-native-tls" , "mysql" ], optional = true }
 rayon = "1.7.0"
 indicatif = { version = "0.17.3", features = ["rayon"] }
 serde = "1.0.157"

schemsearch-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "schemsearch-lib"
-version = "0.1.5"
+version = "0.1.6"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 
@@ -10,3 +10,7 @@ license = "AGPL-3.0-or-later"
 serde = { version = "1.0.160", features = ["derive"] }
 schemsearch-files = { path = "../schemsearch-files" }
 named-binary-tag = "0.6"
+libmath = "0.2.1"
+
+[build-dependencies]
+cc = { version = "1.0.83", features = [] }

schemsearch-lib/build.rs

@@ -0,0 +1,11 @@
+use cc;
+
+fn main() {
+    cc::Build::new()
+        .file("src/compare.c")
+        .flag("-ftree-vectorize")
+        .flag("-march=native")
+        .flag("-mtune=native")
+        .flag("-ffast-math")
+        .compile("compare");
+}

schemsearch-lib/src/CMakeLists.txt

@@ -0,0 +1,7 @@
+cmake_minimum_required(VERSION 3.27)
+project(src C)
+
+set(CMAKE_C_STANDARD 11)
+
+add_executable(src
+        compare.c)

schemsearch-lib/src/compare.c

@@ -0,0 +1,110 @@
+#include <stdint.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+int32_t isMatching(
+        const int32_t *schem_data,
+        const int32_t *pattern_data,
+        size_t pattern_data_length,
+        int32_t x,
+        int32_t y,
+        int32_t z,
+        int32_t schem_width,
+        int32_t schem_length,
+        int32_t pattern_width,
+        int32_t pattern_height,
+        int32_t pattern_length,
+        int32_t *w_ptr
+        ) {
+    for (int j = 0; j < pattern_height; ++j) {
+        for (int k = 0; k < pattern_length; ++k) {
+            int pattern_index_pre = k * pattern_width + j * pattern_width * pattern_length;
+            int schem_index_pre = x + (k + z) * schem_width + (j + y) * schem_width * schem_length;
+            for (int i = 0; i < pattern_width; ++i) {
+                int pattern_index = i + pattern_index_pre;
+                int schem_index = i + schem_index_pre;
+                w_ptr[pattern_index] = schem_data[schem_index];
+            }
+        }
+    }
+
+    int32_t matching = 0;
+    for (int i = 0; i < pattern_data_length; ++i) {
+        matching += w_ptr[i] == pattern_data[i];
+    }
+
+    return matching;
+}
+
+void is_matching_all(
+        const int32_t *__restrict__ schem_data,
+        const int32_t *__restrict__ pattern_data,
+        int32_t schem_width,
+        int32_t schem_height,
+        int32_t schem_length,
+        int32_t pattern_width,
+        int32_t pattern_height,
+        int32_t pattern_length,
+        int32_t *__restrict__ result
+) {
+    if(pattern_width*pattern_height*pattern_length >= 65536) { //TODO check for table size < 65536
+        for (int32_t pz = 0; pz < pattern_length; ++pz) {
+            int32_t maxZ = schem_length - pattern_length + pz + 1;
+            for (int32_t py = 0; py < pattern_height; ++py) {
+                int32_t maxY = schem_height - pattern_height + py + 1;
+                for (int32_t px = 0; px < pattern_width; ++px) {
+                    int32_t pv = pattern_data[px + py * pattern_width + pz * pattern_width * pattern_height];
+                    int32_t maxX = schem_width - pattern_width + px + 1;
+                    for (int32_t z = pz; z < maxZ; ++z) {
+                        int32_t sourceOffsetZ = z * schem_width * schem_height;
+                        int32_t resultOffsetZ = (z - pz) * schem_width * schem_height - py * schem_width;
+                        for (int32_t y = py; y < maxY; ++y) {
+                            int32_t sourceOffsetY = sourceOffsetZ + y * schem_width;
+                            int32_t resultOffsetY = resultOffsetZ + y * schem_width - px;
+                            for (int32_t x = px; x < maxX; ++x) {
+                                result[resultOffsetY + x] += schem_data[sourceOffsetY + x] == pv;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    } else {
+        size_t schem_size = schem_width*schem_height*schem_length;
+        uint16_t *__restrict__ sschem_data = (uint16_t*)malloc(schem_size*2);
+        uint16_t *__restrict__ sresult = (uint16_t*)malloc(schem_size*2);
+        for(size_t i = 0; i < schem_size; i++) {
+            sschem_data[i] = schem_data[i];
+            sresult[i] = 0;
+        }
+
+        for (int32_t pz = 0; pz < pattern_length; ++pz) {
+            int32_t maxZ = schem_length - pattern_length + pz + 1;
+            for (int32_t py = 0; py < pattern_height; ++py) {
+                int32_t maxY = schem_height - pattern_height + py + 1;
+                for (int32_t px = 0; px < pattern_width; ++px) {
+                    uint16_t pv = (uint16_t)pattern_data[px + py * pattern_width + pz * pattern_width * pattern_height];
+                    int32_t maxX = schem_width - pattern_width + px + 1;
+                    for (int32_t z = pz; z < maxZ; ++z) {
+                        int32_t sourceOffsetZ = z * schem_width * schem_height;
+                        int32_t resultOffsetZ = (z - pz) * schem_width * schem_height - py * schem_width;
+                        for (int32_t y = py; y < maxY; ++y) {
+                            int32_t sourceOffsetY = sourceOffsetZ + y * schem_width;
+                            int32_t resultOffsetY = resultOffsetZ + y * schem_width - px;
+                            for (int32_t x = px; x < maxX; ++x) {
+                                sresult[resultOffsetY + x] += sschem_data[sourceOffsetY + x] == pv;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        for(size_t i = 0; i < schem_size; i++) {
+            result[i] = sresult[i];
+        }
+
+        free(sschem_data);
+        free(sresult);
+    }
+}

schemsearch-lib/src/lib.rs

@@ -21,6 +21,7 @@ use serde::{Serialize, Deserialize};
 use pattern_mapper::match_palette;
 use schemsearch_files::SpongeSchematic;
 use crate::pattern_mapper::match_palette_adapt;
+use math::round::ceil;
 
 #[derive(Debug, Clone, Copy, Deserialize, Serialize)]
 pub struct SearchBehavior {
@@ -32,36 +33,68 @@ pub struct SearchBehavior {
     pub threshold: f32,
 }
 
+extern "C" {
+    pub fn isMatching(
+        schem_data: *const i32,
+        pattern_data: *const i32,
+        pattern_data_length: usize,
+        x: usize,
+        y: usize,
+        z: usize,
+        schem_width: usize,
+        schem_length: usize,
+        pattern_width: usize,
+        pattern_height: usize,
+        pattern_length: usize,
+        w_ptr: *mut i32,
+    ) -> i32;
+
+
+    pub fn is_matching_all(
+        schem_data: *const i32,
+        pattern_data: *const i32,
+        schem_width: i32,
+        schem_height: i32,
+        schem_length: i32,
+        pattern_width: i32,
+        pattern_height: i32,
+        pattern_length: i32,
+        result: *mut i32
+    );
+}
+
 pub fn search(
     schem: SpongeSchematic,
     pattern_schem: &SpongeSchematic,
     search_behavior: SearchBehavior,
 ) -> Vec<Match> {
     if schem.width < pattern_schem.width || schem.height < pattern_schem.height || schem.length < pattern_schem.length {
-        return vec![];
+        return Vec::new();
     }
 
     if pattern_schem.palette.len() > schem.palette.len() {
-        return vec![];
+        return Vec::new();
     }
 
     let pattern_schem = match_palette(&schem, &pattern_schem, search_behavior.ignore_block_data);
 
-    let mut matches: Vec<Match> = Vec::new();
+    let mut matches: Vec<Match> = Vec::with_capacity(4);
 
-    let pattern_data = pattern_schem.block_data.as_slice();
+    let pattern_data = pattern_schem.block_data.as_ptr();
 
     let schem_data = if search_behavior.ignore_block_data {
         match_palette_adapt(&schem, &pattern_schem.palette, search_behavior.ignore_block_data)
     } else {
-        schem.block_data.clone()
+        schem.block_data
     };
 
-    let schem_data = schem_data.as_slice();
+    let schem_data = schem_data.as_ptr();
 
     let air_id = if search_behavior.ignore_air || search_behavior.air_as_any { pattern_schem.palette.get("minecraft:air").unwrap_or(&-1) } else { &-1};
 
-    let pattern_blocks = pattern_data.len() as f32;
+    let pattern_blocks_usize = pattern_schem.block_data.len();
+    let pattern_blocks = pattern_blocks_usize as f32;
+    let i_pattern_blocks = pattern_blocks as i32;
 
     let pattern_width = pattern_schem.width as usize;
     let pattern_height = pattern_schem.height as usize;
@@ -71,41 +104,79 @@ pub fn search(
     let schem_height = schem.height as usize;
     let schem_length = schem.length as usize;
 
-    let matching_needed = (pattern_blocks * search_behavior.threshold) as i32;
+    let skip_amount = ceil((pattern_blocks * (1.0 - search_behavior.threshold)) as f64, 0) as i32;
 
-    for y in 0..=schem_height - pattern_height {
+    /*for y in 0..=schem_height - pattern_height {
         for z in 0..=schem_length - pattern_length {
             for x in 0..=schem_width - pattern_width {
-                let mut matching = 0;
+                let matching_count;
-                for j in 0..pattern_height {
+                unsafe {
-                    for k in 0..pattern_length {
+                      matching_count = isMatching(
-                        for i in 0..pattern_width {
+                        schem_data,
-                            let index = (x + i) + schem_width * ((z + k) + (y + j) * schem_length);
+                        pattern_data,
-                            let pattern_index = i + pattern_width * (k + j * pattern_length);
+                        pattern_blocks_usize,
-                            let data = unsafe {schem_data.get_unchecked(index) };
+                        x,
-                            let pattern_data = unsafe { pattern_data.get_unchecked(pattern_index) };
+                        y,
-                            if *data == *pattern_data || (search_behavior.ignore_air && *data == *air_id) || (search_behavior.air_as_any && *pattern_data == *air_id) {
+                        z,
-                                matching += 1;
+                        schem_width,
-                            }
+                        schem_length,
-                        }
+                        pattern_width,
+                        pattern_height,
+                        pattern_length,
+                        w_ptr,
+                    );
+                };
+
+                if matching_count >= i_pattern_blocks - skip_amount {
+                    let percent = matching_count as f32 / pattern_blocks;
+                    if percent >= search_behavior.threshold {
+                        matches.push(Match {
+                            x: x as u16,
+                            y: y as u16,
+                            z: z as u16,
+                            percent,
+                        });
                     }
                 }
-                if matching >= matching_needed {
-                    matches.push(Match {
-                        x: x as u16,
-                        y: y as u16,
-                        z: z as u16,
-                        percent: matching as f32 / pattern_blocks,
-                    });
-                }
             }
         }
+    }*/
+
+    let mut result = Vec::<i32>::with_capacity(schem_width * schem_height * schem_length);
+    result.resize(schem_width * schem_height * schem_length, 0);
+
+    unsafe {
+        is_matching_all(
+            schem_data,
+            pattern_data,
+            schem_width as i32,
+            schem_height as i32,
+            schem_length as i32,
+            pattern_width as i32,
+            pattern_height as i32,
+            pattern_length as i32,
+            result.as_mut_ptr()
+        );
     }
 
+    result.into_iter().enumerate().filter(|(_, matching_count)| *matching_count >= i_pattern_blocks - skip_amount).for_each(|(i, matching_count)| {
+        let percent = matching_count as f32 / pattern_blocks;
+        let x = i % schem_width;
+        let y = (i / schem_width) % schem_height;
+        let z = i / (schem_width * schem_height);
+        matches.push(Match {
+            x: x as u16,
+            y: y as u16,
+            z: z as u16,
+            percent,
+        });
+
+    });
+
     return matches;
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
+#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)]
 pub struct Match {
     pub x: u16,
     pub y: u16,
@@ -113,17 +184,6 @@ pub struct Match {
     pub percent: f32,
 }
 
-impl Default for Match {
-    fn default() -> Self {
-        Self {
-            x: 0,
-            y: 0,
-            z: 0,
-            percent: 0.0,
-        }
-    }
-}
-
 #[inline]
 pub fn normalize_data(data: &str, ignore_data: bool) -> &str {
     if ignore_data {
@@ -209,7 +269,6 @@ mod tests {
             threshold: 0.9
         });
 
-        println!("{:?}", matches);
         assert_eq!(matches.len(), 1);
         assert_eq!(matches[0].x, 1);
         assert_eq!(matches[0].y, 0);
@@ -231,7 +290,6 @@ mod tests {
             threshold: 0.9
         });
 
-        println!("{:?}", matches);
         assert_eq!(matches.len(), 1);
     }
 }

schemsearch-lib/src/pattern_mapper.rs

@@ -64,10 +64,10 @@ pub fn strip_data(schem: &SpongeSchematic) -> SpongeSchematic {
 }
 
 pub fn match_palette_adapt(schem: &SpongeSchematic, matching_palette: &HashMap<String, i32>, ignore_data: bool) -> Vec<i32> {
-    let mut data: Vec<i32> = Vec::new();
+    let mut data = Vec::with_capacity(schem.block_data.len());
     let reverse_palette = create_reverse_palette(schem);
 
-    for x in schem.block_data.iter() {
+    for x in schem.block_data.as_slice().iter() {
         let blockname = reverse_palette[*x as usize];
         let blockname = if ignore_data { normalize_data(blockname, ignore_data) } else { blockname };
         let block_id = match matching_palette.get(&*blockname) {

schemsearch-sql/Cargo.toml

@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sqlx = { version = "0.6", features = [ "runtime-async-std-native-tls" , "mysql" ] }
+sqlx = { version = "0.7", features = [ "runtime-async-std-native-tls" , "mysql" ] }
 
 schemsearch-lib = { path = "../schemsearch-lib" }
 schemsearch-files = { path = "../schemsearch-files" }