From 47bbf25ac7a4b1f8073ac237a88224318980713d Mon Sep 17 00:00:00 2001 From: Chaoscaot Date: Wed, 9 Apr 2025 13:41:50 +0200 Subject: [PATCH] Remove deprecated crates and introduce OpenCL matcher integration --- Cargo.toml | 6 +- schemsearch-cli/Cargo.toml | 1 + schemsearch-cli/src/json_output.rs | 2 +- schemsearch-cli/src/main.rs | 2 +- schemsearch-cli/src/sinks.rs | 2 +- schemsearch-common/Cargo.toml | 7 ++ schemsearch-common/src/lib.rs | 20 ++++++ schemsearch-faster/Cargo.toml | 12 ---- schemsearch-faster/src/lib.rs | 73 -------------------- schemsearch-java/Cargo.toml | 16 ----- schemsearch-java/src/lib.rs | 54 --------------- schemsearch-lib/Cargo.toml | 2 + schemsearch-lib/src/lib.rs | 21 +----- schemsearch-lib/src/search.rs | 16 +++-- schemsearch-ocl-matcher/Cargo.toml | 9 +++ schemsearch-ocl-matcher/src/kernel.cl | 51 ++++++++++++++ schemsearch-ocl-matcher/src/lib.rs | 95 +++++++++++++++++++++++++++ 17 files changed, 202 insertions(+), 187 deletions(-) create mode 100644 schemsearch-common/Cargo.toml create mode 100644 schemsearch-common/src/lib.rs delete mode 100644 schemsearch-faster/Cargo.toml delete mode 100644 schemsearch-faster/src/lib.rs delete mode 100644 schemsearch-java/Cargo.toml delete mode 100644 schemsearch-java/src/lib.rs create mode 100644 schemsearch-ocl-matcher/Cargo.toml create mode 100644 schemsearch-ocl-matcher/src/kernel.cl create mode 100644 schemsearch-ocl-matcher/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 079b6d2..b832d19 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,11 +2,9 @@ members = [ "schemsearch-cli", "schemsearch-lib", - "schemsearch-faster", "schemsearch-files", "schemsearch-sql", - "schemsearch-java" -] + "schemsearch-ocl-matcher"] resolver = "2" [profile.small] @@ -17,4 +15,4 @@ opt-level = "z" codegen-units = 1 [profile.release] -lto = true \ No newline at end of file +lto = true diff --git a/schemsearch-cli/Cargo.toml b/schemsearch-cli/Cargo.toml index a28a424..6264a8c 100755 --- a/schemsearch-cli/Cargo.toml +++ b/schemsearch-cli/Cargo.toml @@ -7,6 +7,7 @@ license = "AGPL-3.0-or-later" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +schemsearch-common = { path = "../schemsearch-common" } schemsearch-lib = { path = "../schemsearch-lib" } schemsearch-files = { path = "../schemsearch-files" } schemsearch-sql = { path = "../schemsearch-sql", optional = true } diff --git a/schemsearch-cli/src/json_output.rs b/schemsearch-cli/src/json_output.rs index a3d4776..3fa7024 100644 --- a/schemsearch-cli/src/json_output.rs +++ b/schemsearch-cli/src/json_output.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use schemsearch_lib::{Match, SearchBehavior}; +use schemsearch_common::{Match, SearchBehavior}; #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "event")] diff --git a/schemsearch-cli/src/main.rs b/schemsearch-cli/src/main.rs index 3e62897..21e4eef 100755 --- a/schemsearch-cli/src/main.rs +++ b/schemsearch-cli/src/main.rs @@ -26,7 +26,6 @@ use clap::{command, Arg, ArgAction, ValueHint}; use std::path::PathBuf; use std::str::FromStr; use clap::error::ErrorKind; -use schemsearch_lib::{Match, SearchBehavior}; use crate::types::{PathSchematicSupplier, SchematicSupplier, SchematicSupplierType}; #[cfg(feature = "sql")] use futures::executor::block_on; @@ -39,6 +38,7 @@ use schemsearch_sql::load_all_schematics; #[cfg(feature = "sql")] use crate::types::SqlSchematicSupplier; use indicatif::*; +use schemsearch_common::{Match, SearchBehavior}; use schemsearch_files::SpongeSchematic; use crate::sinks::{OutputFormat, OutputSink}; use crate::stderr::MaschineStdErr; diff --git a/schemsearch-cli/src/sinks.rs b/schemsearch-cli/src/sinks.rs index ac28a86..e6b52ac 100755 --- a/schemsearch-cli/src/sinks.rs +++ b/schemsearch-cli/src/sinks.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use std::io::Write; use std::time::Duration; use indicatif::HumanDuration; -use schemsearch_lib::{Match, SearchBehavior}; +use schemsearch_common::{Match, SearchBehavior}; use crate::json_output::{EndEvent, FoundEvent, InitEvent, JsonEvent}; #[derive(Debug, Clone)] diff --git a/schemsearch-common/Cargo.toml b/schemsearch-common/Cargo.toml new file mode 100644 index 0000000..f02e4e0 --- /dev/null +++ b/schemsearch-common/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "schemsearch-common" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0.160", features = ["derive"] } \ No newline at end of file diff --git a/schemsearch-common/src/lib.rs b/schemsearch-common/src/lib.rs new file mode 100644 index 0000000..11d5b82 --- /dev/null +++ b/schemsearch-common/src/lib.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +pub struct SearchBehavior { + pub ignore_block_data: bool, + pub ignore_block_entities: bool, + pub ignore_air: bool, + pub air_as_any: bool, + pub ignore_entities: bool, + pub threshold: f32, + pub invalid_nbt: bool, +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)] +pub struct Match { + pub x: u16, + pub y: u16, + pub z: u16, + pub percent: f32, +} diff --git a/schemsearch-faster/Cargo.toml b/schemsearch-faster/Cargo.toml deleted file mode 100644 index 4808577..0000000 --- a/schemsearch-faster/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "schemsearch_faster" -version = "0.1.3" -edition = "2021" -license = "AGPL-3.0-or-later" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -schemsearch-lib = { path = "../schemsearch-lib" } -schemsearch-files = { path = "../schemsearch-files" } -hematite-nbt = "0.5.2" diff --git a/schemsearch-faster/src/lib.rs b/schemsearch-faster/src/lib.rs deleted file mode 100644 index a6ffa93..0000000 --- a/schemsearch-faster/src/lib.rs +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2023 Chaoscaot - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -use nbt::Map; -use schemsearch_files::SpongeV2Schematic; - -pub fn convert_to_search_space(schem: &SpongeV2Schematic, palette: &Vec) -> Vec> { - let mut data: Vec> = Vec::with_capacity(palette.len()); - let block_data = &schem.block_data; - for name in palette { - let mut output: Vec = Vec::with_capacity(block_data.len()); - for block in block_data.iter() { - if schem.palette.get(name).unwrap_or(&-1) == block { - output.push(1); - } else { - output.push(0); - } - } - data.push(output); - } - data -} - -pub fn unwrap_palette(palette: &Map) -> Vec { - let mut output: Vec = Vec::with_capacity(palette.len()); - (0..palette.len()).for_each(|_| output.push(String::new())); - for (key, id) in palette.iter() { - output[*id as usize] = key.clone(); - } - output -} - -#[allow(unused_imports)] -#[cfg(test)] -mod tests { - use std::path::{Path, PathBuf}; - use schemsearch_files::SpongeV2Schematic; - use crate::{convert_to_search_space, unwrap_palette}; - - //#[test] - pub fn test() { - let schematic = SpongeV2Schematic::load(&PathBuf::from("../tests/Pattern.schem")).unwrap(); - dbg!(convert_to_search_space(&schematic, &unwrap_palette(&schematic.palette))); - } - - //#[test] - pub fn test_2() { - let schematic = SpongeV2Schematic::load(&PathBuf::from("../tests/Pattern.schem")).unwrap(); - let schematic2 = SpongeV2Schematic::load(&PathBuf::from("../tests/Random.schem")).unwrap(); - println!("{:?}", convert_to_search_space(&schematic2, &unwrap_palette(&schematic.palette))); - } - - //#[test] - pub fn test_big() { - let schematic = SpongeV2Schematic::load(&PathBuf::from("../tests/endstone.schem")).unwrap(); - let schematic2 = SpongeV2Schematic::load(&PathBuf::from("../tests/simple.schem")).unwrap(); - let _ = convert_to_search_space(&schematic2, &unwrap_palette(&schematic.palette)); - } -} \ No newline at end of file diff --git a/schemsearch-java/Cargo.toml b/schemsearch-java/Cargo.toml deleted file mode 100644 index bbfbd8e..0000000 --- a/schemsearch-java/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "schemsearch-java" -version = "0.1.3" -edition = "2021" -license = "AGPL-3.0-or-later" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -crate_type = ["cdylib"] - -[dependencies] -jni = "0.21.0" - -schemsearch-lib = { path = "../schemsearch-lib" } -schemsearch-files = { path = "../schemsearch-files" } \ No newline at end of file diff --git a/schemsearch-java/src/lib.rs b/schemsearch-java/src/lib.rs deleted file mode 100644 index 35c2b51..0000000 --- a/schemsearch-java/src/lib.rs +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2023 Chaoscaot - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -use std::path::PathBuf; -use jni::JNIEnv; - -use jni::objects::{JClass, JString}; - -use jni::sys::jstring; -use schemsearch_files::SpongeV2Schematic; -use schemsearch_lib::{search, SearchBehavior}; - -#[no_mangle] -#[allow(unused_variables)] -pub extern "system" fn Java_SchemSearch_search<'local>(mut env: JNIEnv<'local>, - class: JClass<'local>, - schematic_path: JString<'local>, - pattern_path: JString<'local>) -> jstring { - let schematic_path: String = env.get_string(&schematic_path).expect("Couldn't get java string!").into(); - let pattern_path: String = env.get_string(&pattern_path).expect("Couldn't get java string!").into(); - let schematic = SpongeV2Schematic::load(&PathBuf::from(&schematic_path)).unwrap(); - let pattern = SpongeV2Schematic::load(&PathBuf::from(&pattern_path)).unwrap(); - - let matches = search(schematic, &pattern, SearchBehavior { - ignore_block_data: true, - ignore_block_entities: true, - ignore_entities: true, - ignore_air: false, - air_as_any: false, - threshold: 0.0, - }); - - let mut result = String::new(); - for m in matches { - result.push_str(&format!("{}, {}, {}, {};", m.x, m.y, m.z, m.percent)); - } - result.remove(result.len() - 1); - let output = env.new_string(result).expect("Couldn't create java string!"); - output.into_raw() -} \ No newline at end of file diff --git a/schemsearch-lib/Cargo.toml b/schemsearch-lib/Cargo.toml index 0ff6766..2eb56cf 100755 --- a/schemsearch-lib/Cargo.toml +++ b/schemsearch-lib/Cargo.toml @@ -9,6 +9,8 @@ license = "AGPL-3.0-or-later" [dependencies] serde = { version = "1.0.160", features = ["derive"] } schemsearch-files = { path = "../schemsearch-files" } +schemsearch-common = { path = "../schemsearch-common" } +schemsearch-ocl-matcher = { path = "../schemsearch-ocl-matcher" } named-binary-tag = "0.6" libmath = "0.2.1" lazy_static = "1.4.0" diff --git a/schemsearch-lib/src/lib.rs b/schemsearch-lib/src/lib.rs index b2f7206..5003922 100755 --- a/schemsearch-lib/src/lib.rs +++ b/schemsearch-lib/src/lib.rs @@ -19,26 +19,7 @@ pub mod pattern_mapper; pub mod search; pub mod nbt_search; -use serde::{Serialize, Deserialize}; - -#[derive(Debug, Clone, Copy, Deserialize, Serialize)] -pub struct SearchBehavior { - pub ignore_block_data: bool, - pub ignore_block_entities: bool, - pub ignore_air: bool, - pub air_as_any: bool, - pub ignore_entities: bool, - pub threshold: f32, - pub invalid_nbt: bool, -} - -#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)] -pub struct Match { - pub x: u16, - pub y: u16, - pub z: u16, - pub percent: f32, -} +use schemsearch_common::SearchBehavior; #[inline] pub fn normalize_data(data: &str, ignore_data: bool) -> &str { diff --git a/schemsearch-lib/src/search.rs b/schemsearch-lib/src/search.rs index e24fa0f..dca7391 100755 --- a/schemsearch-lib/src/search.rs +++ b/schemsearch-lib/src/search.rs @@ -1,6 +1,8 @@ use math::round::ceil; +use schemsearch_common::Match; use schemsearch_files::SpongeSchematic; -use crate::{Match, SearchBehavior}; +use schemsearch_ocl_matcher::{ocl_available, ocl_search}; +use crate::{SearchBehavior}; use crate::pattern_mapper::{match_palette, match_palette_adapt}; pub fn search( @@ -20,16 +22,12 @@ pub fn search( let mut matches: Vec = Vec::with_capacity(4); - let pattern_data = pattern_schem.block_data.as_ptr(); - let schem_data = if search_behavior.ignore_block_data { match_palette_adapt(&schem, &pattern_schem.palette, search_behavior.ignore_block_data) } else { schem.block_data }; - let schem_data = schem_data.as_ptr(); - let air_id = if search_behavior.ignore_air || search_behavior.air_as_any { pattern_schem.palette.get("minecraft:air").unwrap_or(&-1) } else { &-1}; let pattern_blocks = pattern_schem.block_data.len() as f32; @@ -42,6 +40,14 @@ pub fn search( let schem_width = schem.width as usize; let schem_height = schem.height as usize; let schem_length = schem.length as usize; + + if ocl_available() { + return ocl_search(schem_data.as_slice(), [schem_width, schem_height, schem_length], pattern_schem.block_data.as_slice(), [pattern_width, pattern_height, pattern_length], *air_id, search_behavior).unwrap() + } + + let schem_data = schem_data.as_ptr(); + + let pattern_data = pattern_schem.block_data.as_ptr(); let skip_amount = ceil((pattern_blocks * (1.0 - search_behavior.threshold)) as f64, 0) as i32; diff --git a/schemsearch-ocl-matcher/Cargo.toml b/schemsearch-ocl-matcher/Cargo.toml new file mode 100644 index 0000000..6f7e403 --- /dev/null +++ b/schemsearch-ocl-matcher/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "schemsearch-ocl-matcher" +version = "0.1.0" +edition = "2021" + +[dependencies] +schemsearch-common = { path = "../schemsearch-common" } +ocl = "0.19.7" +libmath = "0.2.1" \ No newline at end of file diff --git a/schemsearch-ocl-matcher/src/kernel.cl b/schemsearch-ocl-matcher/src/kernel.cl new file mode 100644 index 0000000..202f22d --- /dev/null +++ b/schemsearch-ocl-matcher/src/kernel.cl @@ -0,0 +1,51 @@ +__kernel void add(__global int* result, + __global uint* schem, + __global uint* pattern, + const int p_width, + const int p_height, + const int p_depth, + const uint air_id, + const int ignore_air, + const int air_as_any, + const int skipamount) { + int x = get_global_id(0); + int y = get_global_id(1); + int z = get_global_id(2); + + int width = get_global_size(0); + int height = get_global_size(1); + int depth = get_global_size(2); + + if (x > width - p_width || y > height - p_height || z > depth - p_depth) { + return; + } + + int wrong_blocks = 0; + for (int py = 0; py < p_height; py++) { + for (int pz = 0; pz < p_depth; pz++) { + for (int px = 0; px < p_width; px++) { + int s_idx = (x + px) + width * ((z + pz) + (y + py) * depth); + int p_idx = px + p_width * (pz + py * p_depth); + + uint schem_block = schem[s_idx]; + uint pattern_block = pattern[p_idx]; + + if ((ignore_air && schem_block != air_id) || (air_as_any && pattern_block != air_id)) { + continue; + } + + if (schem_block != pattern_block) { + wrong_blocks++; + if (wrong_blocks > skipamount) { + int idx = x + z * width + y * width * depth; + result[idx] = wrong_blocks; + return; + } + } + } + } + } + + int idx = x + z * width + y * width * depth; + result[idx] = wrong_blocks; +} diff --git a/schemsearch-ocl-matcher/src/lib.rs b/schemsearch-ocl-matcher/src/lib.rs new file mode 100644 index 0000000..4c860a6 --- /dev/null +++ b/schemsearch-ocl-matcher/src/lib.rs @@ -0,0 +1,95 @@ +use ocl::{Buffer, MemFlags, ProQue, Platform}; +use ocl::SpatialDims::Three; +use schemsearch_common::{Match, SearchBehavior}; +use math::round::ceil; + +const KERNEL: &str = include_str!("kernel.cl"); + +pub fn ocl_available() -> bool { + !Platform::list().is_empty() +} + +pub fn ocl_search( + schem: &[i32], + schem_size: [usize; 3], + pattern: &[i32], + pattern_size: [usize; 3], + air_id: i32, + search_behavior: SearchBehavior, +) -> Result, String> { + search_ocl(schem, schem_size, pattern, pattern_size, air_id, search_behavior).map_err(|e| e.to_string()) +} + +fn search_ocl( + schem: &[i32], + schem_size: [usize; 3], + pattern: &[i32], + pattern_size: [usize; 3], + air_id: i32, + search_behavior: SearchBehavior, +) -> ocl::Result> { + let pattern_width = pattern_size[0]; + let pattern_height = pattern_size[1]; + let pattern_length = pattern_size[2]; + + let schem_width = schem_size[0]; + let schem_height = schem_size[1]; + let schem_length = schem_size[2]; + + let pattern_blocks = (pattern_width * pattern_height * pattern_length) as f32; + + let skip_amount = ceil((pattern_blocks * (1.0 - search_behavior.threshold)) as f64, 0) as i32; + + let pro_que = ProQue::builder() + .src(KERNEL) + .dims(Three(schem_width, schem_height, schem_length)) + .build()?; + + let buffer = Buffer::builder() + .queue(pro_que.queue().clone()) + .flags(MemFlags::new().read_write()) + .fill_val(-1) + .len(schem.len()) + .build()?; + + let schem_buffer = create_schem_buffer(schem, &pro_que)?; + + let pattern_buffer = create_schem_buffer(pattern, &pro_que)?; + + let kernel = pro_que.kernel_builder("add") + .arg(&buffer) + .arg(&schem_buffer) + .arg(&pattern_buffer) + .arg(pattern_width as i32) + .arg(pattern_height as i32) + .arg(pattern_length as i32) + .arg(air_id) // air_id + .arg(search_behavior.ignore_air as u32) // ignore_air + .arg(search_behavior.air_as_any as u32) // air_as_any + .arg(skip_amount) + .build()?; + + unsafe { kernel.enq()?; } + + let mut vec = vec![0i32; buffer.len()]; + buffer.read(&mut vec).enq()?; + + Ok(vec.into_iter().enumerate().filter(|(_, v)| *v < skip_amount && *v != -1).map(|(i, v)| { + Match { + x: (i % schem_width) as u16, + y: ((i / (schem_width * schem_length)) % schem_height) as u16, + z: ((i / schem_width) % schem_length) as u16, + + percent: (pattern_blocks - v as f32) / pattern_blocks, + } + }).collect()) +} + +fn create_schem_buffer(pattern: &[i32], pro_que: &ProQue) -> ocl::Result> { + Buffer::builder() + .queue(pro_que.queue().clone()) + .flags(MemFlags::new().read_only()) + .len(pattern.len()) + .copy_host_slice(pattern) + .build() +} \ No newline at end of file