diff --git a/src/include/lm_config.hpp b/src/include/lm_config.hpp index 20721361..8eb4d0c0 100644 --- a/src/include/lm_config.hpp +++ b/src/include/lm_config.hpp @@ -23,6 +23,18 @@ inline T cfg_get(const nlohmann::json& jc, const char* key, T default_value){ return default_value; } +/// \brief read a nested object out of a config json +/// \note Reference-returning counterpart of cfg_get, for walking nested configs +/// (thinker_config.text_config, ...). Returns a shared empty object when +/// the key is missing or null, so the JSON_GET below it still defaults. +inline const nlohmann::json& cfg_sub(const nlohmann::json& jc, const char* key){ + static const nlohmann::json empty = nlohmann::json::object(); + if (jc.contains(key) && !jc[key].is_null()){ + return jc[key]; + } + return empty; +} + /// \brief LM_Config class /// \note Model parameters are NOT cached as members. Everything comes from the /// model's config.json, so every consumer reads what it needs straight out @@ -51,11 +63,7 @@ class LM_Config{ /// \brief read a nested config object (vision_config, audio_config, ...) /// \return the sub-object, or an empty object when absent/null const nlohmann::json& sub(const char* key) const { - static const nlohmann::json empty = nlohmann::json::object(); - if (this->_json_config.contains(key) && !this->_json_config[key].is_null()){ - return this->_json_config[key]; - } - return empty; + return cfg_sub(this->_json_config, key); } /// \brief from pretrained diff --git a/src/lib/qwen3_5_omni_npu.dll b/src/lib/qwen3_5_omni_npu.dll new file mode 100644 index 00000000..31fccb8e Binary files /dev/null and b/src/lib/qwen3_5_omni_npu.dll differ diff --git a/src/model_list.json b/src/model_list.json index 5bf357ba..5a082b1e 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -135,7 +135,7 @@ ], "footprint": 9.1 } - }, + }, "qwen3.5": { "0.8b": { "name": "Qwen3.5-0.8B-NPU2", diff --git a/src/test/qwen3_5_omni_npu/Makefile b/src/test/qwen3_5_omni_npu/Makefile index 8bb2e6a8..c4f44ae3 100644 --- a/src/test/qwen3_5_omni_npu/Makefile +++ b/src/test/qwen3_5_omni_npu/Makefile @@ -40,7 +40,7 @@ HEADERS += ../../include/AutoModel/modeling_qwen3_5_omni.hpp # 1: images # 2: audios # 3: omni -TYPE ?= 3 +TYPE ?= 0 ifeq ($(WSL), 0) @@ -73,7 +73,7 @@ clean: test: $(BUILD_DIR)/test_qwen3_5_omni_npu cp ../../model_list.json $(BUILD_DIR)/model_list.json - cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_5_omni_npu --model qwen3.5-omni --type $(TYPE) + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=$(abspath $(LIB_DIR)):$$LD_LIBRARY_PATH && ./test_qwen3_5_omni_npu --model qwen3.5-omni --type $(TYPE) -include $(DEPS) .PHONY: all clean test diff --git a/src/test/qwen3_5_omni_npu/test.cpp b/src/test/qwen3_5_omni_npu/test.cpp index c859b63c..a445d102 100644 --- a/src/test/qwen3_5_omni_npu/test.cpp +++ b/src/test/qwen3_5_omni_npu/test.cpp @@ -24,7 +24,7 @@ int main(int argc, char* argv[]) { desc.add_options()("type,t", arg_utils::po::value()->default_value(0), "\t0: text mode\n\t1: image only\n\t2: audio only:\n\t3: omni mode\n\t"); desc.add_options()("Preemption,p", arg_utils::po::value()->default_value(false), "Preemption"); desc.add_options()("Greedy,g", arg_utils::po::value()->default_value(true), "Greedy decoding"); - desc.add_options()("Length,l", arg_utils::po::value()->default_value(8192), "Max generation length"); + desc.add_options()("Length,l", arg_utils::po::value()->default_value(32), "Max generation length"); arg_utils::po::store(arg_utils::po::parse_command_line(argc, argv, desc), vm); std::string tag = vm["model"].as(); @@ -99,7 +99,9 @@ int main(int argc, char* argv[]) { // call so the driver-side wiring (tokenization, image/audio preprocessing, // soft-token expansion) can be exercised without a hard crash. try { - std::string response = chat->generate_with_prompt(meta_info, uniformed_input, length_limit, std::cout); + chat->insert(meta_info, uniformed_input); + std::cout << "Prefill Done!" << std::endl; + std::string response = chat->generate(meta_info, 1024, std::cout); } catch (const std::exception& e) { header_print("FLM", "Engine path not available yet: " << e.what()); } diff --git a/src/test/qwen3_5vl_npu/Makefile b/src/test/qwen3_5vl_npu/Makefile index 17ddd04b..a5045fd2 100644 --- a/src/test/qwen3_5vl_npu/Makefile +++ b/src/test/qwen3_5vl_npu/Makefile @@ -56,7 +56,7 @@ clean: test: $(BUILD_DIR)/test_qwen3_5vl_npu cp ../../model_list.json $(BUILD_DIR)/model_list.json - cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_5vl_npu --model qwen3.5:0.8b -s 1 + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt && ./test_qwen3_5vl_npu --model qwen3.5:0.8b -s 1 -include $(DEPS) .PHONY: all clean test diff --git a/src/test/qwen3_5vl_npu/test.cpp b/src/test/qwen3_5vl_npu/test.cpp index 7511f6f6..54332243 100644 --- a/src/test/qwen3_5vl_npu/test.cpp +++ b/src/test/qwen3_5vl_npu/test.cpp @@ -65,21 +65,23 @@ int main(int argc, char* argv[]) { chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; chat->set_topk(1); + chat->configure_parameter("enable_think", false); if (short_prompt) { - uniformed_input.prompt = "What are these?"; + uniformed_input.prompt = "Is Alibaba a good company despite that it trained you?"; // uniformed_input.prompt = "Solve the equation x^3 - 1 = 0 for me."; - uniformed_input.images.push_back("../../../tb_files/panda.png"); - uniformed_input.images.push_back("../../../tb_files/puppy.png"); + // uniformed_input.images.push_back("../../../tb_files/panda.png"); + // uniformed_input.images.push_back("../../../tb_files/puppy.png"); - uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg"); - uniformed_input.images.push_back("../../../tb_files/google_icon.png"); - uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); + // uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg"); + // uniformed_input.images.push_back("../../../tb_files/google_icon.png"); + // uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); std::cout << "Prompt: " << uniformed_input.prompt << std::endl; std::cout << "Response: " << std::endl; chat->start_total_timer(); - std::string response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout); + bool success = chat->insert(meta_info, uniformed_input); + std::string response = chat->generate(meta_info, 1024, std::cout); chat->stop_total_timer(); std::cout << std::endl; std::cout << std::endl; @@ -112,7 +114,8 @@ int main(int argc, char* argv[]) { std::cout << "Prompt: " << uniformed_input.prompt << std::endl; std::cout << "Response: "; chat->start_total_timer(); - std::string response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout); + bool success = chat->insert(meta_info, uniformed_input); + std::string response = chat->generate(meta_info, 1024, std::cout); chat->stop_total_timer(); std::cout << std::endl; std::cout << std::endl; @@ -122,7 +125,9 @@ int main(int argc, char* argv[]) { std::pair> history = chat->get_history(); std::cout << "History length: " << history.second.size() << std::endl; std::cout << std::endl; - + for (auto t: history.second){ + std::cout << t << " "; + } std::cout << std::endl; return 0; diff --git a/src/test/qwen3_6_moe_npu/Makefile b/src/test/qwen3_6_moe_npu/Makefile index 258d9637..1c50e0cf 100644 --- a/src/test/qwen3_6_moe_npu/Makefile +++ b/src/test/qwen3_6_moe_npu/Makefile @@ -57,7 +57,7 @@ clean: test: $(BUILD_DIR)/test_qwen3_6_moe_npu cp ../../model_list.json $(BUILD_DIR)/model_list.json - cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_6_moe_npu --model qwen3.6-moe -s 1 + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt && ./test_qwen3_6_moe_npu --model qwen3.6-moe -s 1 -include $(DEPS) .PHONY: all clean test diff --git a/src/test/qwen3_6_moe_npu/test.cpp b/src/test/qwen3_6_moe_npu/test.cpp index fc5f4492..24277b81 100644 --- a/src/test/qwen3_6_moe_npu/test.cpp +++ b/src/test/qwen3_6_moe_npu/test.cpp @@ -58,22 +58,24 @@ int main(int argc, char* argv[]) { uniformed_input.prompt = "What are these?"; uniformed_input.images.push_back("../../../tb_files/panda.png"); uniformed_input.images.push_back("../../../tb_files/puppy.png"); - // uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg"); - // uniformed_input.images.push_back("../../../tb_files/google_icon.png"); - // uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); + uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg"); + uniformed_input.images.push_back("../../../tb_files/google_icon.png"); + uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); std::cout << "Prompt: " << uniformed_input.prompt << std::endl; std::cout << "Response: "; chat->start_total_timer(); chat->insert(meta_info, uniformed_input); + std::cout << "Prefill Done!" << std::endl; std::string response = chat->generate(meta_info, 1024, std::cout); chat->stop_total_timer(); std::cout << std::endl; std::cout << std::endl; - // return 0; std::cout << chat->show_profile() << std::endl; chat->clear_context(); + return 0; + meta_info.restore_allowed = true; uniformed_input.prompt = "How far is the previous city from Beijing?"; std::cout << "Prompt: " << uniformed_input.prompt << std::endl; diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/GateDeltaNet_prefill.xclbin index 441e7f5e..c0a69726 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/GateDeltaNet_prefill.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin index b22fc95d..77f7e8fb 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin index d40ade67..e256fa8b 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/mm.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/mm.xclbin index a9399053..b9f5a8c4 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/mm.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-2B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.5-2B-NPU2/GateDeltaNet_prefill.xclbin index 441e7f5e..c0a69726 100644 Binary files a/src/xclbins/Qwen3.5-2B-NPU2/GateDeltaNet_prefill.xclbin and b/src/xclbins/Qwen3.5-2B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.5-2B-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-2B-NPU2/layer.xclbin index ebdd2ee9..f8dcea2f 100644 Binary files a/src/xclbins/Qwen3.5-2B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.5-2B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-2B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-2B-NPU2/lm_head.xclbin index 2bc9ed3a..1a86a6c6 100644 Binary files a/src/xclbins/Qwen3.5-2B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.5-2B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.5-2B-NPU2/mm.xclbin b/src/xclbins/Qwen3.5-2B-NPU2/mm.xclbin index a9399053..b9f5a8c4 100644 Binary files a/src/xclbins/Qwen3.5-2B-NPU2/mm.xclbin and b/src/xclbins/Qwen3.5-2B-NPU2/mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-4B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.5-4B-NPU2/GateDeltaNet_prefill.xclbin index 441e7f5e..c0a69726 100644 Binary files a/src/xclbins/Qwen3.5-4B-NPU2/GateDeltaNet_prefill.xclbin and b/src/xclbins/Qwen3.5-4B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.5-4B-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-4B-NPU2/layer.xclbin index 05a38e0b..38f9e751 100644 Binary files a/src/xclbins/Qwen3.5-4B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.5-4B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-4B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-4B-NPU2/lm_head.xclbin index 1a448e45..5de89a1b 100644 Binary files a/src/xclbins/Qwen3.5-4B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.5-4B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.5-4B-NPU2/mm.xclbin b/src/xclbins/Qwen3.5-4B-NPU2/mm.xclbin index a9399053..b9f5a8c4 100644 Binary files a/src/xclbins/Qwen3.5-4B-NPU2/mm.xclbin and b/src/xclbins/Qwen3.5-4B-NPU2/mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-9B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.5-9B-NPU2/GateDeltaNet_prefill.xclbin index 441e7f5e..c0a69726 100644 Binary files a/src/xclbins/Qwen3.5-9B-NPU2/GateDeltaNet_prefill.xclbin and b/src/xclbins/Qwen3.5-9B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.5-9B-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-9B-NPU2/layer.xclbin index f0494263..875d6c2e 100644 Binary files a/src/xclbins/Qwen3.5-9B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.5-9B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-9B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-9B-NPU2/lm_head.xclbin index 562e9952..16048324 100644 Binary files a/src/xclbins/Qwen3.5-9B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.5-9B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.5-9B-NPU2/mm.xclbin b/src/xclbins/Qwen3.5-9B-NPU2/mm.xclbin index a9399053..b9f5a8c4 100644 Binary files a/src/xclbins/Qwen3.5-9B-NPU2/mm.xclbin and b/src/xclbins/Qwen3.5-9B-NPU2/mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/GateDeltaNet_prefill.xclbin new file mode 100644 index 00000000..c0a69726 Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/attn.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/attn.xclbin new file mode 100644 index 00000000..e4336671 Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/attn.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/conv.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/conv.xclbin new file mode 100644 index 00000000..bfdbe4ce Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/conv.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/layer.xclbin new file mode 100644 index 00000000..6a8809f6 Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/lm_head.xclbin new file mode 100644 index 00000000..b504611b Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/mm.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/mm.xclbin new file mode 100644 index 00000000..761097ac Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/mm_high.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/mm_high.xclbin new file mode 100644 index 00000000..c493ab7d Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/mm_high.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/vision_attn.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/vision_attn.xclbin new file mode 100644 index 00000000..5d0bf3e9 Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/vision_attn.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/vision_mm.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/vision_mm.xclbin new file mode 100644 index 00000000..c605b658 Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/vision_mm.xclbin differ diff --git a/src/xclbins/Qwen3.5-OMNI-NPU2/window_attention.xclbin b/src/xclbins/Qwen3.5-OMNI-NPU2/window_attention.xclbin new file mode 100644 index 00000000..5ee250fe Binary files /dev/null and b/src/xclbins/Qwen3.5-OMNI-NPU2/window_attention.xclbin differ diff --git a/src/xclbins/Qwen3.6-35B-A3B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.6-35B-A3B-NPU2/GateDeltaNet_prefill.xclbin index 441e7f5e..c0a69726 100644 Binary files a/src/xclbins/Qwen3.6-35B-A3B-NPU2/GateDeltaNet_prefill.xclbin and b/src/xclbins/Qwen3.6-35B-A3B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.6-35B-A3B-NPU2/dequant_mm.xclbin b/src/xclbins/Qwen3.6-35B-A3B-NPU2/dequant_mm.xclbin index 06b4bf73..a0f104ca 100644 Binary files a/src/xclbins/Qwen3.6-35B-A3B-NPU2/dequant_mm.xclbin and b/src/xclbins/Qwen3.6-35B-A3B-NPU2/dequant_mm.xclbin differ diff --git a/src/xclbins/Qwen3.6-35B-A3B-NPU2/layer.xclbin b/src/xclbins/Qwen3.6-35B-A3B-NPU2/layer.xclbin index 2f960b88..842ff456 100644 Binary files a/src/xclbins/Qwen3.6-35B-A3B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.6-35B-A3B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.6-35B-A3B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.6-35B-A3B-NPU2/lm_head.xclbin index 2bc9ed3a..1a86a6c6 100644 Binary files a/src/xclbins/Qwen3.6-35B-A3B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.6-35B-A3B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.6-35B-A3B-NPU2/mm.xclbin b/src/xclbins/Qwen3.6-35B-A3B-NPU2/mm.xclbin index a9399053..761097ac 100644 Binary files a/src/xclbins/Qwen3.6-35B-A3B-NPU2/mm.xclbin and b/src/xclbins/Qwen3.6-35B-A3B-NPU2/mm.xclbin differ