FLM quant for Qwen3.5 family

src/include/lm_config.hpp

@@ -23,6 +23,18 @@ inline T cfg_get(const nlohmann::json& jc, const char* key, T default_value){
     return default_value;
 }
 
+/// \brief read a nested object out of a config json
+/// \note Reference-returning counterpart of cfg_get, for walking nested configs
+///       (thinker_config.text_config, ...). Returns a shared empty object when
+///       the key is missing or null, so the JSON_GET below it still defaults.
+inline const nlohmann::json& cfg_sub(const nlohmann::json& jc, const char* key){
+    static const nlohmann::json empty = nlohmann::json::object();
+    if (jc.contains(key) && !jc[key].is_null()){
+        return jc[key];
+    }
+    return empty;
+}
+
 /// \brief LM_Config class
 /// \note Model parameters are NOT cached as members. Everything comes from the
 ///       model's config.json, so every consumer reads what it needs straight out
@@ -51,11 +63,7 @@ class LM_Config{
         /// \brief read a nested config object (vision_config, audio_config, ...)
         /// \return the sub-object, or an empty object when absent/null
         const nlohmann::json& sub(const char* key) const {
-            static const nlohmann::json empty = nlohmann::json::object();
-            if (this->_json_config.contains(key) && !this->_json_config[key].is_null()){
-                return this->_json_config[key];
-            }
-            return empty;
+            return cfg_sub(this->_json_config, key);
         }
 
         /// \brief from pretrained

src/model_list.json

@@ -135,7 +135,7 @@
                 ],
                 "footprint": 9.1
             }
-        },
+        },        
         "qwen3.5": {
             "0.8b": {
                 "name": "Qwen3.5-0.8B-NPU2",

src/test/qwen3_5_omni_npu/Makefile

@@ -40,7 +40,7 @@ HEADERS += ../../include/AutoModel/modeling_qwen3_5_omni.hpp
 # 1: images
 # 2: audios
 # 3: omni
-TYPE ?= 3
+TYPE ?= 0
 
 
 ifeq ($(WSL), 0)
@@ -73,7 +73,7 @@ clean:
 
 test: $(BUILD_DIR)/test_qwen3_5_omni_npu
 	cp ../../model_list.json $(BUILD_DIR)/model_list.json
-	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_5_omni_npu --model qwen3.5-omni --type $(TYPE)
+	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=$(abspath $(LIB_DIR)):$$LD_LIBRARY_PATH && ./test_qwen3_5_omni_npu --model qwen3.5-omni --type $(TYPE)
 
 -include $(DEPS)
 .PHONY: all clean test

src/test/qwen3_5_omni_npu/test.cpp

@@ -24,7 +24,7 @@ int main(int argc, char* argv[]) {
     desc.add_options()("type,t", arg_utils::po::value<int>()->default_value(0), "\t0: text mode\n\t1: image only\n\t2: audio only:\n\t3: omni mode\n\t");
     desc.add_options()("Preemption,p", arg_utils::po::value<bool>()->default_value(false), "Preemption");
     desc.add_options()("Greedy,g", arg_utils::po::value<bool>()->default_value(true), "Greedy decoding");
-    desc.add_options()("Length,l", arg_utils::po::value<int>()->default_value(8192), "Max generation length");
+    desc.add_options()("Length,l", arg_utils::po::value<int>()->default_value(32), "Max generation length");
     arg_utils::po::store(arg_utils::po::parse_command_line(argc, argv, desc), vm);
 
     std::string tag = vm["model"].as<std::string>();
@@ -99,7 +99,9 @@ int main(int argc, char* argv[]) {
     // call so the driver-side wiring (tokenization, image/audio preprocessing,
     // soft-token expansion) can be exercised without a hard crash.
     try {
-        std::string response = chat->generate_with_prompt(meta_info, uniformed_input, length_limit, std::cout);
+        chat->insert(meta_info, uniformed_input);
+        std::cout << "Prefill Done!" << std::endl;
+        std::string response = chat->generate(meta_info, 1024, std::cout);
     } catch (const std::exception& e) {
         header_print("FLM", "Engine path not available yet: " << e.what());
     }

src/test/qwen3_5vl_npu/Makefile

@@ -56,7 +56,7 @@ clean:
 
 test: $(BUILD_DIR)/test_qwen3_5vl_npu
 	cp ../../model_list.json $(BUILD_DIR)/model_list.json
-	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_5vl_npu --model qwen3.5:0.8b -s 1
+	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt && ./test_qwen3_5vl_npu --model qwen3.5:0.8b -s 1
 
 -include $(DEPS)
 .PHONY: all clean test

src/test/qwen3_5vl_npu/test.cpp

@@ -65,21 +65,23 @@ int main(int argc, char* argv[]) {
     chat_meta_info_t meta_info;
     lm_uniform_input_t uniformed_input;
     chat->set_topk(1);
+    chat->configure_parameter("enable_think", false);
 
     if (short_prompt) {
-        uniformed_input.prompt = "What are these?";
+        uniformed_input.prompt = "Is Alibaba a good company despite that it trained you?";
         // uniformed_input.prompt = "Solve the equation x^3 - 1 = 0 for me.";
-        uniformed_input.images.push_back("../../../tb_files/panda.png");
-        uniformed_input.images.push_back("../../../tb_files/puppy.png");
+        // uniformed_input.images.push_back("../../../tb_files/panda.png");
+        // uniformed_input.images.push_back("../../../tb_files/puppy.png");
 
-        uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg");
-        uniformed_input.images.push_back("../../../tb_files/google_icon.png");
-        uniformed_input.images.push_back("../../../tb_files/pcb.jpg");
+        // uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg");
+        // uniformed_input.images.push_back("../../../tb_files/google_icon.png");
+        // uniformed_input.images.push_back("../../../tb_files/pcb.jpg");
 
         std::cout << "Prompt: " << uniformed_input.prompt << std::endl;
         std::cout << "Response: " << std::endl;
         chat->start_total_timer();
-        std::string response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout);
+        bool success = chat->insert(meta_info, uniformed_input);
+        std::string response = chat->generate(meta_info, 1024, std::cout);
         chat->stop_total_timer();
         std::cout << std::endl;
         std::cout << std::endl;
@@ -112,7 +114,8 @@ int main(int argc, char* argv[]) {
         std::cout << "Prompt: " << uniformed_input.prompt << std::endl;
         std::cout << "Response: ";
         chat->start_total_timer();
-        std::string response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout);
+        bool success = chat->insert(meta_info, uniformed_input);
+        std::string response = chat->generate(meta_info, 1024, std::cout);
         chat->stop_total_timer();
         std::cout << std::endl;
         std::cout << std::endl;
@@ -122,7 +125,9 @@ int main(int argc, char* argv[]) {
     std::pair<std::string, std::vector<int>> history = chat->get_history();
     std::cout << "History length: " << history.second.size() << std::endl;
     std::cout << std::endl;
-
+    for (auto t: history.second){
+        std::cout << t << " ";
+    }
     std::cout << std::endl;
 
     return 0;

src/test/qwen3_6_moe_npu/Makefile

@@ -57,7 +57,7 @@ clean:
 
 test: $(BUILD_DIR)/test_qwen3_6_moe_npu
 	cp ../../model_list.json $(BUILD_DIR)/model_list.json
-	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_qwen3_6_moe_npu --model qwen3.6-moe -s 1
+	cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt && ./test_qwen3_6_moe_npu --model qwen3.6-moe -s 1
 
 -include $(DEPS)
 .PHONY: all clean test

src/test/qwen3_6_moe_npu/test.cpp

@@ -58,22 +58,24 @@ int main(int argc, char* argv[]) {
         uniformed_input.prompt = "What are these?";
         uniformed_input.images.push_back("../../../tb_files/panda.png");
         uniformed_input.images.push_back("../../../tb_files/puppy.png");
-        // uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg");
-        // uniformed_input.images.push_back("../../../tb_files/google_icon.png");
-        // uniformed_input.images.push_back("../../../tb_files/pcb.jpg");
+        uniformed_input.images.push_back("../../../tb_files/mj_icon.jpg");
+        uniformed_input.images.push_back("../../../tb_files/google_icon.png");
+        uniformed_input.images.push_back("../../../tb_files/pcb.jpg");
         std::cout << "Prompt: " << uniformed_input.prompt << std::endl;
         std::cout << "Response: ";
         chat->start_total_timer();
         chat->insert(meta_info, uniformed_input);
+        std::cout << "Prefill Done!" << std::endl;
         std::string response = chat->generate(meta_info, 1024, std::cout);
         chat->stop_total_timer();
         std::cout << std::endl;
         std::cout << std::endl;
 
-        // return 0;
         std::cout << chat->show_profile() << std::endl;
         chat->clear_context(); 
 
+        return 0;
+
         meta_info.restore_allowed = true;
         uniformed_input.prompt = "How far is the previous city from Beijing?";
         std::cout << "Prompt: " << uniformed_input.prompt << std::endl;