diff --git a/llama-cpp-bindings-sys/llama.cpp b/llama-cpp-bindings-sys/llama.cpp index bb4caa75..c1d0e7a0 160000 --- a/llama-cpp-bindings-sys/llama.cpp +++ b/llama-cpp-bindings-sys/llama.cpp @@ -1 +1 @@ -Subproject commit bb4caa7540188872173c44d161602d9271386413 +Subproject commit c1d0e7a004015f23bc0233470b747b596f29b264 diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp index 093c0dd5..5b822e8d 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp @@ -1,9 +1,9 @@ #include "wrapper_chat_apply.h" -#include "nlohmann/json_fwd.hpp" #include "wrapper_token_text.h" #include "llama.cpp/common/chat-auto-parser.h" #include "llama.cpp/common/chat.h" +#include "llama.cpp/common/json.h" #include "llama.cpp/include/llama.h" #include "wrapper_utils.h" @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -60,7 +59,7 @@ extern "C" auto llama_rs_apply_chat_template( common_chat_template const tmpl(template_src, bos_token, eos_token); - nlohmann::ordered_json messages = nlohmann::ordered_json::array(); + common_json messages = common_json::array(); const gsl::span role_span(roles, n_messages); const gsl::span content_span(contents, n_messages); for (size_t index = 0; index < n_messages; index++) { @@ -72,7 +71,7 @@ extern "C" auto llama_rs_apply_chat_template( autoparser::generation_params inputs; inputs.messages = std::move(messages); - inputs.tools = nlohmann::ordered_json::array(); + inputs.tools = common_json::array(); inputs.add_generation_prompt = add_generation_prompt != 0; inputs.enable_thinking = enable_thinking != 0; diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp index 63c8f4e6..bae6ba6b 100644 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp +++ b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp @@ -1,11 +1,10 @@ #include "wrapper_chat_parse.h" -#include // IWYU pragma: keep -#include #include "peg-parser.h" #include "wrapper_token_text.h" #include "llama.cpp/common/chat-auto-parser.h" #include "llama.cpp/common/chat.h" +#include "llama.cpp/common/json.h" #include "llama.cpp/include/llama.h" #include "wrapper_utils.h" @@ -152,9 +151,9 @@ extern "C" auto llama_rs_parse_chat_message( autoparser::generation_params inputs; if ((tools_json != nullptr) && *tools_json != '\0') { - inputs.tools = nlohmann::ordered_json::parse(tools_json); + inputs.tools = common_json::parse(tools_json); } else { - inputs.tools = nlohmann::ordered_json::array(); + inputs.tools = common_json::array(); } common_peg_arena const chat_parser = parser->parser.build_parser(inputs, std::string()); diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index 67765eb5..96b15508 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -11,9 +11,8 @@ #include #include "llama.cpp/common/json-schema-to-grammar.h" +#include "llama.cpp/common/json.h" #include "llama.cpp/include/llama.h" -#include // IWYU pragma: keep -#include #include "wrapper_utils.h" extern "C" auto llama_rs_json_schema_to_grammar( @@ -38,7 +37,7 @@ extern "C" auto llama_rs_json_schema_to_grammar( } try { - const auto schema = nlohmann::ordered_json::parse(schema_json); + const auto schema = common_json::parse(schema_json); const auto grammar = json_schema_to_grammar(schema, force_gbnf); *out_grammar = llama_rs_dup_string(grammar); if (*out_grammar == nullptr) { diff --git a/llama-cpp-bindings-sys/wrapper_fit.cpp b/llama-cpp-bindings-sys/wrapper_fit.cpp index 44ba371d..d9f0370e 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.cpp +++ b/llama-cpp-bindings-sys/wrapper_fit.cpp @@ -18,6 +18,7 @@ extern "C" auto llama_rs_fit_params( struct llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins, uint32_t n_ctx_min, + const llama_rs_fit_extra_model * extra, enum ggml_log_level log_level, int32_t * out_unrecognized_status_code, char ** out_error) -> llama_rs_fit_params_status { @@ -42,11 +43,30 @@ extern "C" auto llama_rs_fit_params( if (out_error == nullptr) { return LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG; } + if (extra != nullptr) { + if (extra->path_model == nullptr) { + return LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG; + } + if (extra->mparams == nullptr) { + return LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG; + } + if (extra->cparams == nullptr) { + return LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG; + } + } try { + common_fit_extra_model vendored_extra{}; + if (extra != nullptr) { + vendored_extra.path_model = extra->path_model; + vendored_extra.mparams = extra->mparams; + vendored_extra.cparams = extra->cparams; + vendored_extra.shares_model = extra->shares_model; + } + const common_params_fit_status status = common_fit_params( path_model, mparams, cparams, tensor_split, tensor_buft_overrides, - margins, n_ctx_min, log_level); + margins, n_ctx_min, extra == nullptr ? nullptr : &vendored_extra, log_level); switch (status) { case COMMON_PARAMS_FIT_STATUS_SUCCESS: return LLAMA_RS_FIT_PARAMS_OK; diff --git a/llama-cpp-bindings-sys/wrapper_fit.h b/llama-cpp-bindings-sys/wrapper_fit.h index ee64938f..d9e16200 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.h +++ b/llama-cpp-bindings-sys/wrapper_fit.h @@ -3,6 +3,7 @@ #include "llama.cpp/ggml/include/ggml.h" #include "llama.cpp/include/llama.h" +#include #include #include @@ -23,8 +24,18 @@ typedef enum llama_rs_fit_params_status { LLAMA_RS_FIT_PARAMS_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_FIT_PARAMS_VENDORED_OUT_OF_MEMORY, LLAMA_RS_FIT_PARAMS_VENDORED_THREW_CXX_EXCEPTION, + LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG, + LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG, + LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG, } llama_rs_fit_params_status; +typedef struct llama_rs_fit_extra_model { + const char * path_model; + struct llama_model_params * mparams; + struct llama_context_params * cparams; + bool shares_model; +} llama_rs_fit_extra_model; + llama_rs_fit_params_status llama_rs_fit_params( const char * path_model, struct llama_model_params * mparams, @@ -33,6 +44,7 @@ llama_rs_fit_params_status llama_rs_fit_params( struct llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins, uint32_t n_ctx_min, + const llama_rs_fit_extra_model * extra, enum ggml_log_level log_level, int32_t * out_unrecognized_status_code, char ** out_error); diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.cpp b/llama-cpp-bindings-sys/wrapper_reasoning.cpp index 66252ef9..96330115 100644 --- a/llama-cpp-bindings-sys/wrapper_reasoning.cpp +++ b/llama-cpp-bindings-sys/wrapper_reasoning.cpp @@ -3,9 +3,8 @@ #include "llama.cpp/common/chat-auto-parser.h" #include "llama.cpp/common/chat.h" +#include "llama.cpp/common/json.h" #include "llama.cpp/include/llama.h" -#include // IWYU pragma: keep -#include #include "wrapper_utils.h" #include @@ -36,8 +35,8 @@ auto find_reasoning_markers( probe_params.is_inference = false; probe_params.add_inference = false; probe_params.mark_input = false; - probe_params.messages = nlohmann::ordered_json::array({ - nlohmann::ordered_json{ { "role", "user" }, { "content", "ping" } }, + probe_params.messages = common_json::array({ + common_json{ { "role", "user" }, { "content", "ping" } }, }); const std::string tmpl_src_str = tmpl_src; diff --git a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp b/llama-cpp-bindings-sys/wrapper_tool_calls.cpp index 78312556..e8f0f00c 100644 --- a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp +++ b/llama-cpp-bindings-sys/wrapper_tool_calls.cpp @@ -1,11 +1,10 @@ #include "wrapper_tool_calls.h" -#include // IWYU pragma: keep -#include #include "wrapper_token_text.h" #include "llama.cpp/common/chat-auto-parser.h" #include "llama.cpp/common/chat-auto-parser-helpers.h" #include "llama.cpp/common/chat.h" +#include "llama.cpp/common/json.h" #include "llama.cpp/include/llama.h" #include "wrapper_utils.h" @@ -18,18 +17,21 @@ using wrapper_helpers::token_text_or_empty; namespace { -auto detect_tool_call_haystack( - const common_chat_template & tmpl, - const autoparser::analyze_reasoning & reasoning) -> std::string { - nlohmann::ordered_json const user_msg = { +struct tool_call_probe_params { + template_params without_tool_calls; + template_params with_tool_calls; +}; + +auto build_tool_call_probe_params() -> tool_call_probe_params { + common_json const user_msg = { { "role", "user" }, { "content", "Please use the tool" } }; - nlohmann::ordered_json const assistant_no_tools = { + common_json const assistant_no_tools = { { "role", "assistant" }, { "content", "Sure, calling." } }; - nlohmann::ordered_json const first_tool_call = { + common_json const first_tool_call = { { "id", "call_001" }, { "type", "function" }, { "function", { @@ -40,12 +42,12 @@ auto detect_tool_call_haystack( }} }} }; - nlohmann::ordered_json const assistant_with_tools = { - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", nlohmann::ordered_json::array({ first_tool_call }) } + common_json const assistant_with_tools = { + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", common_json::array({ first_tool_call }) } }; - nlohmann::ordered_json const tool_definition = { + common_json const tool_definition = { { "type", "function" }, { "function", { { "name", "tool_first" }, @@ -56,23 +58,30 @@ auto detect_tool_call_haystack( { "arg_first", { { "type", "string" }, { "description", "first arg" } } }, { "arg_second", { { "type", "string" }, { "description", "second arg" } } }, }}, - { "required", nlohmann::ordered_json::array({ "arg_first", "arg_second" }) }, + { "required", common_json::array({ "arg_first", "arg_second" }) }, }} }} }; - template_params params_no_tools; - params_no_tools.messages = nlohmann::ordered_json::array({ user_msg, assistant_no_tools }); - params_no_tools.tools = nlohmann::ordered_json::array({ tool_definition }); - params_no_tools.add_generation_prompt = false; - params_no_tools.enable_thinking = true; + tool_call_probe_params probe; + probe.without_tool_calls.messages = common_json::array({ user_msg, assistant_no_tools }); + probe.without_tool_calls.tools = common_json::array({ tool_definition }); + probe.without_tool_calls.add_generation_prompt = false; + probe.without_tool_calls.enable_thinking = true; + + probe.with_tool_calls = probe.without_tool_calls; + probe.with_tool_calls.messages = common_json::array({ user_msg, assistant_with_tools }); + + return probe; +} - template_params params_with_tools = params_no_tools; - params_with_tools.messages = - nlohmann::ordered_json::array({ user_msg, assistant_with_tools }); +auto detect_tool_call_haystack( + const common_chat_template & tmpl, + const autoparser::analyze_reasoning & reasoning) -> std::string { + tool_call_probe_params const probe = build_tool_call_probe_params(); - std::string const output_no_tools = autoparser::apply_template(tmpl, params_no_tools); - std::string const output_with_tools = autoparser::apply_template(tmpl, params_with_tools); + std::string const output_no_tools = autoparser::apply_template(tmpl, probe.without_tool_calls); + std::string const output_with_tools = autoparser::apply_template(tmpl, probe.with_tool_calls); if (output_no_tools.empty() || output_with_tools.empty()) { return {}; @@ -210,58 +219,10 @@ extern "C" auto llama_rs_diagnose_tool_call_synthetic_renders( common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - nlohmann::ordered_json const user_msg = { - { "role", "user" }, - { "content", "Please use the tool" } - }; - nlohmann::ordered_json const assistant_no_tools = { - { "role", "assistant" }, - { "content", "Sure, calling." } - }; - nlohmann::ordered_json const first_tool_call = { - { "id", "call_001" }, - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "arguments", { - { "arg_first", "XXXX" }, - { "arg_second", "YYYY" }, - }} - }} - }; - nlohmann::ordered_json const assistant_with_tools = { - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", nlohmann::ordered_json::array({ first_tool_call }) } - }; - nlohmann::ordered_json const tool_definition = { - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "description", "First test tool" }, - { "parameters", { - { "type", "object" }, - { "properties", { - { "arg_first", { { "type", "string" }, { "description", "first arg" } } }, - { "arg_second", { { "type", "string" }, { "description", "second arg" } } }, - }}, - { "required", nlohmann::ordered_json::array({ "arg_first", "arg_second" }) }, - }} - }} - }; - - template_params params_no_tools; - params_no_tools.messages = nlohmann::ordered_json::array({ user_msg, assistant_no_tools }); - params_no_tools.tools = nlohmann::ordered_json::array({ tool_definition }); - params_no_tools.add_generation_prompt = false; - params_no_tools.enable_thinking = true; - - template_params params_with_tools = params_no_tools; - params_with_tools.messages = - nlohmann::ordered_json::array({ user_msg, assistant_with_tools }); + tool_call_probe_params const probe = build_tool_call_probe_params(); - std::string const output_a = autoparser::apply_template(tmpl, params_no_tools); - std::string const output_b = autoparser::apply_template(tmpl, params_with_tools); + std::string const output_a = autoparser::apply_template(tmpl, probe.without_tool_calls); + std::string const output_b = autoparser::apply_template(tmpl, probe.with_tool_calls); std::unique_ptr a_dup(llama_rs_dup_string(output_a)); std::unique_ptr b_dup(llama_rs_dup_string(output_b)); diff --git a/llama-cpp-bindings-tests/tests/model_introspection.rs b/llama-cpp-bindings-tests/tests/model_introspection.rs index 060baed3..ca691cbd 100644 --- a/llama-cpp-bindings-tests/tests/model_introspection.rs +++ b/llama-cpp-bindings-tests/tests/model_introspection.rs @@ -1,5 +1,6 @@ use std::ffi::CString; use std::num::NonZeroU16; +use std::num::NonZeroU32; use std::pin::pin; use anyhow::Result; @@ -8,6 +9,7 @@ use llama_cpp_bindings::context::params::LlamaContextParams; use llama_cpp_bindings::max_devices; use llama_cpp_bindings::model::AddBos; use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; use llama_cpp_test_harness::LlamaFixture; use llama_cpp_test_harness::llama_test; @@ -745,6 +747,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> &mut context_params, &mut margins, 512, + None, llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); @@ -754,6 +757,62 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model_path_utf8 = fixture + .model_path + .to_str() + .ok_or_else(|| anyhow::anyhow!("model path is not valid UTF-8"))?; + let model_path_c = CString::new(model_path_utf8)?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + let unfittable_extra_n_ctx = NonZeroU32::MIN; + let mut extra_params = pin!(LlamaModelParams::default()); + let mut extra_context_params = + LlamaContextParams::default().with_n_ctx(Some(unfittable_extra_n_ctx)); + let mut extra = FitExtraModel { + model_path: &model_path_c, + model_params: extra_params.as_mut(), + context_params: &mut extra_context_params, + shares_model: true, + }; + + let result = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + Some(&mut extra), + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + let fit = result.map_err(|fit_error| anyhow::anyhow!("fit_params failed: {fit_error:?}"))?; + assert!(fit.n_ctx > 0); + assert_ne!( + extra_context_params.n_ctx(), + Some(unfittable_extra_n_ctx), + "the vendored fit must overwrite the extra model's context size" + ); + assert_eq!( + extra_context_params.context_params.n_ctx, + context_params.context_params.n_ctx + ); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 9f58fad4..fe55927f 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -9,11 +9,13 @@ use crate::error::{FitError, ModelParamsError}; use crate::model::llama_load_mode::LlamaLoadMode; use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; +use crate::model::params::fit_extra_model::FitExtraModel; use crate::model::params::fit_result::FitResult; use crate::model::params::kv_overrides::KvOverrides; use crate::model::split_mode::LlamaSplitMode; use llama_cpp_ffi_status::read_and_free_cpp_string; +pub mod fit_extra_model; pub mod fit_result; pub mod kv_override_entry; pub mod kv_override_value_iterator; @@ -92,6 +94,27 @@ fn fit_params_status_to_result( } .into()) } + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given an extra model with a null path_model argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given an extra model with a null mparams argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given an extra model with a null cparams argument", + } + .into()) + } other => Err(crate::FfiStatusError { operation: "llama_rs_fit_params", code: i64::from(other), @@ -372,6 +395,7 @@ impl LlamaModelParams { context_params: &mut LlamaContextParams, margins: &mut [usize], n_ctx_min: u32, + mut extra: Option<&mut FitExtraModel<'_>>, log_level: llama_cpp_bindings_sys::ggml_log_level, ) -> Result { let max_devices = unsafe { llama_cpp_bindings_sys::llama_max_devices() }; @@ -395,6 +419,11 @@ impl LlamaModelParams { let mut out_unrecognized_status_code: i32 = 0; let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut extra_ffi = extra.as_mut().map(|extra_model| extra_model.as_ffi()); + let extra_ptr = extra_ffi + .as_mut() + .map_or_else(null, |extra_model| &raw const *extra_model); + let status = unsafe { llama_cpp_bindings_sys::llama_rs_fit_params( model_path.as_ptr(), @@ -404,6 +433,7 @@ impl LlamaModelParams { self.buft_overrides.as_mut_ptr(), margins.as_mut_ptr(), n_ctx_min, + extra_ptr, log_level, &raw mut out_unrecognized_status_code, &raw mut out_error, @@ -790,6 +820,7 @@ mod tests { &mut context_params, &mut margins, 512, + None, llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); @@ -895,88 +926,71 @@ mod ffi_contract_status_tests { use crate::error::fit_error::FitError; use std::ptr; + struct ContractStatusCase { + status: llama_cpp_bindings_sys::llama_rs_fit_params_status, + detail: &'static str, + } + #[test] fn fit_params_status_to_result_maps_every_contract_status() { - let outcome_0 = fit_params_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_0.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null path_model argument", - } - .into() - ) - ); - let outcome_1 = fit_params_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_1.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null mparams argument", - } - .into() - ) - ); - let outcome_2 = fit_params_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_2.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null cparams argument", - } - .into() - ) - ); - let outcome_3 = fit_params_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_3.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null out_unrecognized_status_code argument", - } - .into() - ) - ); - let outcome_4 = fit_params_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_4.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null out_error argument", - } - .into() - ) - ); - let outcome_5 = fit_params_status_to_result( + let cases = [ + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG, + detail: "was given a null path_model argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG, + detail: "was given a null mparams argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG, + detail: "was given a null cparams argument", + }, + ContractStatusCase { + status: + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG, + detail: "was given a null out_unrecognized_status_code argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG, + detail: "was given a null out_error argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG, + detail: "was given an extra model with a null path_model argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG, + detail: "was given an extra model with a null mparams argument", + }, + ContractStatusCase { + status: llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG, + detail: "was given an extra model with a null cparams argument", + }, + ]; + + for ContractStatusCase { status, detail } in cases { + assert_eq!( + fit_params_status_to_result(status, 0, ptr::null_mut()).err(), + Some( + crate::FfiContractError { + operation: "llama_rs_fit_params", + detail, + } + .into() + ) + ); + } + } + + #[test] + fn fit_params_status_vendored_out_of_memory_returns_vendored_out_of_memory() { + let outcome = fit_params_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_VENDORED_OUT_OF_MEMORY, 0, ptr::null_mut(), ); - assert_eq!(outcome_5.err(), Some(FitError::VendoredOutOfMemory)); + + assert_eq!(outcome.err(), Some(FitError::VendoredOutOfMemory)); } } diff --git a/llama-cpp-bindings/src/model/params/fit_extra_model.rs b/llama-cpp-bindings/src/model/params/fit_extra_model.rs new file mode 100644 index 00000000..769e7b90 --- /dev/null +++ b/llama-cpp-bindings/src/model/params/fit_extra_model.rs @@ -0,0 +1,28 @@ +use std::ffi::CStr; +use std::pin::Pin; + +use crate::context::params::LlamaContextParams; +use crate::model::params::LlamaModelParams; + +/// A second model that shares the devices of the model being fitted, such as a draft model. +/// +/// Its context follows the fitted model's context, so `context_params` is rewritten by +/// [`LlamaModelParams::fit_params`]. Set `shares_model` when the weights are already +/// accounted for by the fitted model, as they are for an MTP context. +pub struct FitExtraModel<'extra_model> { + pub model_path: &'extra_model CStr, + pub model_params: Pin<&'extra_model mut LlamaModelParams>, + pub context_params: &'extra_model mut LlamaContextParams, + pub shares_model: bool, +} + +impl FitExtraModel<'_> { + pub fn as_ffi(&mut self) -> llama_cpp_bindings_sys::llama_rs_fit_extra_model { + llama_cpp_bindings_sys::llama_rs_fit_extra_model { + path_model: self.model_path.as_ptr(), + mparams: &raw mut self.model_params.params, + cparams: &raw mut self.context_params.context_params, + shares_model: self.shares_model, + } + } +}