Let the tool-call opener win over a reasoning close, and treat an absent KV cache as already cleared

llama-cpp-bindings-tests/tests/context_state.rs

@@ -6,7 +6,8 @@ use anyhow::Result;
 use llama_cpp_bindings::DecodeError;
 use llama_cpp_bindings::LogitsError;
 use llama_cpp_bindings::context::LlamaContext;
-use llama_cpp_bindings::error::KvCacheConversionError;
+use llama_cpp_bindings::error::ClearKvCacheSeqError;
+use llama_cpp_bindings::error::CopyKvCacheSeqError;
 use llama_cpp_bindings::error::KvCacheSeqAddError;
 use llama_cpp_bindings::error::KvCacheSeqDivError;
 use llama_cpp_bindings::llama_batch::LlamaBatch;
@@ -726,7 +727,7 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> {
 
     prime_kv_cache(fixture, &mut context)?;
 
-    context.clear_kv_cache()?;
+    context.clear_kv_cache();
     assert_eq!(context.kv_cache_seq_pos_max(0)?, -1);
 
     Ok(())
@@ -1000,7 +1001,7 @@ fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> R
 
     prime_kv_cache(fixture, &mut context)?;
 
-    context.kv_cache_seq_keep(0)?;
+    context.kv_cache_seq_keep(0);
 
     assert!(context.kv_cache_seq_pos_max(0)? >= 0);
 
@@ -1133,7 +1134,7 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) ->
 
     assert_eq!(
         result.unwrap_err(),
-        KvCacheConversionError::P0TooLarge(
+        CopyKvCacheSeqError::P0TooLarge(
             i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32")
         )
     );
@@ -1180,7 +1181,7 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) ->
 
     assert_eq!(
         result.unwrap_err(),
-        KvCacheConversionError::P1TooLarge(
+        CopyKvCacheSeqError::P1TooLarge(
             i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32")
         )
     );
@@ -1220,14 +1221,16 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) ->
     n_batch = 512,
     n_ubatch = 128,
 )]
-fn clear_kv_cache_seq_rejects_src_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> {
+fn clear_kv_cache_seq_rejects_sequence_id_exceeding_i32_max(
+    fixture: &LlamaFixture<'_>,
+) -> Result<()> {
     let mut context = fixture.build_context()?;
 
     let result = context.clear_kv_cache_seq(Some(u32::MAX), None, None);
 
     assert_eq!(
         result.unwrap_err(),
-        KvCacheConversionError::SeqIdTooLarge(
+        ClearKvCacheSeqError::SeqIdTooLarge(
             i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32")
         )
     );
@@ -1274,7 +1277,7 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -
 
     assert_eq!(
         result.unwrap_err(),
-        KvCacheConversionError::P0TooLarge(
+        ClearKvCacheSeqError::P0TooLarge(
             i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32")
         )
     );
@@ -1321,7 +1324,7 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -
 
     assert_eq!(
         result.unwrap_err(),
-        KvCacheConversionError::P1TooLarge(
+        ClearKvCacheSeqError::P1TooLarge(
             i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32")
         )
     );

llama-cpp-bindings-tests/tests/embedding_models.rs

@@ -4,6 +4,11 @@ use std::time::Duration;
 use anyhow::Context;
 use anyhow::Result;
 use anyhow::bail;
+use llama_cpp_bindings::ClearKvCacheSeqError;
+use llama_cpp_bindings::CopyKvCacheSeqError;
+use llama_cpp_bindings::KvCacheSeqAddError;
+use llama_cpp_bindings::KvCacheSeqDivError;
+use llama_cpp_bindings::KvCacheSeqPosMaxError;
 use llama_cpp_bindings::context::LlamaContext;
 use llama_cpp_bindings::ggml_time_us;
 use llama_cpp_bindings::llama_batch::LlamaBatch;
@@ -67,7 +72,7 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<(
     assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count);
     assert_eq!(classifier.usage().prompt_tokens, 0);
 
-    ctx.clear_kv_cache()?;
+    ctx.clear_kv_cache();
     ctx.decode(&mut batch)
         .with_context(|| "llama_decode() failed")?;
 
@@ -180,7 +185,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> {
     assert_eq!(classifier.pending_prompt_tokens(), total_token_count);
     assert_eq!(classifier.usage().prompt_tokens, 0);
 
-    ctx.clear_kv_cache()?;
+    ctx.clear_kv_cache();
     ctx.decode(&mut batch)
         .with_context(|| "llama_decode() failed")?;
 
@@ -393,7 +398,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity(
             batch.add_sequence(&tokens, sequence_id, true)?;
         }
 
-        context.clear_kv_cache()?;
+        context.clear_kv_cache();
         context.decode(&mut batch)?;
 
         for sequence_index in 0..iteration_inputs.len() {
@@ -581,7 +586,51 @@ fn embedding_model_exposes_tool_call_markers(fixture: &LlamaFixture<'_>) -> Resu
 fn embedding_model_exposes_streaming_markers(fixture: &LlamaFixture<'_>) -> Result<()> {
     let markers = fixture.model.streaming_markers()?;
 
-    assert!(markers.has_any());
+    assert!(!markers.is_empty());
+
+    Ok(())
+}
+
+#[llama_test(
+    model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"),
+    n_gpu_layers = 999,
+    load_mode = Mmap,
+    n_ctx = 512,
+    n_batch = 2048,
+    n_ubatch = 512,
+    n_threads_batch = 8,
+    embeddings = true,
+)]
+fn kv_cache_operations_respect_an_embedding_context_without_memory(
+    fixture: &LlamaFixture<'_>,
+) -> Result<()> {
+    let mut context = fixture.build_context()?;
+
+    context.clear_kv_cache();
+    assert_eq!(context.clear_kv_cache_seq(Some(0), None, None), Ok(()));
+    context.kv_cache_seq_keep(0);
+
+    assert_eq!(
+        context.copy_kv_cache_seq(0, 1, None, None),
+        Err(CopyKvCacheSeqError::MemoryHandleUnavailable)
+    );
+    assert_eq!(
+        context.kv_cache_seq_add(0, None, None, 1),
+        Err(KvCacheSeqAddError::MemoryHandleUnavailable)
+    );
+    let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow::anyhow!("2 is non-zero"))?;
+    assert_eq!(
+        context.kv_cache_seq_div(0, None, None, divisor),
+        Err(KvCacheSeqDivError::MemoryHandleUnavailable)
+    );
+    assert_eq!(
+        context.kv_cache_seq_pos_max(0),
+        Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable)
+    );
+    assert!(matches!(
+        context.clear_kv_cache_seq(Some(u32::MAX), None, None),
+        Err(ClearKvCacheSeqError::SeqIdTooLarge(_))
+    ));
 
     Ok(())
 }

llama-cpp-bindings-tests/tests/generation_control.rs

@@ -15,10 +15,8 @@ use llama_cpp_bindings::llama_batch::LlamaBatch;
 use llama_cpp_bindings::llguidance_sampler::create_llg_sampler;
 use llama_cpp_bindings::model::AddBos;
 use llama_cpp_bindings::model::LlamaChatMessage;
-use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier;
 use llama_cpp_bindings::sampled_token_section::SampledTokenSection;
 use llama_cpp_bindings::sampling::LlamaSampler;
-use llama_cpp_bindings::streaming_markers::StreamingMarkers;
 use llama_cpp_bindings::token::LlamaToken;
 use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop;
 use llama_cpp_test_harness::LlamaFixture;
@@ -1903,13 +1901,14 @@ fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_>
     n_batch = 128,
     n_ubatch = 64,
 )]
-fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece(
+fn ingest_flushes_an_unmatched_token_with_its_visible_and_raw_piece(
     fixture: &LlamaFixture<'_>,
 ) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
 
-    let outcomes = classifier.ingest(model.token_bos())?;
+    let mut outcomes = classifier.ingest(model.token_bos())?;
+    outcomes.extend(classifier.flush());
 
     assert_eq!(outcomes.len(), 1);
     let outcome = &outcomes[0];
@@ -1954,14 +1953,13 @@ fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece(
     n_batch = 128,
     n_ubatch = 64,
 )]
-fn ingest_with_no_markers_decodes_each_token_independently(
-    fixture: &LlamaFixture<'_>,
-) -> Result<()> {
+fn ingest_accounts_for_each_unmatched_token_after_flush(fixture: &LlamaFixture<'_>) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
 
     classifier.ingest(model.token_bos())?;
     classifier.ingest(model.token_eos())?;
+    classifier.flush();
 
     assert_eq!(classifier.usage().undeterminable_tokens, 2);
     Ok(())
@@ -1999,9 +1997,9 @@ fn ingest_with_no_markers_decodes_each_token_independently(
     n_batch = 128,
     n_ubatch = 64,
 )]
-fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) -> Result<()> {
+fn ingest_unmatched_prompt_tokens_does_not_record_usage(fixture: &LlamaFixture<'_>) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
     let usage_before = *classifier.usage();
 
     classifier.ingest_prompt_token(model.token_bos());
@@ -2046,7 +2044,7 @@ fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) ->
 )]
 fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
     let mut batch = LlamaBatch::new(8, 1)?;
 
     classifier.feed_prompt_to_batch(&mut batch, model.token_bos(), 0, &[0], false)?;
@@ -2092,7 +2090,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<
 )]
 fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
     let mut batch = LlamaBatch::new(8, 1)?;
 
     let tokens = vec![model.token_bos(), model.token_eos(), model.token_nl()];
@@ -2140,7 +2138,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears(
     fixture: &LlamaFixture<'_>,
 ) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
     let mut batch = LlamaBatch::new(8, 1)?;
 
     classifier.feed_prompt_to_batch(&mut batch, model.token_bos(), 0, &[0], false)?;
@@ -2191,7 +2189,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage(
     fixture: &LlamaFixture<'_>,
 ) -> Result<()> {
     let model = fixture.model;
-    let mut classifier = SampledTokenClassifier::new(model, StreamingMarkers::default());
+    let mut classifier = model.sampled_token_classifier()?;
     let mut batch = LlamaBatch::new(8, 1)?;
 
     classifier.feed_prompt_to_batch(&mut batch, model.token_bos(), 0, &[0], false)?;

llama-cpp-bindings-tests/tests/structured_chat_output.rs

@@ -1,7 +1,9 @@
 use anyhow::Result;
 use anyhow::bail;
 use llama_cpp_bindings::ChatMessageParseOutcome;
+use llama_cpp_bindings::MarkerRole;
 use llama_cpp_bindings::ParsedChatMessage;
+use llama_cpp_bindings::SampledTokenSection;
 use llama_cpp_bindings::TokenUsage;
 use llama_cpp_bindings::ToolCallArgsShape;
 use llama_cpp_bindings::ToolCallArguments;
@@ -1465,7 +1467,7 @@ fn qwen35_chat_inference_emits_reasoning_when_template_auto_opens(
     n_batch = 512,
     n_ubatch = 128,
 )]
-fn qwen35_streaming_markers_tokenize_every_reasoning_boundary(
+fn qwen35_shared_reasoning_close_and_tool_call_open_is_one_transition(
     fixture: &LlamaFixture<'_>,
 ) -> Result<()> {
     let reasoning_markers = fixture
@@ -1474,18 +1476,40 @@ fn qwen35_streaming_markers_tokenize_every_reasoning_boundary(
         .expect("Qwen3.5 must expose reasoning markers");
     let streaming_markers = fixture.model.streaming_markers()?;
 
-    assert!(streaming_markers.reasoning_open.is_some());
+    assert!(streaming_markers.iter().any(|marker| {
+        marker.roles().contains(&MarkerRole::ReasoningOpen) && !marker.tokens().is_empty()
+    }));
     assert_eq!(
-        streaming_markers.reasoning_closes.len(),
-        reasoning_markers.closes.len()
-    );
-    assert!(
         streaming_markers
-            .reasoning_closes
             .iter()
-            .all(|tokens| !tokens.is_empty())
+            .filter(|marker| marker.roles().contains(&MarkerRole::ReasoningClose))
+            .count(),
+        reasoning_markers.closes.len()
     );
 
+    let reasoning_open = streaming_markers
+        .iter()
+        .find(|marker| marker.roles().contains(&MarkerRole::ReasoningOpen))
+        .expect("Qwen3.5 must expose a reasoning opener")
+        .tokens()
+        .to_vec();
+    let shared_boundary = streaming_markers
+        .iter()
+        .find(|marker| {
+            marker.roles().contains(&MarkerRole::ReasoningClose)
+                && marker.roles().contains(&MarkerRole::ToolCallOpen)
+        })
+        .expect("Qwen3.5 must share its tool-call opener with a reasoning close")
+        .tokens()
+        .to_vec();
+
+    let mut classifier = fixture.model.sampled_token_classifier()?;
+    classifier.ingest_prompt_tokens(&reasoning_open);
+    assert_eq!(classifier.current_section(), SampledTokenSection::Reasoning);
+
+    classifier.ingest_prompt_tokens(&shared_boundary);
+    assert_eq!(classifier.current_section(), SampledTokenSection::ToolCall);
+
     Ok(())
 }