Skip to content

support file-based ASR for nemotron streaming model #885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rui-ren wants to merge 2 commits into
base: main
Choose a base branch
from
+337 −0
Open

support file-based ASR for nemotron streaming model #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77170bd
add file based support for streaming model
Jul 14, 2026
d5eafd6
update the cpp code for streaming model
rui-ren Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
326 changes: 326 additions & 0 deletions sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,15 @@
#include "utils.h"

#include <filesystem>
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <fmt/format.h>
#include <nlohmann/json.hpp>
#include <ort_genai.h>
#include <unordered_map>

namespace fl {

Expand Down Expand Up @@ -71,8 +77,47 @@ std::string JoinTokens(const std::vector<std::string>& token_texts) {
return out;
}

const std::unordered_map<std::string, std::string>& NemotronLanguageIdMap() {
static const std::unordered_map<std::string, std::string> kMap = {
{"en", "0"}, {"en-us", "0"}, {"en-gb", "1"}, {"es-es", "2"}, {"es", "3"},
{"es-us", "3"}, {"zh-cn", "4"}, {"hi", "6"}, {"hi-in", "6"}, {"ar", "7"},
{"ar-ar", "7"}, {"fr", "8"}, {"fr-fr", "8"}, {"fr-ca", "100"}, {"de", "9"},
{"de-de", "9"}, {"ja", "10"}, {"ja-jp", "10"}, {"ru", "11"}, {"ru-ru", "11"},
{"pt-br", "12"}, {"pt", "13"}, {"pt-pt", "13"}, {"ko", "14"}, {"ko-kr", "14"},
{"it", "15"}, {"it-it", "15"}, {"nl", "16"}, {"nl-nl", "16"}, {"pl", "17"},
{"pl-pl", "17"}, {"tr", "18"}, {"tr-tr", "18"}, {"uk", "19"}, {"uk-ua", "19"},
{"ro", "20"}, {"ro-ro", "20"}, {"el", "21"}, {"el-gr", "21"}, {"cs", "22"},
{"cs-cz", "22"}, {"hu", "23"}, {"hu-hu", "23"}, {"sv", "24"}, {"sv-se", "24"},
{"da", "25"}, {"da-dk", "25"}, {"fi", "26"}, {"fi-fi", "26"}, {"sk", "28"},
{"sk-sk", "28"}, {"hr", "29"}, {"hr-hr", "29"}, {"bg", "30"}, {"bg-bg", "30"},
{"lt", "31"}, {"lt-lt", "31"}, {"th", "32"}, {"th-th", "32"}, {"vi", "33"},
{"vi-vn", "33"}, {"et", "60"}, {"et-ee", "60"}, {"lv", "61"}, {"lv-lv", "61"},
{"sl", "62"}, {"sl-si", "62"}, {"he", "64"}, {"he-il", "64"}, {"auto", "101"},
{"mt", "102"}, {"mt-mt", "102"}, {"nb", "103"}, {"nb-no", "103"}, {"nn", "104"},
{"nn-no", "104"},
};
return kMap;
}

std::string ToLowerAscii(std::string s) {
std::transform(s.begin(), s.end(), s.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return s;
}

bool IsLanguageToken(const std::string& token) {
size_t start = token.find_first_not_of(" \t\r\n");
if (start == std::string::npos) {
return false;
}

size_t end = token.find_last_not_of(" \t\r\n");
return end >= start && token[start] == '<' && token[end] == '>';
}

} // namespace


AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model,
ILogger& logger, ITelemetry& telemetry)
: Session(catalog_model, logger, telemetry), logger_(logger), model_(model) {
Expand Down Expand Up @@ -422,6 +467,13 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json
FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, fmt::format("Audio file not found: '{}'", req.filename));
}

// If the model is a Nemotron speech model, process the transcription using the Nemotron-specific method.
// e.g. RNNT models require Nemotron-specific transcription handling.
if (IsNemotronSpeechModel()) {
ProcessNemotronFileTranscription(req, original_request, response);
return;
}

// Build generation options from session defaults
SearchOptions options = session_options_;

Expand Down Expand Up @@ -502,4 +554,278 @@ void AudioSession::ProcessAudioTranscriptionJson(const std::string& request_json
total_tokens, prompt_tokens, completion_tokens));
}


bool AudioSession::IsNemotronSpeechModel() const {
const auto& cfg = Model().GetGenAIConfig();
return cfg.model.has_value() && cfg.model->type == "nemotron_speech";
}

void AudioSession::TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const {
if (language.empty()) {
return;
}

const auto key = ToLowerAscii(language);
auto it = NemotronLanguageIdMap().find(key);
if (it == NemotronLanguageIdMap().end()) {
return;
}

try {
generator.SetRuntimeOption("lang_id", it->second.c_str());
} catch (const std::exception& e) {
logger_.Log(LogLevel::Warning,
fmt::format("Failed to set Nemotron lang_id '{}' for '{}': {}",
it->second, language, e.what()));
}
}

void AudioSession::ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req,
const Request& original_request,
Response& response) {
if (original_request.canceled) {
response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE;
return;
}

std::optional<float> temperature = req.temperature.has_value() ? req.temperature : session_options_.temperature;

std::string language;
if (req.language.has_value()) {
language = *req.language;
} else {
auto session_lang_it = SessionOptions().find("language");
if (session_lang_it != SessionOptions().end()) {
language = session_lang_it->second;
}
}

auto samples = LoadPcmWavAsFloatSamples(req.filename);
auto& oga_model = Model().GetOgaModel();
auto processor = OgaStreamingProcessor::Create(oga_model);
auto tokenizer = OgaTokenizer::Create(oga_model);
auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer);
auto generator_params = OgaGeneratorParams::Create(oga_model);
if (temperature.has_value()) {
generator_params->SetSearchOption("temperature", *temperature);
} else {
generator_params->SetSearchOption("temperature", 0.0f);
}
auto generator = OgaGenerator::Create(oga_model, *generator_params);
TrySetNemotronLanguageId(*generator, language);

auto streaming_callback = CreateCallbackHandler(original_request);
const bool is_streaming = (streaming_callback != nullptr);
std::string response_id = ResponseConverter::GenerateId("audio");

std::string text;
text.reserve(512);
int completion_tokens = 0;

auto decode_all_tokens = [&]() {
while (!generator->IsDone() && !original_request.canceled) {
generator->GenerateNextToken();
auto next_tokens = generator->GetNextTokens();
if (next_tokens.empty()) {
continue;
}

++completion_tokens;
const char* decoded = tokenizer_stream->Decode(next_tokens[0]);
if (!decoded || decoded[0] == '\0') {
continue;
}

std::string token(decoded);
if (IsLanguageToken(token)) {
continue;
}

text += token;

if (is_streaming) {
AudioTranscriptionResponse chunk;
chunk.id = response_id;
chunk.text = token;
streaming_callback->PushItem(std::make_unique<TextItem>(nlohmann::json(chunk).dump(),
FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON));
}
}
};

auto run_one_pass = [&](std::unique_ptr<OgaNamedTensors> tensors) {
if (!tensors || original_request.canceled) {
return;
}
generator->SetInputs(*tensors);
decode_all_tokens();
};

constexpr size_t kNemotronSamplesPerChunk = 1600; // 100ms at 16kHz
for (size_t offset = 0; offset < samples.size() && !original_request.canceled;
offset += kNemotronSamplesPerChunk) {
size_t count = std::min(kNemotronSamplesPerChunk, samples.size() - offset);
run_one_pass(processor->Process(samples.data() + offset, count));
}
run_one_pass(processor->Flush());

response.finish_reason = original_request.canceled ? FOUNDRY_LOCAL_FINISH_NONE : FOUNDRY_LOCAL_FINISH_STOP;
response.usage.prompt_tokens = 0;
response.usage.completion_tokens = completion_tokens;
response.usage.total_tokens = completion_tokens;

AudioTranscriptionResponse output;
output.id = response_id;
output.text = std::move(text);
response.items.push_back(std::make_unique<TextItem>(nlohmann::json(output).dump(),
FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON));
}

std::vector<float> AudioSession::LoadPcmWavAsFloatSamples(const std::string& audio_file_path) {
std::ifstream in(audio_file_path, std::ios::binary);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Failed to open audio file: '{}'", audio_file_path));
}

in.seekg(0, std::ios::end);
std::streamoff file_size = in.tellg();
in.seekg(0, std::ios::beg);
if (file_size < 12) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: too small for RIFF/WAVE header.");
}

char riff[4];
uint32_t riff_size = 0;
char wave[4];
in.read(riff, sizeof(riff));
in.read(reinterpret_cast<char*>(&riff_size), sizeof(riff_size));
in.read(wave, sizeof(wave));

if (!in || std::strncmp(riff, "RIFF", 4) != 0 || std::strncmp(wave, "WAVE", 4) != 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Invalid WAV file: missing RIFF/WAVE header.");
}

int16_t audio_format = 0;
int16_t channels = 0;
int32_t sample_rate = 0;
int16_t bits_per_sample = 0;
std::vector<uint8_t> data;

while (in && in.tellg() < file_size) {
char chunk_id_chars[4];
uint32_t chunk_size = 0;
in.read(chunk_id_chars, sizeof(chunk_id_chars));
in.read(reinterpret_cast<char*>(&chunk_size), sizeof(chunk_size));
if (!in) {
break;
}

const std::string chunk_id(chunk_id_chars, sizeof(chunk_id_chars));
const std::streamoff chunk_start = in.tellg();
if (chunk_start < 0 ||
chunk_start + static_cast<std::streamoff>(chunk_size) >
file_size - (chunk_size % 2 == 1 ? 1 : 0)) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk.");
}

if (chunk_id == "fmt ") {
if (chunk_size < 16) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Invalid WAV fmt chunk size {}; expected at least 16.", chunk_size));
}

in.read(reinterpret_cast<char*>(&audio_format), sizeof(audio_format));
in.read(reinterpret_cast<char*>(&channels), sizeof(channels));
in.read(reinterpret_cast<char*>(&sample_rate), sizeof(sample_rate));

int32_t byte_rate = 0;
int16_t block_align = 0;
in.read(reinterpret_cast<char*>(&byte_rate), sizeof(byte_rate));
in.read(reinterpret_cast<char*>(&block_align), sizeof(block_align));
in.read(reinterpret_cast<char*>(&bits_per_sample), sizeof(bits_per_sample));

int32_t remaining = static_cast<int32_t>(chunk_size) - 16;
if (remaining > 0) {
in.seekg(remaining, std::ios::cur);
}
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV fmt chunk.");
}
} else if (chunk_id == "data") {
data.resize(chunk_size);
if (chunk_size > 0) {
in.read(reinterpret_cast<char*>(data.data()), chunk_size);
}
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data chunk.");
}
} else {
in.seekg(chunk_size, std::ios::cur);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV chunk.");
}
}

if (chunk_size % 2 == 1) {
in.seekg(1, std::ios::cur);
if (!in) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV padding byte.");
}
}
}

if (channels <= 0 || sample_rate <= 0 || bits_per_sample <= 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Missing or invalid WAV fmt chunk.");
}
if (sample_rate != 16000) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Expected 16kHz WAV input, got {}Hz.", sample_rate));
}
if (data.empty()) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "WAV data chunk is missing or empty.");
}

const size_t bytes_per_sample = static_cast<size_t>(bits_per_sample / 8);
if ((audio_format != 1 || bits_per_sample != 16) && (audio_format != 3 || bits_per_sample != 32)) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
fmt::format("Unsupported WAV format: audioFormat={}, bitsPerSample={}.", audio_format, bits_per_sample));
}
if (bytes_per_sample == 0 || data.size() % (bytes_per_sample * static_cast<size_t>(channels)) != 0) {
FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Corrupted WAV data size.");
}

const size_t frame_count = data.size() / (bytes_per_sample * static_cast<size_t>(channels));
std::vector<float> samples;
samples.reserve(frame_count);

if (audio_format == 1 && bits_per_sample == 16) {
for (size_t frame = 0; frame < frame_count; ++frame) {
float mixed = 0.0f;
for (int16_t ch = 0; ch < channels; ++ch) {
size_t idx = frame * static_cast<size_t>(channels) + static_cast<size_t>(ch);
size_t byte_offset = idx * bytes_per_sample;
int16_t sample = 0;
std::memcpy(&sample, data.data() + byte_offset, sizeof(sample));
mixed += static_cast<float>(sample) / 32768.0f;
}
samples.push_back(mixed / static_cast<float>(channels));
}
return samples;
}

for (size_t frame = 0; frame < frame_count; ++frame) {
float mixed = 0.0f;
for (int16_t ch = 0; ch < channels; ++ch) {
size_t idx = frame * static_cast<size_t>(channels) + static_cast<size_t>(ch);
float value = 0.0f;
std::memcpy(&value, data.data() + (idx * bytes_per_sample), sizeof(float));
mixed += value;
}
samples.push_back(mixed / static_cast<float>(channels));
}

return samples;
}

} // namespace fl
11 changes: 11 additions & 0 deletions sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ struct OgaTokenizerStream;
namespace fl {

class GenAIModelInstance;
struct AudioTranscriptionRequest;
struct AudioItem;
struct ItemQueue;
struct SpeechSegmentItem;
Expand Down Expand Up @@ -53,6 +54,16 @@ class AudioSession : public Session {
void ProcessAudioTranscriptionJson(const std::string& request_json, const Request& original_request,
Response& response);

bool IsNemotronSpeechModel() const;

void ProcessNemotronFileTranscription(const AudioTranscriptionRequest& req,
const Request& original_request,
Response& response);

void TrySetNemotronLanguageId(OgaGenerator& generator, const std::string& language) const;

static std::vector<float> LoadPcmWavAsFloatSamples(const std::string& audio_file_path);

/// Process a streaming audio request: an AudioItem (format descriptor) + an ItemQueue (PCM chunks).
void ProcessStreamingAudio(const AudioItem& format_item, ItemQueue& queue,
const Request& request, Response& response);
Expand Down