From ee4b208f95bd99a6481431206260dff3ef8dbcf9 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 5 Mar 2026 10:19:03 -0800 Subject: [PATCH] native tool calling detection for novita --- server/utils/AiProviders/novita/index.js | 30 ++++- .../utils/agents/aibitat/providers/novita.js | 122 +++++++++++++++--- 2 files changed, 131 insertions(+), 21 deletions(-) diff --git a/server/utils/AiProviders/novita/index.js b/server/utils/AiProviders/novita/index.js index 957605a9..d2fb62db 100644 --- a/server/utils/AiProviders/novita/index.js +++ b/server/utils/AiProviders/novita/index.js @@ -82,7 +82,7 @@ class NovitaLLM { // from the current date. If it is, then we will refetch the API so that all the models are up // to date. #cacheIsStale() { - const MAX_STALE = 6.048e8; // 1 Week in MS + const MAX_STALE = 2.592e8; // 3 days in MS if (!fs.existsSync(this.cacheAtPath)) return true; const now = Number(new Date()); const timestampMs = Number(fs.readFileSync(this.cacheAtPath)); @@ -143,6 +143,32 @@ class NovitaLLM { return availableModels[this.model]?.maxLength || 4096; } + /** + * Get the capabilities of a model from the Novita API. + * @returns {Promise<{tools: 'unknown' | boolean, reasoning: 'unknown' | boolean, imageGeneration: 'unknown' | boolean, vision: 'unknown' | boolean}>} + */ + async getModelCapabilities() { + try { + await this.#syncModels(); + const availableModels = this.models(); + const modelInfo = availableModels[this.model]; + return { + tools: modelInfo.features.includes("function-calling"), + reasoning: modelInfo.features.includes("reasoning"), + imageGeneration: false, // no image generation capabilities for Novita yet. + vision: modelInfo.input_modalities.includes("image"), + }; + } catch (error) { + console.error("Error getting model capabilities:", error); + return { + tools: "unknown", + reasoning: "unknown", + imageGeneration: "unknown", + vision: "unknown", + }; + } + } + async isValidChatCompletionModel(model = "") { await this.#syncModels(); const availableModels = this.models(); @@ -398,6 +424,8 @@ async function fetchNovitaModels() { model.id.split("/")[0].charAt(0).toUpperCase() + model.id.split("/")[0].slice(1), maxLength: model.context_size, + features: model.features ?? [], + input_modalities: model.input_modalities ?? [], }; }); diff --git a/server/utils/agents/aibitat/providers/novita.js b/server/utils/agents/aibitat/providers/novita.js index 524e003d..07e3fd7b 100644 --- a/server/utils/agents/aibitat/providers/novita.js +++ b/server/utils/agents/aibitat/providers/novita.js @@ -2,9 +2,14 @@ const OpenAI = require("openai"); const Provider = require("./ai-provider.js"); const InheritMultiple = require("./helpers/classes.js"); const UnTooled = require("./helpers/untooled.js"); +const { tooledStream, tooledComplete } = require("./helpers/tooled.js"); +const { RetryError } = require("../error.js"); +const { NovitaLLM } = require("../../../AiProviders/novita/index.js"); /** * The agent provider for the Novita AI provider. + * Supports true OpenAI-compatible tool calling when the model supports it, + * falling back to the UnTooled prompt-based approach otherwise. */ class NovitaProvider extends InheritMultiple([Provider, UnTooled]) { model; @@ -25,8 +30,13 @@ class NovitaProvider extends InheritMultiple([Provider, UnTooled]) { this._client = client; this.model = model; this.verbose = true; + this._supportsToolCalling = null; } + /** + * Get the Novita client. + * @returns {import("openai").OpenAI} + */ get client() { return this._client; } @@ -36,12 +46,16 @@ class NovitaProvider extends InheritMultiple([Provider, UnTooled]) { } /** - * Whether this provider supports native OpenAI-compatible tool calling. - * Override in subclass and return true to use native tool calling instead of UnTooled. - * @returns {boolean|Promise} + * Whether the loaded model supports native OpenAI-compatible tool calling. + * Checks the Novita model capabilities and caches the result. + * @returns {Promise} */ - supportsNativeToolCalling() { - return false; + async supportsNativeToolCalling() { + if (this._supportsToolCalling !== null) return this._supportsToolCalling; + const novita = new NovitaLLM(null, this.model); + const capabilities = await novita.getModelCapabilities(); + this._supportsToolCalling = capabilities.tools === true; + return this._supportsToolCalling; } async #handleFunctionCallChat({ messages = [] }) { @@ -70,33 +84,101 @@ class NovitaProvider extends InheritMultiple([Provider, UnTooled]) { }); } + /** + * Stream a chat completion with tool calling support. + * Uses native tool calling when supported, otherwise falls back to UnTooled. + */ async stream(messages, functions = [], eventHandler = null) { - return await UnTooled.prototype.stream.call( - this, - messages, - functions, - this.#handleFunctionCallStream.bind(this), - eventHandler + const useNative = + functions.length > 0 && (await this.supportsNativeToolCalling()); + + if (!useNative) { + return await UnTooled.prototype.stream.call( + this, + messages, + functions, + this.#handleFunctionCallStream.bind(this), + eventHandler + ); + } + + this.providerLog( + "Provider.stream (tooled) - will process this chat completion." ); + + try { + return await tooledStream( + this.client, + this.model, + messages, + functions, + eventHandler + ); + } catch (error) { + console.error(error.message, error); + if (error instanceof OpenAI.AuthenticationError) throw error; + if ( + error instanceof OpenAI.RateLimitError || + error instanceof OpenAI.InternalServerError || + error instanceof OpenAI.APIError + ) { + throw new RetryError(error.message); + } + throw error; + } } + /** + * Create a non-streaming completion with tool calling support. + * Uses native tool calling when supported, otherwise falls back to UnTooled. + */ async complete(messages, functions = []) { - return await UnTooled.prototype.complete.call( - this, - messages, - functions, - this.#handleFunctionCallChat.bind(this) - ); + const useNative = + functions.length > 0 && (await this.supportsNativeToolCalling()); + + if (!useNative) { + return await UnTooled.prototype.complete.call( + this, + messages, + functions, + this.#handleFunctionCallChat.bind(this) + ); + } + + try { + const result = await tooledComplete( + this.client, + this.model, + messages, + functions, + this.getCost.bind(this) + ); + + if (result.retryWithError) { + return this.complete([...messages, result.retryWithError], functions); + } + + return result; + } catch (error) { + if (error instanceof OpenAI.AuthenticationError) throw error; + if ( + error instanceof OpenAI.RateLimitError || + error instanceof OpenAI.InternalServerError || + error instanceof OpenAI.APIError + ) { + throw new RetryError(error.message); + } + throw error; + } } /** * Get the cost of the completion. - * + * Stubbed since Novita AI has no cost basis. * @param _usage The completion to get the cost for. * @returns The cost of the completion. - * Stubbed since Novita AI has no cost basis. */ - getCost() { + getCost(_usage) { return 0; } }