d6f0d305ab
* WIP agentic tool call streaming - OpenAI - Anthropic - Azure OpenAI * WIP rest of providers EXCLUDES Bedrock and GenericOpenAI * patch untooled complete/streaming to use chatCallback provider from provider class and not assume OpenAI client struct example: Ollama * modify ollama to function with its own overrides normalize completion/stream outputs across providers/untooled * dev build * fix message sanization for anthropic agent streaming * wip fix anthropic agentic streaming sanitization * patch gemini, webgenui, generic aibitat providers + disable providers unable to test * refactor anthropic aibitat provider for empty message and tool call formatting * Add frontend missing prop check update Azure for streaming support update Gemini to streamting support on gemini-* models generic OpenAI disable streaming verify localAI support verify NVIDIA Nim support * DPAIS, remove temp from call, support streaming' * remove 0 temp to remove possibility of bad temp error/500s/400s * Patch condition where model is non-streamable and no tools are present or called resulting in the provider `handleFunctionCallChat` being called - which returns a string. This would then fail in Untooled.complete since response would be a string and not the expected `response.choices?.[0]?.message` Modified this line to handle both conditions for stream/non-streaming and tool presence or lack thereof * Allow generic Openai to be streamable since using untooled it should work fine honor disabled streaming for provider where that concern may apply for regular chats * rename function and more gemini-specific function to gemini provider * add comments for readability .complete on azure should be non-streaming as this is the sync response * migrate CometAPI, but disable as we cannot test --------- Co-authored-by: shatfield4 <seanhatfield5@gmail.com>
93 lines
2.2 KiB
JavaScript
93 lines
2.2 KiB
JavaScript
const OpenAI = require("openai");
|
|
const Provider = require("./ai-provider.js");
|
|
const InheritMultiple = require("./helpers/classes.js");
|
|
const UnTooled = require("./helpers/untooled.js");
|
|
|
|
/**
|
|
* The agent provider for the GroqAI provider.
|
|
* We wrap Groq in UnTooled because its tool-calling built in is quite bad and wasteful.
|
|
*/
|
|
class GroqProvider extends InheritMultiple([Provider, UnTooled]) {
|
|
model;
|
|
|
|
constructor(config = {}) {
|
|
const { model = "llama3-8b-8192" } = config;
|
|
super();
|
|
const client = new OpenAI({
|
|
baseURL: "https://api.groq.com/openai/v1",
|
|
apiKey: process.env.GROQ_API_KEY,
|
|
maxRetries: 3,
|
|
});
|
|
|
|
this._client = client;
|
|
this.model = model;
|
|
this.verbose = true;
|
|
}
|
|
|
|
get client() {
|
|
return this._client;
|
|
}
|
|
|
|
get supportsAgentStreaming() {
|
|
return true;
|
|
}
|
|
|
|
async #handleFunctionCallChat({ messages = [] }) {
|
|
return await this.client.chat.completions
|
|
.create({
|
|
model: this.model,
|
|
messages,
|
|
})
|
|
.then((result) => {
|
|
if (!result.hasOwnProperty("choices"))
|
|
throw new Error("GroqAI chat: No results!");
|
|
if (result.choices.length === 0)
|
|
throw new Error("GroqAI chat: No results length!");
|
|
return result.choices[0].message.content;
|
|
})
|
|
.catch((_) => {
|
|
return null;
|
|
});
|
|
}
|
|
|
|
async #handleFunctionCallStream({ messages = [] }) {
|
|
return await this.client.chat.completions.create({
|
|
model: this.model,
|
|
stream: true,
|
|
messages,
|
|
});
|
|
}
|
|
|
|
async stream(messages, functions = [], eventHandler = null) {
|
|
return await UnTooled.prototype.stream.call(
|
|
this,
|
|
messages,
|
|
functions,
|
|
this.#handleFunctionCallStream.bind(this),
|
|
eventHandler
|
|
);
|
|
}
|
|
|
|
async complete(messages, functions = []) {
|
|
return await UnTooled.prototype.complete.call(
|
|
this,
|
|
messages,
|
|
functions,
|
|
this.#handleFunctionCallChat.bind(this)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get the cost of the completion.
|
|
*
|
|
* @param _usage The completion to get the cost for.
|
|
* @returns The cost of the completion.
|
|
* Stubbed since LMStudio has no cost basis.
|
|
*/
|
|
getCost(_usage) {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
module.exports = GroqProvider;
|