From 94673303890327af18ce6eb2550430a37812277e Mon Sep 17 00:00:00 2001 From: Tim Millwood Date: Tue, 18 Feb 2025 11:02:55 +0000 Subject: [PATCH 1/4] Added Cloudflare llm provider --- app/lib/modules/llm/providers/cloudflare.ts | 55 +++++++++++++++++++++ app/lib/modules/llm/registry.ts | 2 + worker-configuration.d.ts | 5 +- 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 app/lib/modules/llm/providers/cloudflare.ts diff --git a/app/lib/modules/llm/providers/cloudflare.ts b/app/lib/modules/llm/providers/cloudflare.ts new file mode 100644 index 00000000..b2b0a907 --- /dev/null +++ b/app/lib/modules/llm/providers/cloudflare.ts @@ -0,0 +1,55 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1 } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; + +export default class CloudflareProvider extends BaseProvider { + name = 'Cloudflare'; + getApiKeyLink = 'https://dash.cloudflare.com/profile/api-tokens'; + + config = { + apiTokenKey: 'CLOUDFLARE_API_TOKEN', + accountIdKey: 'CLOUDFLARE_ACCOUNT_ID', + }; + + staticModels: ModelInfo[] = [ + { name: '@cf/meta/llama-2-7b-chat-int8', label: 'Llama-2-7b-chat-int8', provider: 'Cloudflare', maxTokenAllowed: 4096 }, + { name: '@cf/meta/llama-2-7b-chat-fp16', label: 'Llama-2-7b-chat-fp16', provider: 'Cloudflare', maxTokenAllowed: 4096 }, + { name: '@cf/mistral/mistral-7b-instruct-v0.1', label: 'Mistral-7b-instruct', provider: 'Cloudflare', maxTokenAllowed: 4096 }, + ]; + + getModelInstance(options: { + model: string; + serverEnv: Env; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv, apiKeys, providerSettings } = options; + + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'CLOUDFLARE_API_TOKEN', + }); + + const accountId = serverEnv?.CLOUDFLARE_ACCOUNT_ID || process?.env?.CLOUDFLARE_ACCOUNT_ID; + + if (!apiKey) { + throw new Error(`Missing API token for ${this.name} provider`); + } + + if (!accountId) { + throw new Error(`Missing Account ID for ${this.name} provider`); + } + + const openai = createOpenAI({ + baseURL: `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/v1`, + apiKey, + }); + + return openai(model); + } +} diff --git a/app/lib/modules/llm/registry.ts b/app/lib/modules/llm/registry.ts index 6edba6d8..496688eb 100644 --- a/app/lib/modules/llm/registry.ts +++ b/app/lib/modules/llm/registry.ts @@ -1,5 +1,6 @@ import AnthropicProvider from './providers/anthropic'; import CohereProvider from './providers/cohere'; +import CloudflareProvider from './providers/cloudflare'; import DeepseekProvider from './providers/deepseek'; import GoogleProvider from './providers/google'; import GroqProvider from './providers/groq'; @@ -19,6 +20,7 @@ import GithubProvider from './providers/github'; export { AnthropicProvider, + CloudflareProvider, CohereProvider, DeepseekProvider, GoogleProvider, diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index b2ae1ce7..c61a48c4 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,6 +1,5 @@ interface Env { - RUNNING_IN_DOCKER: Settings; - DEFAULT_NUM_CTX: Settings; + DEFAULT_NUM_CTX:Settings; ANTHROPIC_API_KEY: string; OPENAI_API_KEY: string; GROQ_API_KEY: string; @@ -18,4 +17,6 @@ interface Env { XAI_API_KEY: string; PERPLEXITY_API_KEY: string; AWS_BEDROCK_CONFIG: string; + CLOUDFLARE_API_KEY: string; + CLOUDFLARE_ACCOUNT_ID: string; } From b08b5e1196603b08f286cb6f4dd95f4c274aeb85 Mon Sep 17 00:00:00 2001 From: Tim Millwood <151519+timmillwood@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:42:50 +0000 Subject: [PATCH 2/4] Dynamically get the models --- app/lib/modules/llm/providers/cloudflare.ts | 50 ++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/app/lib/modules/llm/providers/cloudflare.ts b/app/lib/modules/llm/providers/cloudflare.ts index b2b0a907..9eae12d1 100644 --- a/app/lib/modules/llm/providers/cloudflare.ts +++ b/app/lib/modules/llm/providers/cloudflare.ts @@ -13,11 +13,51 @@ export default class CloudflareProvider extends BaseProvider { accountIdKey: 'CLOUDFLARE_ACCOUNT_ID', }; - staticModels: ModelInfo[] = [ - { name: '@cf/meta/llama-2-7b-chat-int8', label: 'Llama-2-7b-chat-int8', provider: 'Cloudflare', maxTokenAllowed: 4096 }, - { name: '@cf/meta/llama-2-7b-chat-fp16', label: 'Llama-2-7b-chat-fp16', provider: 'Cloudflare', maxTokenAllowed: 4096 }, - { name: '@cf/mistral/mistral-7b-instruct-v0.1', label: 'Mistral-7b-instruct', provider: 'Cloudflare', maxTokenAllowed: 4096 }, - ]; + staticModels: ModelInfo[] = []; + + async getDynamicModels( + apiKeys?: Record, + serverEnv: Record = {}, + ): Promise { + const { apiKey } = this.getProviderBaseUrlAndKey({ + apiKeys, + serverEnv: serverEnv as any, + defaultBaseUrlKey: '', + defaultApiTokenKey: 'CLOUDFLARE_API_TOKEN', + }); + + const accountId = serverEnv?.CLOUDFLARE_ACCOUNT_ID || process?.env?.CLOUDFLARE_ACCOUNT_ID; + + if (!apiKey || !accountId) { + return []; + } + + try { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/models/search`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }, + ); + + if (!response.ok) { + throw new Error(`Failed to fetch models: ${response.statusText}`); + } + + const data = (await response.json()) as { result: Array<{ id: string }> }; + return data.result.map((model: any) => ({ + name: model.name, + label: model.name, + provider: this.name, + maxTokenAllowed: 100000, + })); + } catch (error) { + console.error('Error fetching Cloudflare models:', error); + return []; + } + } getModelInstance(options: { model: string; From 2751cb12f5e709206d2da0aef30f565854b03c34 Mon Sep 17 00:00:00 2001 From: Tim Millwood <151519+timmillwood@users.noreply.github.com> Date: Wed, 19 Feb 2025 13:31:54 +0000 Subject: [PATCH 3/4] Fixed worker-configuration.d.ts --- worker-configuration.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index c61a48c4..f30a56f9 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,4 +1,5 @@ interface Env { + RUNNING_IN_DOCKER: Settings; DEFAULT_NUM_CTX:Settings; ANTHROPIC_API_KEY: string; OPENAI_API_KEY: string; From a1c3ba84ca2b70f5b51ea0d796f5e2c47d4e54a6 Mon Sep 17 00:00:00 2001 From: Tim Millwood <151519+timmillwood@users.noreply.github.com> Date: Wed, 19 Feb 2025 13:32:39 +0000 Subject: [PATCH 4/4] Added space back --- worker-configuration.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index f30a56f9..f9044968 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,6 +1,6 @@ interface Env { RUNNING_IN_DOCKER: Settings; - DEFAULT_NUM_CTX:Settings; + DEFAULT_NUM_CTX: Settings; ANTHROPIC_API_KEY: string; OPENAI_API_KEY: string; GROQ_API_KEY: string;