From ee2c969c6b5dc5f8b823d39b18944c79c50f2185 Mon Sep 17 00:00:00 2001 From: Gauravacad99 <32621234+Gauravacad99@users.noreply.github.com> Date: Fri, 7 Feb 2025 23:02:41 +0530 Subject: [PATCH 1/3] integration of vertex-ai --- app/lib/modules/llm/providers/vertex-ai.ts | 166 +++++++++++++++++++++ app/lib/modules/llm/registry.ts | 3 +- app/types/model.ts | 2 + package.json | 1 + pnpm-lock.yaml | 165 ++++++++++++++++++++ 5 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 app/lib/modules/llm/providers/vertex-ai.ts diff --git a/app/lib/modules/llm/providers/vertex-ai.ts b/app/lib/modules/llm/providers/vertex-ai.ts new file mode 100644 index 00000000..39391791 --- /dev/null +++ b/app/lib/modules/llm/providers/vertex-ai.ts @@ -0,0 +1,166 @@ +import { BaseProvider } from '~/lib/modules/llm/base-provider'; +import type { ModelInfo } from '~/lib/modules/llm/types'; +import type { IProviderSetting } from '~/types/model'; +import type { LanguageModelV1, LanguageModelV1CallOptions } from 'ai'; +import { VertexAI } from '@google-cloud/vertexai'; + +export default class VertexAIProvider extends BaseProvider { + name = 'VertexAI'; + getApiKeyLink = 'https://console.cloud.google.com/'; + + config = { + apiTokenKey: 'GOOGLE_APPLICATION_CREDENTIALS', + projectIdKey: 'GOOGLE_PROJECT_ID', + locationKey: 'GOOGLE_LOCATION', + }; + + staticModels: ModelInfo[] = [ + { + name: 'gemini-pro', + label: 'Gemini Pro', + provider: 'VertexAI', + maxTokenAllowed: 30720, + }, + { + name: 'gemini-pro-vision', + label: 'Gemini Pro Vision', + provider: 'VertexAI', + maxTokenAllowed: 30720, + }, + { + name: 'code-bison', + label: 'Code Bison', + provider: 'VertexAI', + maxTokenAllowed: 6144, + }, + ]; + + async getDynamicModels( + _apiKeys?: Record, + _settings?: IProviderSetting, + _serverEnv: Record = {}, + ): Promise { + // Vertex AI doesn't have a public API to fetch models dynamically + // Return static models instead + return this.staticModels; + } + + +getModelInstance(options: { + model: string; + serverEnv?: any; + apiKeys?: Record; + providerSettings?: Record; + }): LanguageModelV1 { + const { model, serverEnv = {}, apiKeys, providerSettings } = options; + + const { projectId, location } = this.getVertexAIConfig({ + apiKeys, + providerSettings: providerSettings?.[this.name], + serverEnv, + }); + + if (!projectId || !location) { + throw new Error(`Missing configuration for ${this.name} provider`); + } + + const vertexai = new VertexAI({ + project: projectId, + location: location, + }); + + const generativeModel = vertexai.preview.getGenerativeModel({ + model: model, + generationConfig: { + maxOutputTokens: 2048, + temperature: 0.9, + topP: 1, + }, + }); + + // Wrap the Vertex AI model to conform to LanguageModelV1 interface + const instance: LanguageModelV1 = { + specificationVersion: 'v1', + provider: this.name, + modelId: model, + defaultObjectGenerationMode: undefined, + + async doGenerate(options: LanguageModelV1CallOptions) { + const messages = options.prompt.map(msg => { + switch (msg.role) { + case 'system': + return { + role: 'system', + parts: [{ text: msg.content }] + }; + + case 'user': + case 'assistant': + case 'tool': + return { + role: msg.role, + parts: Array.isArray(msg.content) ? msg.content.map(part => { + if ('text' in part) { + return { text: part.text }; + } + throw new Error(`Unsupported content type for Vertex AI`); + }) : [{ text: msg.content }] + }; + + + } + }); + + const response = await generativeModel.generateContent({ + contents: messages, + }); + + if (!response.response?.candidates?.[0]?.content) { + throw new Error('No response generated from Vertex AI'); + } + + return { + text: response.response.candidates[0].content.parts[0].text, + finishReason: 'stop', + usage: { + promptTokens: 0, // Add actual token counts if available from Vertex AI + completionTokens: 0, + }, + rawCall: { + rawPrompt: messages, + rawSettings: {}, + }, + }; + }, + + async doStream(options: LanguageModelV1CallOptions) { + throw new Error('Streaming not implemented for Vertex AI'); + }, + }; + + return instance; + } + + private getVertexAIConfig({ + apiKeys, + providerSettings, + serverEnv, + }: { + apiKeys?: Record; + providerSettings?: IProviderSetting; + serverEnv: Record; + }) { + const projectId = + apiKeys?.GOOGLE_PROJECT_ID || + providerSettings?.projectId || + serverEnv[this.config.projectIdKey]; + + const location = + apiKeys?.GOOGLE_LOCATION || + providerSettings?.location || + serverEnv[this.config.locationKey] || + 'us-central1'; + + return { projectId, location }; + } +} \ No newline at end of file diff --git a/app/lib/modules/llm/registry.ts b/app/lib/modules/llm/registry.ts index 6edba6d8..461e3c53 100644 --- a/app/lib/modules/llm/registry.ts +++ b/app/lib/modules/llm/registry.ts @@ -16,7 +16,7 @@ import XAIProvider from './providers/xai'; import HyperbolicProvider from './providers/hyperbolic'; import AmazonBedrockProvider from './providers/amazon-bedrock'; import GithubProvider from './providers/github'; - +import VertexAIProvider from './providers/vertex-ai'; export { AnthropicProvider, CohereProvider, @@ -36,4 +36,5 @@ export { LMStudioProvider, AmazonBedrockProvider, GithubProvider, + VertexAIProvider }; diff --git a/app/types/model.ts b/app/types/model.ts index d16b10ae..99cf9bed 100644 --- a/app/types/model.ts +++ b/app/types/model.ts @@ -17,6 +17,8 @@ export type ProviderInfo = { export interface IProviderSetting { enabled?: boolean; baseUrl?: string; + projectId?: string; + location?: string; } export type IProviderConfig = ProviderInfo & { diff --git a/package.json b/package.json index c5422779..7ca488e4 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@codemirror/search": "^6.5.8", "@codemirror/state": "^6.4.1", "@codemirror/view": "^6.35.0", + "@google-cloud/vertexai": "^1.9.3", "@iconify-json/ph": "^1.2.1", "@iconify-json/svg-spinners": "^1.2.1", "@lezer/highlight": "^1.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 346f53bb..8c6fb8e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,9 @@ importers: '@codemirror/view': specifier: ^6.35.0 version: 6.35.0 + '@google-cloud/vertexai': + specifier: ^1.9.3 + version: 1.9.3 '@iconify-json/ph': specifier: ^1.2.1 version: 1.2.1 @@ -1458,6 +1461,10 @@ packages: '@floating-ui/utils@0.2.8': resolution: {integrity: sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig==} + '@google-cloud/vertexai@1.9.3': + resolution: {integrity: sha512-35o5tIEMLW3JeFJOaaMNR2e5sq+6rpnhrF97PuAxeOm0GlqVTESKhkGj7a5B5mmJSSSU3hUfIhcQCRRsw4Ipzg==} + engines: {node: '>=18.0.0'} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -2865,6 +2872,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@7.1.3: + resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} + engines: {node: '>= 14'} + aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -2953,6 +2964,9 @@ packages: before-after-hook@3.0.2: resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} + bignumber.js@9.1.2: + resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} + binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -3027,6 +3041,9 @@ packages: buffer-builder@0.2.0: resolution: {integrity: sha512-7VPMEPuYznPSoR21NE1zvd2Xna6c/CloiZCfcMXR1Jny6PjX0N4Nsa38zcBFo/FMK+BlA+FLKbJCQ0i2yxp+Xg==} + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -3395,6 +3412,9 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + editions@6.21.0: resolution: {integrity: sha512-ofkXJtn7z0urokN62DI3SBo/5xAtF0rR7tn+S/bSYV79Ka8pTajIIl+fFQ1q88DQEImymmo97M4azY3WX/nUdg==} engines: {node: '>=4'} @@ -3772,6 +3792,14 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + gaxios@6.7.1: + resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} + engines: {node: '>=14'} + + gcp-metadata@6.1.1: + resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} + engines: {node: '>=14'} + generic-names@4.0.0: resolution: {integrity: sha512-ySFolZQfw9FoDb3ed9d80Cm9f0+r7qj+HJkWjeD9RBfpxEVTlVhol+gvaQB/78WbwYfbnNh8nWHHBSlg072y6A==} @@ -3831,6 +3859,14 @@ packages: globrex@0.1.2: resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==} + google-auth-library@9.15.1: + resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + engines: {node: '>=14'} + + google-logging-utils@0.0.2: + resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} + engines: {node: '>=14'} + gopd@1.1.0: resolution: {integrity: sha512-FQoVQnqcdk4hVM4JN1eromaun4iuS34oStkdlLENLdpULsuQcTyXj8w7ayhuUfPwEYZ1ZOooOTT6fdA9Vmx/RA==} engines: {node: '>= 0.4'} @@ -3841,6 +3877,10 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + gtoken@7.1.0: + resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} + engines: {node: '>=14.0.0'} + gunzip-maybe@1.4.2: resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} hasBin: true @@ -3932,6 +3972,10 @@ packages: https-browserify@1.0.0: resolution: {integrity: sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -4152,6 +4196,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -4188,6 +4235,12 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@2.0.0: + resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==} + + jws@4.0.0: + resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -4705,6 +4758,15 @@ packages: node-fetch-native@1.6.4: resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-fetch@3.3.2: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5791,6 +5853,9 @@ packages: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -6165,6 +6230,12 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which-typed-array@1.1.16: resolution: {integrity: sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==} engines: {node: '>= 0.4'} @@ -7570,6 +7641,13 @@ snapshots: '@floating-ui/utils@0.2.8': {} + '@google-cloud/vertexai@1.9.3': + dependencies: + google-auth-library: 9.15.1 + transitivePeerDependencies: + - encoding + - supports-color + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -9322,6 +9400,8 @@ snapshots: acorn@8.14.0: {} + agent-base@7.1.3: {} + aggregate-error@3.1.0: dependencies: clean-stack: 2.2.0 @@ -9407,6 +9487,8 @@ snapshots: before-after-hook@3.0.2: {} + bignumber.js@9.1.2: {} + binary-extensions@2.3.0: {} binaryextensions@6.11.0: @@ -9521,6 +9603,8 @@ snapshots: buffer-builder@0.2.0: {} + buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} buffer-xor@1.0.3: {} @@ -9849,6 +9933,10 @@ snapshots: eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + editions@6.21.0: dependencies: version-range: 4.14.0 @@ -10351,6 +10439,26 @@ snapshots: function-bind@1.1.2: {} + gaxios@6.7.1: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + is-stream: 2.0.1 + node-fetch: 2.7.0 + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + gcp-metadata@6.1.1: + dependencies: + gaxios: 6.7.1 + google-logging-utils: 0.0.2 + json-bigint: 1.0.0 + transitivePeerDependencies: + - encoding + - supports-color + generic-names@4.0.0: dependencies: loader-utils: 3.3.1 @@ -10407,6 +10515,20 @@ snapshots: globrex@0.1.2: {} + google-auth-library@9.15.1: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 6.7.1 + gcp-metadata: 6.1.1 + gtoken: 7.1.0 + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + + google-logging-utils@0.0.2: {} + gopd@1.1.0: dependencies: get-intrinsic: 1.2.4 @@ -10415,6 +10537,14 @@ snapshots: graphemer@1.4.0: {} + gtoken@7.1.0: + dependencies: + gaxios: 6.7.1 + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + gunzip-maybe@1.4.2: dependencies: browserify-zlib: 0.1.4 @@ -10597,6 +10727,13 @@ snapshots: https-browserify@1.0.0: {} + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.3 + debug: 4.3.7 + transitivePeerDependencies: + - supports-color + human-signals@2.1.0: {} husky@9.1.7: {} @@ -10780,6 +10917,10 @@ snapshots: jsesc@3.0.2: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.1.2 + json-buffer@3.0.1: {} json-parse-even-better-errors@3.0.2: {} @@ -10818,6 +10959,17 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 + jwa@2.0.0: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.0: + dependencies: + jwa: 2.0.0 + safe-buffer: 5.2.1 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -11704,6 +11856,10 @@ snapshots: node-fetch-native@1.6.4: {} + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-fetch@3.3.2: dependencies: data-uri-to-buffer: 4.0.1 @@ -12855,6 +13011,8 @@ snapshots: totalist@3.0.1: {} + tr46@0.0.3: {} + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -13283,6 +13441,13 @@ snapshots: web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which-typed-array@1.1.16: dependencies: available-typed-arrays: 1.0.7 From ee1be90a6408af92853dec078866ab7063662f6f Mon Sep 17 00:00:00 2001 From: Gauravacad99 <32621234+Gauravacad99@users.noreply.github.com> Date: Fri, 7 Feb 2025 23:24:16 +0530 Subject: [PATCH 2/3] using vertex ai apis --- .env.example | 5 + app/lib/modules/llm/providers/vertex-ai.ts | 126 ++++++++-------- package.json | 1 - pnpm-lock.yaml | 165 --------------------- 4 files changed, 66 insertions(+), 231 deletions(-) diff --git a/.env.example b/.env.example index 2d736a72..97b5843a 100644 --- a/.env.example +++ b/.env.example @@ -94,6 +94,11 @@ PERPLEXITY_API_KEY= # {"region": "us-east-1", "accessKeyId": "yourAccessKeyId", "secretAccessKey": "yourSecretAccessKey", "sessionToken": "yourSessionToken"} AWS_BEDROCK_CONFIG= +# Google Cloud / Vertex AI Configuration +GOOGLE_PROJECT_ID= +GOOGLE_LOCATION= # or your preferred region +GOOGLE_ACCESS_TOKEN= + # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug diff --git a/app/lib/modules/llm/providers/vertex-ai.ts b/app/lib/modules/llm/providers/vertex-ai.ts index 39391791..949ca097 100644 --- a/app/lib/modules/llm/providers/vertex-ai.ts +++ b/app/lib/modules/llm/providers/vertex-ai.ts @@ -2,7 +2,9 @@ import { BaseProvider } from '~/lib/modules/llm/base-provider'; import type { ModelInfo } from '~/lib/modules/llm/types'; import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1, LanguageModelV1CallOptions } from 'ai'; -import { VertexAI } from '@google-cloud/vertexai'; + + +// ... rest of the code remains the same ... export default class VertexAIProvider extends BaseProvider { name = 'VertexAI'; @@ -40,13 +42,14 @@ export default class VertexAIProvider extends BaseProvider { _settings?: IProviderSetting, _serverEnv: Record = {}, ): Promise { - // Vertex AI doesn't have a public API to fetch models dynamically - // Return static models instead + /* + * Vertex AI doesn't have a public API to fetch models dynamically + * Return static models instead + */ return this.staticModels; } - -getModelInstance(options: { + getModelInstance(options: { model: string; serverEnv?: any; apiKeys?: Record; @@ -54,7 +57,7 @@ getModelInstance(options: { }): LanguageModelV1 { const { model, serverEnv = {}, apiKeys, providerSettings } = options; - const { projectId, location } = this.getVertexAIConfig({ + const { projectId, location } = this._getVertexAIConfig({ apiKeys, providerSettings: providerSettings?.[this.name], serverEnv, @@ -64,21 +67,6 @@ getModelInstance(options: { throw new Error(`Missing configuration for ${this.name} provider`); } - const vertexai = new VertexAI({ - project: projectId, - location: location, - }); - - const generativeModel = vertexai.preview.getGenerativeModel({ - model: model, - generationConfig: { - maxOutputTokens: 2048, - temperature: 0.9, - topP: 1, - }, - }); - - // Wrap the Vertex AI model to conform to LanguageModelV1 interface const instance: LanguageModelV1 = { specificationVersion: 'v1', provider: this.name, @@ -86,44 +74,58 @@ getModelInstance(options: { defaultObjectGenerationMode: undefined, async doGenerate(options: LanguageModelV1CallOptions) { - const messages = options.prompt.map(msg => { - switch (msg.role) { - case 'system': - return { - role: 'system', - parts: [{ text: msg.content }] - }; - - case 'user': - case 'assistant': - case 'tool': - return { - role: msg.role, - parts: Array.isArray(msg.content) ? msg.content.map(part => { - if ('text' in part) { - return { text: part.text }; - } - throw new Error(`Unsupported content type for Vertex AI`); - }) : [{ text: msg.content }] - }; - - - } + const messages = options.prompt.map((msg) => ({ + role: msg.role, + parts: Array.isArray(msg.content) + ? msg.content.map((part) => { + if ('text' in part) { + return { text: part.text }; + } + throw new Error(`Unsupported content type for Vertex AI`); + }) + : [{ text: msg.content }], + })); + + const endpoint = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${model}:generateContent`; + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKeys?.GOOGLE_ACCESS_TOKEN}`, + }, + body: JSON.stringify({ + contents: messages, + generationConfig: { + maxOutputTokens: 2048, + temperature: 0.9, + topP: 1, + }, + }), }); - - const response = await generativeModel.generateContent({ - contents: messages, - }); - - if (!response.response?.candidates?.[0]?.content) { + + if (!response.ok) { + const error = await response.json() as { error?: { message?: string } }; + throw new Error(`Vertex AI API error: ${error.error?.message || 'Unknown error'}`); + } + + const data = await response.json() as { + candidates?: Array<{ + content: { + parts: Array<{ text: string }>; + }; + }>; + }; + + if (!data.candidates?.[0]?.content) { throw new Error('No response generated from Vertex AI'); } - + return { - text: response.response.candidates[0].content.parts[0].text, + text: data.candidates[0].content.parts[0].text, finishReason: 'stop', usage: { - promptTokens: 0, // Add actual token counts if available from Vertex AI + promptTokens: 0, completionTokens: 0, }, rawCall: { @@ -133,15 +135,15 @@ getModelInstance(options: { }; }, - async doStream(options: LanguageModelV1CallOptions) { + async doStream(_options: LanguageModelV1CallOptions) { throw new Error('Streaming not implemented for Vertex AI'); }, }; return instance; } - - private getVertexAIConfig({ + + private _getVertexAIConfig({ apiKeys, providerSettings, serverEnv, @@ -150,17 +152,11 @@ getModelInstance(options: { providerSettings?: IProviderSetting; serverEnv: Record; }) { - const projectId = - apiKeys?.GOOGLE_PROJECT_ID || - providerSettings?.projectId || - serverEnv[this.config.projectIdKey]; + const projectId = apiKeys?.GOOGLE_PROJECT_ID || providerSettings?.projectId || serverEnv[this.config.projectIdKey]; const location = - apiKeys?.GOOGLE_LOCATION || - providerSettings?.location || - serverEnv[this.config.locationKey] || - 'us-central1'; + apiKeys?.GOOGLE_LOCATION || providerSettings?.location || serverEnv[this.config.locationKey] || 'us-central1'; return { projectId, location }; } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 7ca488e4..c5422779 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "@codemirror/search": "^6.5.8", "@codemirror/state": "^6.4.1", "@codemirror/view": "^6.35.0", - "@google-cloud/vertexai": "^1.9.3", "@iconify-json/ph": "^1.2.1", "@iconify-json/svg-spinners": "^1.2.1", "@lezer/highlight": "^1.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c6fb8e6..346f53bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,9 +80,6 @@ importers: '@codemirror/view': specifier: ^6.35.0 version: 6.35.0 - '@google-cloud/vertexai': - specifier: ^1.9.3 - version: 1.9.3 '@iconify-json/ph': specifier: ^1.2.1 version: 1.2.1 @@ -1461,10 +1458,6 @@ packages: '@floating-ui/utils@0.2.8': resolution: {integrity: sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig==} - '@google-cloud/vertexai@1.9.3': - resolution: {integrity: sha512-35o5tIEMLW3JeFJOaaMNR2e5sq+6rpnhrF97PuAxeOm0GlqVTESKhkGj7a5B5mmJSSSU3hUfIhcQCRRsw4Ipzg==} - engines: {node: '>=18.0.0'} - '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -2872,10 +2865,6 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - agent-base@7.1.3: - resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} - engines: {node: '>= 14'} - aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -2964,9 +2953,6 @@ packages: before-after-hook@3.0.2: resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} - bignumber.js@9.1.2: - resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} - binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -3041,9 +3027,6 @@ packages: buffer-builder@0.2.0: resolution: {integrity: sha512-7VPMEPuYznPSoR21NE1zvd2Xna6c/CloiZCfcMXR1Jny6PjX0N4Nsa38zcBFo/FMK+BlA+FLKbJCQ0i2yxp+Xg==} - buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -3412,9 +3395,6 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - editions@6.21.0: resolution: {integrity: sha512-ofkXJtn7z0urokN62DI3SBo/5xAtF0rR7tn+S/bSYV79Ka8pTajIIl+fFQ1q88DQEImymmo97M4azY3WX/nUdg==} engines: {node: '>=4'} @@ -3792,14 +3772,6 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - gaxios@6.7.1: - resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} - engines: {node: '>=14'} - - gcp-metadata@6.1.1: - resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} - engines: {node: '>=14'} - generic-names@4.0.0: resolution: {integrity: sha512-ySFolZQfw9FoDb3ed9d80Cm9f0+r7qj+HJkWjeD9RBfpxEVTlVhol+gvaQB/78WbwYfbnNh8nWHHBSlg072y6A==} @@ -3859,14 +3831,6 @@ packages: globrex@0.1.2: resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==} - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} - - google-logging-utils@0.0.2: - resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} - engines: {node: '>=14'} - gopd@1.1.0: resolution: {integrity: sha512-FQoVQnqcdk4hVM4JN1eromaun4iuS34oStkdlLENLdpULsuQcTyXj8w7ayhuUfPwEYZ1ZOooOTT6fdA9Vmx/RA==} engines: {node: '>= 0.4'} @@ -3877,10 +3841,6 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} - gunzip-maybe@1.4.2: resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} hasBin: true @@ -3972,10 +3932,6 @@ packages: https-browserify@1.0.0: resolution: {integrity: sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==} - https-proxy-agent@7.0.6: - resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} - engines: {node: '>= 14'} - human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -4196,9 +4152,6 @@ packages: engines: {node: '>=6'} hasBin: true - json-bigint@1.0.0: - resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} - json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -4235,12 +4188,6 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} - jwa@2.0.0: - resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==} - - jws@4.0.0: - resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} - keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -4758,15 +4705,6 @@ packages: node-fetch-native@1.6.4: resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==} - node-fetch@2.7.0: - resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - node-fetch@3.3.2: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5853,9 +5791,6 @@ packages: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} - tr46@0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -6230,12 +6165,6 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} - webidl-conversions@3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - - whatwg-url@5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - which-typed-array@1.1.16: resolution: {integrity: sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==} engines: {node: '>= 0.4'} @@ -7641,13 +7570,6 @@ snapshots: '@floating-ui/utils@0.2.8': {} - '@google-cloud/vertexai@1.9.3': - dependencies: - google-auth-library: 9.15.1 - transitivePeerDependencies: - - encoding - - supports-color - '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -9400,8 +9322,6 @@ snapshots: acorn@8.14.0: {} - agent-base@7.1.3: {} - aggregate-error@3.1.0: dependencies: clean-stack: 2.2.0 @@ -9487,8 +9407,6 @@ snapshots: before-after-hook@3.0.2: {} - bignumber.js@9.1.2: {} - binary-extensions@2.3.0: {} binaryextensions@6.11.0: @@ -9603,8 +9521,6 @@ snapshots: buffer-builder@0.2.0: {} - buffer-equal-constant-time@1.0.1: {} - buffer-from@1.1.2: {} buffer-xor@1.0.3: {} @@ -9933,10 +9849,6 @@ snapshots: eastasianwidth@0.2.0: {} - ecdsa-sig-formatter@1.0.11: - dependencies: - safe-buffer: 5.2.1 - editions@6.21.0: dependencies: version-range: 4.14.0 @@ -10439,26 +10351,6 @@ snapshots: function-bind@1.1.2: {} - gaxios@6.7.1: - dependencies: - extend: 3.0.2 - https-proxy-agent: 7.0.6 - is-stream: 2.0.1 - node-fetch: 2.7.0 - uuid: 9.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - gcp-metadata@6.1.1: - dependencies: - gaxios: 6.7.1 - google-logging-utils: 0.0.2 - json-bigint: 1.0.0 - transitivePeerDependencies: - - encoding - - supports-color - generic-names@4.0.0: dependencies: loader-utils: 3.3.1 @@ -10515,20 +10407,6 @@ snapshots: globrex@0.1.2: {} - google-auth-library@9.15.1: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 - jws: 4.0.0 - transitivePeerDependencies: - - encoding - - supports-color - - google-logging-utils@0.0.2: {} - gopd@1.1.0: dependencies: get-intrinsic: 1.2.4 @@ -10537,14 +10415,6 @@ snapshots: graphemer@1.4.0: {} - gtoken@7.1.0: - dependencies: - gaxios: 6.7.1 - jws: 4.0.0 - transitivePeerDependencies: - - encoding - - supports-color - gunzip-maybe@1.4.2: dependencies: browserify-zlib: 0.1.4 @@ -10727,13 +10597,6 @@ snapshots: https-browserify@1.0.0: {} - https-proxy-agent@7.0.6: - dependencies: - agent-base: 7.1.3 - debug: 4.3.7 - transitivePeerDependencies: - - supports-color - human-signals@2.1.0: {} husky@9.1.7: {} @@ -10917,10 +10780,6 @@ snapshots: jsesc@3.0.2: {} - json-bigint@1.0.0: - dependencies: - bignumber.js: 9.1.2 - json-buffer@3.0.1: {} json-parse-even-better-errors@3.0.2: {} @@ -10959,17 +10818,6 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 - jwa@2.0.0: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - - jws@4.0.0: - dependencies: - jwa: 2.0.0 - safe-buffer: 5.2.1 - keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -11856,10 +11704,6 @@ snapshots: node-fetch-native@1.6.4: {} - node-fetch@2.7.0: - dependencies: - whatwg-url: 5.0.0 - node-fetch@3.3.2: dependencies: data-uri-to-buffer: 4.0.1 @@ -13011,8 +12855,6 @@ snapshots: totalist@3.0.1: {} - tr46@0.0.3: {} - trim-lines@3.0.1: {} trough@2.2.0: {} @@ -13441,13 +13283,6 @@ snapshots: web-streams-polyfill@3.3.3: {} - webidl-conversions@3.0.1: {} - - whatwg-url@5.0.0: - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - which-typed-array@1.1.16: dependencies: available-typed-arrays: 1.0.7 From d39cf87dff7c66487802175719c5fb14070538c6 Mon Sep 17 00:00:00 2001 From: Gauravacad99 <32621234+Gauravacad99@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:17:37 +0530 Subject: [PATCH 3/3] Change in fetching of access keys from env --- app/lib/modules/llm/providers/vertex-ai.ts | 76 +++++++++++++--------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/app/lib/modules/llm/providers/vertex-ai.ts b/app/lib/modules/llm/providers/vertex-ai.ts index 949ca097..f7ee4dde 100644 --- a/app/lib/modules/llm/providers/vertex-ai.ts +++ b/app/lib/modules/llm/providers/vertex-ai.ts @@ -4,14 +4,12 @@ import type { IProviderSetting } from '~/types/model'; import type { LanguageModelV1, LanguageModelV1CallOptions } from 'ai'; -// ... rest of the code remains the same ... - export default class VertexAIProvider extends BaseProvider { name = 'VertexAI'; getApiKeyLink = 'https://console.cloud.google.com/'; config = { - apiTokenKey: 'GOOGLE_APPLICATION_CREDENTIALS', + apiTokenKey: 'GOOGLE_ACCESS_TOKEN', projectIdKey: 'GOOGLE_PROJECT_ID', locationKey: 'GOOGLE_LOCATION', }; @@ -57,16 +55,29 @@ export default class VertexAIProvider extends BaseProvider { }): LanguageModelV1 { const { model, serverEnv = {}, apiKeys, providerSettings } = options; - const { projectId, location } = this._getVertexAIConfig({ + // Get all required credentials using base provider's method + const { apiKey: accessToken, baseUrl: projectId } = this.getProviderBaseUrlAndKey({ apiKeys, providerSettings: providerSettings?.[this.name], serverEnv, + defaultBaseUrlKey: 'GOOGLE_PROJECT_ID', + defaultApiTokenKey: 'GOOGLE_ACCESS_TOKEN', }); - if (!projectId || !location) { - throw new Error(`Missing configuration for ${this.name} provider`); + if (!accessToken) { + throw new Error(`Missing API key for ${this.name} provider`); } + if (!projectId) { + throw new Error(`Missing project ID for ${this.name} provider`); + } + + // Get location from settings or default + const location = apiKeys?.GOOGLE_LOCATION || + providerSettings?.[this.name]?.location || + serverEnv?.GOOGLE_LOCATION || + 'us-central1'; + const instance: LanguageModelV1 = { specificationVersion: 'v1', provider: this.name, @@ -75,7 +86,7 @@ export default class VertexAIProvider extends BaseProvider { async doGenerate(options: LanguageModelV1CallOptions) { const messages = options.prompt.map((msg) => ({ - role: msg.role, + role: msg.role === 'system' ? 'user' : msg.role, parts: Array.isArray(msg.content) ? msg.content.map((part) => { if ('text' in part) { @@ -87,12 +98,12 @@ export default class VertexAIProvider extends BaseProvider { })); const endpoint = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${model}:generateContent`; - + const response = await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKeys?.GOOGLE_ACCESS_TOKEN}`, + Authorization: `Bearer ${accessToken}`, }, body: JSON.stringify({ contents: messages, @@ -105,11 +116,11 @@ export default class VertexAIProvider extends BaseProvider { }); if (!response.ok) { - const error = await response.json() as { error?: { message?: string } }; + const error = (await response.json()) as { error?: { message?: string } }; throw new Error(`Vertex AI API error: ${error.error?.message || 'Unknown error'}`); } - const data = await response.json() as { + const data = (await response.json()) as { candidates?: Array<{ content: { parts: Array<{ text: string }>; @@ -135,28 +146,33 @@ export default class VertexAIProvider extends BaseProvider { }; }, - async doStream(_options: LanguageModelV1CallOptions) { - throw new Error('Streaming not implemented for Vertex AI'); + async doStream(options: LanguageModelV1CallOptions) { + const response = await this.doGenerate(options); + return { + stream: new ReadableStream({ + start(controller) { + if (response.text) { + controller.enqueue({ + type: 'text-delta', + textDelta: response.text, + }); + } + controller.enqueue({ + type: 'finish', + finishReason: response.finishReason, + usage: response.usage, + }); + controller.close(); + }, + }), + rawCall: { + rawPrompt: options.prompt, + rawSettings: {}, + }, + }; }, }; return instance; } - - private _getVertexAIConfig({ - apiKeys, - providerSettings, - serverEnv, - }: { - apiKeys?: Record; - providerSettings?: IProviderSetting; - serverEnv: Record; - }) { - const projectId = apiKeys?.GOOGLE_PROJECT_ID || providerSettings?.projectId || serverEnv[this.config.projectIdKey]; - - const location = - apiKeys?.GOOGLE_LOCATION || providerSettings?.location || serverEnv[this.config.locationKey] || 'us-central1'; - - return { projectId, location }; - } }