Refines AI limits & retries

src/ai/aiProviderService.ts

@@ -10,7 +10,9 @@ import type { Repository } from '../git/models/repository';
 import { isRepository } from '../git/models/repository';
 import { configuration } from '../system/configuration';
 import type { Storage } from '../system/storage';
+import type { AnthropicModels } from './anthropicProvider';
 import { AnthropicProvider } from './anthropicProvider';
+import type { OpenAIModels } from './openaiProvider';
 import { OpenAIProvider } from './openaiProvider';
 
 export interface AIProvider extends Disposable {
@@ -192,3 +194,41 @@ async function confirmAIProviderToS(provider: AIProvider, storage: Storage): Pro
 
 	return false;
 }
+
+export function getMaxCharacters(model: OpenAIModels | AnthropicModels, outputLength: number): number {
+	const tokensPerCharacter = 3.1;
+
+	let tokens;
+	switch (model) {
+		case 'gpt-4-1106-preview': // 128,000 tokens (4,096 max output tokens)
+			tokens = 128000;
+			break;
+		case 'gpt-4-32k': // 32,768 tokens
+		case 'gpt-4-32k-0613':
+			tokens = 32768;
+			break;
+		case 'gpt-4': // 8,192 tokens
+		case 'gpt-4-0613':
+			tokens = 8192;
+			break;
+		case 'gpt-3.5-turbo-1106': // 16,385 tokens (4,096 max output tokens)
+			tokens = 16385;
+			break;
+		case 'gpt-3.5-turbo-16k': // 16,385 tokens; Will point to gpt-3.5-turbo-1106 starting Dec 11, 2023
+			tokens = 16385;
+			break;
+		case 'gpt-3.5-turbo': // Will point to gpt-3.5-turbo-1106 starting Dec 11, 2023
+			tokens = 4096;
+			break;
+		case 'claude-2': // 100,000 tokens
+		case 'claude-instant-1':
+			tokens = 100000;
+			break;
+		default: // 4,096 tokens
+			tokens = 4096;
+			break;
+	}
+
+	const max = tokens * tokensPerCharacter - outputLength / tokensPerCharacter;
+	return Math.floor(max - max * 0.1);
+}

src/ai/anthropicProvider.ts

@@ -5,7 +5,8 @@ import type { Container } from '../container';
 import { configuration } from '../system/configuration';
 import type { Storage } from '../system/storage';
 import { supportedInVSCodeVersion } from '../system/utils';
-import type { AIProvider } from './aiProviderService';
+import type {AIProvider} from './aiProviderService';
+import {  getMaxCharacters } from './aiProviderService';
 
 export class AnthropicProvider implements AIProvider {
 	readonly id = 'anthropic';
@@ -24,21 +25,18 @@ export class AnthropicProvider implements AIProvider {
 		if (apiKey == null) return undefined;
 
 		const model = this.model;
-		const maxCodeCharacters = getMaxCharacters(model, 1600);
 
-		const code = diff.substring(0, maxCodeCharacters);
-		if (diff.length > maxCodeCharacters) {
-			void window.showWarningMessage(
-				`The diff of the staged changes had to be truncated to ${maxCodeCharacters} characters to fit within the Anthropic's limits.`,
-			);
-		}
+		let retries = 0;
+		let maxCodeCharacters = getMaxCharacters(model, 2600);
+		while (true) {
+			const code = diff.substring(0, maxCodeCharacters);
 
-		let customPrompt = configuration.get('experimental.generateCommitMessagePrompt');
-		if (!customPrompt.endsWith('.')) {
-			customPrompt += '.';
-		}
+			let customPrompt = configuration.get('experimental.generateCommitMessagePrompt');
+			if (!customPrompt.endsWith('.')) {
+				customPrompt += '.';
+			}
 
-		const prompt = `\n\nHuman: You are an advanced AI programming assistant tasked with summarizing code changes into a concise and meaningful commit message. Compose a commit message that:
+			const prompt = `\n\nHuman: You are an advanced AI programming assistant tasked with summarizing code changes into a concise and meaningful commit message. Compose a commit message that:
 - Strictly synthesizes meaningful information from the provided code diff
 - Utilizes any additional user-provided context to comprehend the rationale behind the code changes
 - Is clear and brief, with an informal yet professional tone, and without superfluous descriptions
@@ -62,31 +60,48 @@ Human: ${customPrompt}
 
 Assistant:`;
 
-		const request: AnthropicCompletionRequest = {
-			model: model,
-			prompt: prompt,
-			stream: false,
-			max_tokens_to_sample: 5000,
-			stop_sequences: ['\n\nHuman:'],
-		};
-		const rsp = await this.fetch(apiKey, request);
-		if (!rsp.ok) {
-			let json;
-			try {
-				json = (await rsp.json()) as { error: { type: string; message: string } } | undefined;
-			} catch {}
-
-			debugger;
-			throw new Error(
-				`Unable to generate commit message: (${this.name}:${rsp.status}) ${
-					json?.error.message || rsp.statusText
-				})`,
-			);
-		}
+			const request: AnthropicCompletionRequest = {
+				model: model,
+				prompt: prompt,
+				stream: false,
+				max_tokens_to_sample: 5000,
+				stop_sequences: ['\n\nHuman:'],
+			};
+			const rsp = await this.fetch(apiKey, request);
+			if (!rsp.ok) {
+				let json;
+				try {
+					json = (await rsp.json()) as { error?: { type: string; message: string } } | undefined;
+				} catch {}
+
+				debugger;
+
+				if (
+					retries++ < 2 &&
+					json?.error?.type === 'invalid_request_error' &&
+					json?.error?.message?.includes('prompt is too long')
+				) {
+					maxCodeCharacters -= 500 * retries;
+					continue;
+				}
+
+				throw new Error(
+					`Unable to generate commit message: (${this.name}:${rsp.status}) ${
+						json?.error?.message || rsp.statusText
+					})`,
+				);
+			}
 
-		const data: AnthropicCompletionResponse = await rsp.json();
-		const message = data.completion.trim();
-		return message;
+			if (diff.length > maxCodeCharacters) {
+				void window.showWarningMessage(
+					`The diff of the staged changes had to be truncated to ${maxCodeCharacters} characters to fit within the Anthropic's limits.`,
+				);
+			}
+
+			const data: AnthropicCompletionResponse = await rsp.json();
+			const message = data.completion.trim();
+			return message;
+		}
 	}
 
 	async explainChanges(message: string, diff: string): Promise<string | undefined> {
@@ -94,16 +109,13 @@ Assistant:`;
 		if (apiKey == null) return undefined;
 
 		const model = this.model;
-		const maxCodeCharacters = getMaxCharacters(model, 2400);
 
-		const code = diff.substring(0, maxCodeCharacters);
-		if (diff.length > maxCodeCharacters) {
-			void window.showWarningMessage(
-				`The diff of the commit changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
-			);
-		}
+		let retries = 0;
+		let maxCodeCharacters = getMaxCharacters(model, 3000);
+		while (true) {
+			const code = diff.substring(0, maxCodeCharacters);
 
-		const prompt = `\n\nHuman: You are an advanced AI programming assistant tasked with summarizing code changes into an explanation that is both easy to understand and meaningful. Construct an explanation that:
+			const prompt = `\n\nHuman: You are an advanced AI programming assistant tasked with summarizing code changes into an explanation that is both easy to understand and meaningful. Construct an explanation that:
 			- Concisely synthesizes meaningful information from the provided code diff
 			- Incorporates any additional context provided by the user to understand the rationale behind the code changes
 			- Places the emphasis on the 'why' of the change, clarifying its benefits or addressing the problem that necessitated the change, beyond just detailing the 'what' has changed
@@ -122,30 +134,47 @@ Human: Remember to frame your explanation in a way that is suitable for a review
 
 Assistant:`;
 
-		const request: AnthropicCompletionRequest = {
-			model: model,
-			prompt: prompt,
-			stream: false,
-			max_tokens_to_sample: 5000,
-			stop_sequences: ['\n\nHuman:'],
-		};
-
-		const rsp = await this.fetch(apiKey, request);
-		if (!rsp.ok) {
-			let json;
-			try {
-				json = (await rsp.json()) as { error: { type: string; message: string } } | undefined;
-			} catch {}
-
-			debugger;
-			throw new Error(
-				`Unable to explain commit: (${this.name}:${rsp.status}) ${json?.error.message || rsp.statusText})`,
-			);
-		}
+			const request: AnthropicCompletionRequest = {
+				model: model,
+				prompt: prompt,
+				stream: false,
+				max_tokens_to_sample: 5000,
+				stop_sequences: ['\n\nHuman:'],
+			};
+
+			const rsp = await this.fetch(apiKey, request);
+			if (!rsp.ok) {
+				let json;
+				try {
+					json = (await rsp.json()) as { error?: { type: string; message: string } } | undefined;
+				} catch {}
+
+				debugger;
+
+				if (
+					retries++ < 2 &&
+					json?.error?.type === 'invalid_request_error' &&
+					json?.error?.message?.includes('prompt is too long')
+				) {
+					maxCodeCharacters -= 500 * retries;
+					continue;
+				}
+
+				throw new Error(
+					`Unable to explain commit: (${this.name}:${rsp.status}) ${json?.error?.message || rsp.statusText})`,
+				);
+			}
+
+			if (diff.length > maxCodeCharacters) {
+				void window.showWarningMessage(
+					`The diff of the commit changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
+				);
+			}
 
-		const data: AnthropicCompletionResponse = await rsp.json();
-		const summary = data.completion.trim();
-		return summary;
+			const data: AnthropicCompletionResponse = await rsp.json();
+			const summary = data.completion.trim();
+			return summary;
+		}
 	}
 
 	private fetch(apiKey: string, request: AnthropicCompletionRequest) {
@@ -226,21 +255,6 @@ async function getApiKey(storage: Storage): Promise {
 	return apiKey;
 }
 
-function getMaxCharacters(model: AnthropicModels, outputLength: number): number {
-	let tokens;
-	switch (model) {
-		case 'claude-2': // 100,000 tokens
-		case 'claude-instant-1':
-			tokens = 100000;
-			break;
-		default: // 4,096 tokens
-			tokens = 4096;
-			break;
-	}
-
-	return tokens * 4 - outputLength / 4;
-}
-
 export type AnthropicModels = 'claude-instant-1' | 'claude-2';
 
 interface AnthropicCompletionRequest {

src/ai/openaiProvider.ts

@@ -6,6 +6,7 @@ import { configuration } from '../system/configuration';
 import type { Storage } from '../system/storage';
 import { supportedInVSCodeVersion } from '../system/utils';
 import type { AIProvider } from './aiProviderService';
+import { getMaxCharacters } from './aiProviderService';
 
 export class OpenAIProvider implements AIProvider {
 	readonly id = 'openai';
@@ -28,26 +29,23 @@ export class OpenAIProvider implements AIProvider {
 		if (apiKey == null) return undefined;
 
 		const model = this.model;
-		const maxCodeCharacters = getMaxCharacters(model, 1600);
 
-		const code = diff.substring(0, maxCodeCharacters);
-		if (diff.length > maxCodeCharacters) {
-			void window.showWarningMessage(
-				`The diff of the staged changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
-			);
-		}
+		let retries = 0;
+		let maxCodeCharacters = getMaxCharacters(model, 2600);
+		while (true) {
+			const code = diff.substring(0, maxCodeCharacters);
 
-		let customPrompt = configuration.get('experimental.generateCommitMessagePrompt');
-		if (!customPrompt.endsWith('.')) {
-			customPrompt += '.';
-		}
+			let customPrompt = configuration.get('experimental.generateCommitMessagePrompt');
+			if (!customPrompt.endsWith('.')) {
+				customPrompt += '.';
+			}
 
-		const request: OpenAIChatCompletionRequest = {
-			model: model,
-			messages: [
-				{
-					role: 'system',
-					content: `You are an advanced AI programming assistant tasked with summarizing code changes into a concise and meaningful commit message. Compose a commit message that:
+			const request: OpenAIChatCompletionRequest = {
+				model: model,
+				messages: [
+					{
+						role: 'system',
+						content: `You are an advanced AI programming assistant tasked with summarizing code changes into a concise and meaningful commit message. Compose a commit message that:
 - Strictly synthesizes meaningful information from the provided code diff
 - Utilizes any additional user-provided context to comprehend the rationale behind the code changes
 - Is clear and brief, with an informal yet professional tone, and without superfluous descriptions
@@ -56,40 +54,68 @@ export class OpenAIProvider implements AIProvider {
 - Most importantly emphasizes the 'why' of the change, its benefits, or the problem it addresses rather than only the 'what' that changed
 
 Follow the user's instructions carefully, don't repeat yourself, don't include the code in the output, or make anything up!`,
-				},
-				{
-					role: 'user',
-					content: `Here is the code diff to use to generate the commit message:\n\n${code}`,
-				},
-				...(options?.context
-					? [
-							{
-								role: 'user' as const,
-								content: `Here is additional context which should be taken into account when generating the commit message:\n\n${options.context}`,
-							},
-					  ]
-					: []),
-				{
-					role: 'user',
-					content: customPrompt,
-				},
-			],
-		};
+					},
+					{
+						role: 'user',
+						content: `Here is the code diff to use to generate the commit message:\n\n${code}`,
+					},
+					...(options?.context
+						? [
+								{
+									role: 'user' as const,
+									content: `Here is additional context which should be taken into account when generating the commit message:\n\n${options.context}`,
+								},
+						  ]
+						: []),
+					{
+						role: 'user',
+						content: customPrompt,
+					},
+				],
+			};
+
+			const rsp = await this.fetch(apiKey, request);
+			if (!rsp.ok) {
+				if (rsp.status === 404) {
+					throw new Error(
+						`Unable to generate commit message: Your API key doesn't seem to have access to the selected '${model}' model`,
+					);
+				}
+				if (rsp.status === 429) {
+					throw new Error(
+						`Unable to generate commit message: (${this.name}:${rsp.status}) Too many requests (rate limit exceeded) or your API key is associated with an expired trial`,
+					);
+				}
+
+				let json;
+				try {
+					json = (await rsp.json()) as { error?: { code: string; message: string } } | undefined;
+				} catch {}
+
+				debugger;
+
+				if (retries++ < 2 && json?.error?.code === 'context_length_exceeded') {
+					maxCodeCharacters -= 500 * retries;
+					continue;
+				}
 
-		const rsp = await this.fetch(apiKey, request);
-		if (!rsp.ok) {
-			debugger;
-			if (rsp.status === 429) {
 				throw new Error(
-					`Unable to generate commit message: (${this.name}:${rsp.status}) Too many requests (rate limit exceeded) or your API key is associated with an expired trial`,
+					`Unable to generate commit message: (${this.name}:${rsp.status}) ${
+						json?.error?.message || rsp.statusText
+					}`,
+				);
+			}
+
+			if (diff.length > maxCodeCharacters) {
+				void window.showWarningMessage(
+					`The diff of the staged changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
 				);
 			}
-			throw new Error(`Unable to generate commit message: (${this.name}:${rsp.status}) ${rsp.statusText}`);
-		}
 
-		const data: OpenAIChatCompletionResponse = await rsp.json();
-		const message = data.choices[0].message.content.trim();
-		return message;
+			const data: OpenAIChatCompletionResponse = await rsp.json();
+			const message = data.choices[0].message.content.trim();
+			return message;
+		}
 	}
 
 	async explainChanges(message: string, diff: string): Promise<string | undefined> {
@@ -97,62 +123,80 @@ Follow the user's instructions carefully, don't repeat yourself, don't include t
 		if (apiKey == null) return undefined;
 
 		const model = this.model;
-		const maxCodeCharacters = getMaxCharacters(model, 2400);
-
-		const code = diff.substring(0, maxCodeCharacters);
-		if (diff.length > maxCodeCharacters) {
-			void window.showWarningMessage(
-				`The diff of the commit changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
-			);
-		}
 
-		const request: OpenAIChatCompletionRequest = {
-			model: model,
-			messages: [
-				{
-					role: 'system',
-					content: `You are an advanced AI programming assistant tasked with summarizing code changes into an explanation that is both easy to understand and meaningful. Construct an explanation that:
+		let retries = 0;
+		let maxCodeCharacters = getMaxCharacters(model, 3000);
+		while (true) {
+			const code = diff.substring(0, maxCodeCharacters);
+
+			const request: OpenAIChatCompletionRequest = {
+				model: model,
+				messages: [
+					{
+						role: 'system',
+						content: `You are an advanced AI programming assistant tasked with summarizing code changes into an explanation that is both easy to understand and meaningful. Construct an explanation that:
 - Concisely synthesizes meaningful information from the provided code diff
 - Incorporates any additional context provided by the user to understand the rationale behind the code changes
 - Places the emphasis on the 'why' of the change, clarifying its benefits or addressing the problem that necessitated the change, beyond just detailing the 'what' has changed
 
 Do not make any assumptions or invent details that are not supported by the code diff or the user-provided context.`,
-				},
-				{
-					role: 'user',
-					content: `Here is additional context provided by the author of the changes, which should provide some explanation to why these changes where made. Please strongly consider this information when generating your explanation:\n\n${message}`,
-				},
-				{
-					role: 'user',
-					content: `Now, kindly explain the following code diff in a way that would be clear to someone reviewing or trying to understand these changes:\n\n${code}`,
-				},
-				{
-					role: 'user',
-					content:
-						'Remember to frame your explanation in a way that is suitable for a reviewer to quickly grasp the essence of the changes, the issues they resolve, and their implications on the codebase.',
-				},
-			],
-		};
+					},
+					{
+						role: 'user',
+						content: `Here is additional context provided by the author of the changes, which should provide some explanation to why these changes where made. Please strongly consider this information when generating your explanation:\n\n${message}`,
+					},
+					{
+						role: 'user',
+						content: `Now, kindly explain the following code diff in a way that would be clear to someone reviewing or trying to understand these changes:\n\n${code}`,
+					},
+					{
+						role: 'user',
+						content:
+							'Remember to frame your explanation in a way that is suitable for a reviewer to quickly grasp the essence of the changes, the issues they resolve, and their implications on the codebase.',
+					},
+				],
+			};
+
+			const rsp = await this.fetch(apiKey, request);
+			if (!rsp.ok) {
+				if (rsp.status === 404) {
+					throw new Error(
+						`Unable to explain commit: Your API key doesn't seem to have access to the selected '${model}' model`,
+					);
+				}
+				if (rsp.status === 429) {
+					throw new Error(
+						`Unable to explain commit: (${this.name}:${rsp.status}) Too many requests (rate limit exceeded) or your API key is associated with an expired trial`,
+					);
+				}
+
+				let json;
+				try {
+					json = (await rsp.json()) as { error?: { code: string; message: string } } | undefined;
+				} catch {}
+
+				debugger;
+
+				if (retries++ < 2 && json?.error?.code === 'context_length_exceeded') {
+					maxCodeCharacters -= 500 * retries;
+					continue;
+				}
 
-		const rsp = await this.fetch(apiKey, request);
-		if (!rsp.ok) {
-			debugger;
-			if (rsp.status === 404) {
 				throw new Error(
-					`Unable to explain commit: Your API key doesn't seem to have access to the selected '${model}' model`,
+					`Unable to explain commit: (${this.name}:${rsp.status}) ${json?.error?.message || rsp.statusText}`,
 				);
 			}
-			if (rsp.status === 429) {
-				throw new Error(
-					`Unable to explain commit: (${this.name}:${rsp.status}) Too many requests (rate limit exceeded) or your API key is associated with an expired trial`,
+
+			if (diff.length > maxCodeCharacters) {
+				void window.showWarningMessage(
+					`The diff of the commit changes had to be truncated to ${maxCodeCharacters} characters to fit within the OpenAI's limits.`,
 				);
 			}
-			throw new Error(`Unable to explain commit: (${this.name}:${rsp.status}) ${rsp.statusText}`);
-		}
 
-		const data: OpenAIChatCompletionResponse = await rsp.json();
-		const summary = data.choices[0].message.content.trim();
-		return summary;
+			const data: OpenAIChatCompletionResponse = await rsp.json();
+			const summary = data.choices[0].message.content.trim();
+			return summary;
+		}
 	}
 
 	private fetch(apiKey: string, request: OpenAIChatCompletionRequest) {
@@ -233,35 +277,6 @@ async function getApiKey(storage: Storage): Promise {
 	return openaiApiKey;
 }
 
-function getMaxCharacters(model: OpenAIModels, outputLength: number): number {
-	let tokens;
-	switch (model) {
-		case 'gpt-4-1106-preview': // 128,000 tokens (4,096 max output tokens)
-			tokens = 128000;
-			break;
-		case 'gpt-4-32k': // 32,768 tokens
-		case 'gpt-4-32k-0613':
-			tokens = 32768;
-			break;
-		case 'gpt-4': // 8,192 tokens
-		case 'gpt-4-0613':
-			tokens = 8192;
-			break;
-		case 'gpt-3.5-turbo-1106': // 16,385 tokens (4,096 max output tokens)
-			tokens = 16385;
-			break;
-		case 'gpt-3.5-turbo-16k': // 16,385 tokens; Will point to gpt-3.5-turbo-1106 starting Dec 11, 2023
-			tokens = 16385;
-			break;
-		case 'gpt-3.5-turbo': // Will point to gpt-3.5-turbo-1106 starting Dec 11, 2023
-		default: // 4,096 tokens
-			tokens = 4096;
-			break;
-	}
-
-	return tokens * 4 - outputLength / 4;
-}
-
 export type OpenAIModels =
 	| 'gpt-3.5-turbo-1106'
 	| 'gpt-3.5-turbo'