From e3e87c1bfd74baf1a5d65e5c111cc2764b510df2 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 18:19:24 +0200 Subject: [PATCH 01/54] add: templates with new improved diff formatting - also remove extra whitespace for token efficiency --- src/templates/codegen-diff-no-plan-prompt.hbs | 373 +++++++---------- src/templates/codegen-diff-prompt.hbs | 378 ++++++++---------- 2 files changed, 310 insertions(+), 441 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 67cf275..d712f4e 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -1,4 +1,4 @@ -You are an expert developer tasked with implementing a given task. Your goal is to write all the code changes needed to complete the task, ensuring it integrates well with the existing codebase and follows best practices. +You are an expert developer tasked with implementing a given task. Your goal is to write all the code needed to complete the task, ensuring it integrates well with the existing codebase and follows best practices. You will be given: - A task description @@ -17,234 +17,169 @@ CODEBASE: Files from the codebase you have access to. FORMAT: Instructions for how to format your response. --- - - {{var_taskDescription}} +{{var_taskDescription}} - --- - - Follow these instructions: - - {{var_instructions}} +Follow these instructions: +{{var_instructions}} - --- - - ## Code Summary - - {{tableOfContents files}} - - ## Selected Files: - {{#each files}} - ### {{relativePath this.path}} - - {{#codeblock this.content this.language}}{{/codeblock}} - - {{/each}} - +## Code Summary +{{tableOfContents files}} + +## Selected Files: +{{#each files}} +### {{relativePath this.path}} +{{#codeblock this.content this.language}}{{/codeblock}} +{{/each}} - --- - - - Generate diffs for modified files, full content for new files, and only the file path for deleted files. - - If you don't need to modify a file, don't include it - this simplifies Git diffs. - - Format your response as follows: - - FILE_PATH_1 - FILE_PATH_2 - ... - - - __GIT_BRANCH_NAME__ - - - - __GIT_COMMIT_MESSAGE__ - - - - __BRIEF_SUMMARY_OF_CHANGES__ - - - - __LIST_OF_POTENTIAL_ISSUES_OR_TRADE_OFFS__ - - - Then, for each file: - - __FILE_PATH__ - __STATUS__ - - __FILE_CONTENT_OR_DIFF__ - - - __EXPLANATION__ (if necessary) - - - - Please adhere to the following guidelines: - - FILE_PATH: Use the full path from the project root. - Example: 'src/components/Button.tsx' - - LANGUAGE: Specify the language or file type. For example: - 'tsx' for .tsx files - 'javascript' for .js files - 'css' for .css files - 'json' for .json files - etc - - FILE_CONTENT_OR_DIFF: - - For new files: Provide the complete file content, including all necessary imports, function definitions, and - exports. - - For modified files: Provide a unified diff format. This MUST include: - 1. File header lines (starting with "---" and "+++") - 2. Hunk headers (starting with "@@") - 3. Context lines (starting with a space) - 4. Removed lines (starting with "-") - 5. Added lines (starting with "+") - - For deleted files: Leave this section empty. - - Ensure proper indentation and follow the project's coding standards. - - STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for - files that are being deleted. - - EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. - - When creating diffs for modified files: - - Always include file header lines with the correct file paths. - - Always include hunk headers with the correct line numbers. - - Use '+' at the beginning of the line to indicate added lines, including new imports. - - Use '-' at the beginning of the line to indicate removed lines. - - Use ' ' (space) at the beginning of the line for unchanged lines (context). - - Include at least 3 lines of unchanged context before and after changes to help with patch application. - - Example of a correct diff format: - - --- src/types/index.ts - +++ src/types/index.ts - @@ -1,4 +1,6 @@ - import type { ParsedDiff } from 'diff'; - +import type { LogLevel } from '../utils/logger'; - + - export interface GitHubIssue { - number: number; - title: string; - @@ -40,6 +42,7 @@ - | 'noCodeblock' - > & { - dryRun: boolean; - + logLevel?: LogLevel; - maxCostThreshold?: number; - task?: string; - description?: string; - - - Before modifying a file, carefully review its entire content. Ensure that your changes, especially new imports, are - placed in the correct location and don't duplicate existing code. - - When generating diffs: - 1. Start with the original file content. - 2. Make your changes, keeping track of line numbers. - 3. Generate the diff by comparing the original and modified versions. - 4. Include at least 3 lines of unchanged context before and after each change. - 5. Verify that the diff accurately represents your intended changes. - - After generating each diff: - - Verify that all new lines (including imports and blank lines) start with '+'. - - Ensure that the line numbers in the diff headers (@@ -old,oldlines +new,newlines @@) are correct and account for - added/removed lines. - - Check that there are sufficient unchanged context lines around modifications. - - Example for a new file: - - src/components/IssueList.tsx - new - - import React from 'react'; - import { Issue } from '../types'; - - interface IssueListProps { - issues: Issue[]; - } - - export const IssueList: React.FC = ({ issues }) => { - return ( -
    - {issues.map((issue) => ( -
  • {issue.title}
    • - ))} -
- ); - }; - - - Created a new IssueList component to display a list of issues. Used React.FC for type-safety and map function for - efficient rendering of multiple issues. - - - - Example for a modified file: - - src/components/App.tsx - modified - - --- src/components/App.tsx - +++ src/components/App.tsx - @@ -1,5 +1,6 @@ - import React from 'react'; - import { Header } from './Header'; - +import { IssueList } from './IssueList'; - - export const App: React.FC = () => { - return ( - @@ -7,6 +8,7 @@ -
-
-
- - {/* TODO: Add issue list */} - + -
-
- ); - - - Updated App component to import and use the new IssueList component. - - - - - Ensure that: - - You have thoroughly analyzed the task description and have planned your implementation strategy. - - Everything specified in the task description and instructions is implemented. - - All new files contain the full code. - - All modified files have accurate and clear diffs in the correct unified diff format. - - Diff formatting across all modified files is consistent and includes all required elements (file headers, hunk headers, context lines). - - The content includes all necessary imports, function definitions, and exports. - - The code is clean, maintainable, and efficient. - - The code is properly formatted and follows the project's coding standards. - - Necessary comments for clarity are included if needed. - - Any conceptual or high-level descriptions are translated into actual, executable code. - - You've considered and handled potential edge cases. - - Your changes are consistent with the existing codebase. - - You haven't introduced any potential bugs or performance issues. - - Your code is easy to understand and maintain. - - You complete all necessary work. - - Note: The accuracy of the diff format is crucial for successful patch application. Even small errors in formatting can - cause the entire patch to fail. Pay extra attention to the correctness of your diff output, ensuring all required - elements (file headers, hunk headers, context lines) are present and correctly formatted. - +Generate SEARCH/REPLACE blocks for modified files, full content for new files, and only the file path for deleted files. + +If you don't need to modify a file, don't include it - this simplifies Git diffs. + +Format your response as follows: + +FILE_PATH_1 +FILE_PATH_2 +... + + +__GIT_BRANCH_NAME__ + + +__GIT_COMMIT_MESSAGE__ + + +__BRIEF_SUMMARY_OF_CHANGES__ + + +__LIST_OF_POTENTIAL_ISSUES_OR_TRADE_OFFS__ + + +Then, for each file: + +__FILE_PATH__ + +__FILE_PATH__ +<<<<<<< SEARCH +__ORIGINAL_CONTENT__ +======= +__MODIFIED_CONTENT__ +>>>>>>> REPLACE + +__STATUS__ + +__EXPLANATION__ (if necessary) + + + +Please adhere to the following guidelines: + +FILE_PATH: Use the full path from the project root. +Example: 'src/components/Button.tsx' + +For modified files: +- Use one or more SEARCH/REPLACE blocks within the tags. +- Each SEARCH/REPLACE block should focus on a specific change. +- You can include multiple SEARCH/REPLACE blocks for a single file if needed. + +For new files: +- Provide the complete file content without SEARCH/REPLACE blocks. + +For deleted files: +- Only include the and deleted. + +STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for files that are being deleted. + +EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. + +Example for a modified file with multiple changes: + +src/components/Button.tsx + +src/components/Button.tsx +<<<<<<< SEARCH +import React from 'react'; + +const Button = ({ text }) => ( + +); +======= +import React from 'react'; + +const Button = ({ text, onClick }) => ( + +); +>>>>>>> REPLACE + +<<<<<<< SEARCH +export default Button; +======= +export default React.memo(Button); +>>>>>>> REPLACE + +modified + +Added onClick prop to make the button interactive and wrapped the component with React.memo for performance optimization. + + + +Example for a new file: + +src/utils/helpers.ts + +import { useState, useEffect } from 'react'; + +export const useDebounce = (value: any, delay: number) => { + const [debouncedValue, setDebouncedValue] = useState(value); + + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedValue(value); + }, delay); + + return () => { + clearTimeout(handler); + }; + }, [value, delay]); + + return debouncedValue; +}; + +new + +Created a new utility hook for debouncing values, useful for optimizing performance in input fields or search functionality. + + + +Example for a deleted file: + +src/deprecated/OldComponent.tsx +deleted + +Removed deprecated component that is no longer used in the project. + + + +Ensure that: +- You have thoroughly analyzed the task and planned your implementation strategy. +- Everything specified in the task description and instructions is implemented. +- All new or modified files contain the full code or necessary changes. +- The content includes all necessary imports, function definitions, and exports. +- The code is clean, maintainable, efficient, and considers performance implications. +- The code is properly formatted and follows the project's coding standards. +- Necessary comments for clarity are included if needed. +- Any conceptual or high-level descriptions are translated into actual, executable code. +- You've considered and handled potential edge cases. +- Your changes are consistent with the existing codebase. +- You haven't introduced any potential bugs or performance issues. +- Your code is easy to understand and maintain. +- You complete all necessary work to fully implement the task. - --- - Now, implement the task described above. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index b08fe06..bbf862e 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -1,5 +1,4 @@ -You are an expert developer tasked with implementing a given task. Your goal is to write all the code changes needed to -complete the task, ensuring it integrates well with the existing codebase and follows best practices. +You are an expert developer tasked with implementing a given task. Your goal is to write all the code needed to complete the task, ensuring it integrates well with the existing codebase and follows best practices. You will be given: - A task description @@ -7,8 +6,7 @@ You will be given: - Instructions - An implementation plan -Use the plan to guide your implementation, but be prepared to make necessary adjustments or improvements. Double-check -your work as you go. +Use the plan to guide your implementation, but be prepared to make necessary adjustments or improvements. Double-check your work as you go. Note: Focus solely on the technical implementation. Ignore any mentions of human tasks or non-technical aspects. @@ -21,235 +19,171 @@ IMPLEMENTATION_PLAN: A detailed implementation plan for the given issue. FORMAT: Instructions for how to format your response. --- - - {{var_taskDescription}} +{{var_taskDescription}} - --- - - Follow these instructions: - - {{var_instructions}} +Follow these instructions: +{{var_instructions}} - --- - - ## Code Summary - - {{tableOfContents files}} - - ## Selected Files: - {{#each files}} - ### {{relativePath this.path}} - - {{#codeblock this.content this.language}}{{/codeblock}} - - {{/each}} - +## Code Summary +{{tableOfContents files}} + +## Selected Files: +{{#each files}} +### {{relativePath this.path}} +{{#codeblock this.content this.language}}{{/codeblock}} +{{/each}} - --- - - {{var_plan}} +{{var_plan}} - --- - - - Generate diffs for modified files, full content for new files, and only the file path for deleted files. - - If you don't need to modify a file, don't include it - this simplifies Git diffs. - - Format your response as follows: - - FILE_PATH_1 - FILE_PATH_2 - ... - - - __GIT_BRANCH_NAME__ - - - - __GIT_COMMIT_MESSAGE__ - - - - __BRIEF_SUMMARY_OF_CHANGES__ - - - - __LIST_OF_POTENTIAL_ISSUES_OR_TRADE_OFFS__ - - - Then, for each file: - - __FILE_PATH__ - __STATUS__ - - __FILE_CONTENT_OR_DIFF__ - - - __EXPLANATION__ (if necessary) - - - - Please adhere to the following guidelines: - - FILE_PATH: Use the full path from the project root. - Example: 'src/components/Button.tsx' - - LANGUAGE: Specify the language or file type. For example: - 'tsx' for .tsx files - 'javascript' for .js files - 'css' for .css files - 'json' for .json files - etc - - FILE_CONTENT_OR_DIFF: - - For new files: Provide the complete file content, including all necessary imports, function definitions, and - exports. - - For modified files: Provide a unified diff format. This MUST include: - 1. File header lines (starting with "---" and "+++") - 2. Hunk headers (starting with "@@") - 3. Context lines (starting with a space) - 4. Removed lines (starting with "-") - 5. Added lines (starting with "+") - - For deleted files: Leave this section empty. - - Ensure proper indentation and follow the project's coding standards. - - STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for - files that are being deleted. - - EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. - - When creating diffs for modified files: - - Always include file header lines with the correct file paths. - - Always include hunk headers with the correct line numbers. - - Use '+' at the beginning of the line to indicate added lines, including new imports. - - Use '-' at the beginning of the line to indicate removed lines. - - Use ' ' (space) at the beginning of the line for unchanged lines (context). - - Include at least 3 lines of unchanged context before and after changes to help with patch application. - - Example of a correct diff format: - - --- src/types/index.ts - +++ src/types/index.ts - @@ -1,4 +1,6 @@ - import type { ParsedDiff } from 'diff'; - +import type { LogLevel } from '../utils/logger'; - + - export interface GitHubIssue { - number: number; - title: string; - @@ -40,6 +42,7 @@ - | 'noCodeblock' - > & { - dryRun: boolean; - + logLevel?: LogLevel; - maxCostThreshold?: number; - task?: string; - description?: string; - - - Before modifying a file, carefully review its entire content. Ensure that your changes, especially new imports, are - placed in the correct location and don't duplicate existing code. - - When generating diffs: - 1. Start with the original file content. - 2. Make your changes, keeping track of line numbers. - 3. Generate the diff by comparing the original and modified versions. - 4. Include at least 3 lines of unchanged context before and after each change. - 5. Verify that the diff accurately represents your intended changes. - - After generating each diff: - - Verify that all new lines (including imports and blank lines) start with '+'. - - Ensure that the line numbers in the diff headers (@@ -old,oldlines +new,newlines @@) are correct and account for - added/removed lines. - - Check that there are sufficient unchanged context lines around modifications. - - Example for a new file: - - src/components/IssueList.tsx - new - - import React from 'react'; - import { Issue } from '../types'; - - interface IssueListProps { - issues: Issue[]; - } - - export const IssueList: React.FC = ({ issues }) => { - return ( -
    - {issues.map((issue) => ( -
  • {issue.title}
    • - ))} -
- ); - }; - - - Created a new IssueList component to display a list of issues. Used React.FC for type-safety and map function for - efficient rendering of multiple issues. - - - - Example for a modified file: - - src/components/App.tsx - modified - - --- src/components/App.tsx - +++ src/components/App.tsx - @@ -1,5 +1,6 @@ - import React from 'react'; - import { Header } from './Header'; - +import { IssueList } from './IssueList'; - - export const App: React.FC = () => { - return ( - @@ -7,6 +8,7 @@ -
-
-
- - {/* TODO: Add issue list */} - + -
-
- ); - - - Updated App component to import and use the new IssueList component. - - - - Ensure that: - - You have thoroughly analyzed the task and plan and have planned your implementation strategy. - - Everything specified in the task description and plan is implemented. - - All new files contain the full code. - - All modified files have accurate and clear diffs in the correct unified diff format. - - Diff formatting across all modified files is consistent and includes all required elements (file headers, hunk headers, context lines). - - The content includes all necessary imports, function definitions, and exports. - - The code is clean, maintainable, and efficient. - - The code is properly formatted and follows the project's coding standards. - - Necessary comments for clarity are included if needed. - - Any conceptual or high-level descriptions are translated into actual, executable code. - - You've considered and handled potential edge cases. - - Your changes are consistent with the existing codebase. - - You haven't introduced any potential bugs or performance issues. - - Your code is easy to understand and maintain. - - You complete all necessary work. - - Note: The accuracy of the diff format is crucial for successful patch application. Even small errors in formatting can - cause the entire patch to fail. Pay extra attention to the correctness of your diff output, ensuring all required - elements (file headers, hunk headers, context lines) are present and correctly formatted. - +Generate SEARCH/REPLACE blocks for modified files, full content for new files, and only the file path for deleted files. + +If you don't need to modify a file, don't include it - this simplifies Git diffs. + +Format your response as follows: + +FILE_PATH_1 +FILE_PATH_2 +... + + +__GIT_BRANCH_NAME__ + + +__GIT_COMMIT_MESSAGE__ + + +__BRIEF_SUMMARY_OF_CHANGES__ + + +__LIST_OF_POTENTIAL_ISSUES_OR_TRADE_OFFS__ + + +Then, for each file: + +__FILE_PATH__ + +__FILE_PATH__ +<<<<<<< SEARCH +__ORIGINAL_CONTENT__ +======= +__MODIFIED_CONTENT__ +>>>>>>> REPLACE + +__STATUS__ + +__EXPLANATION__ (if necessary) + + + +Please adhere to the following guidelines: + +FILE_PATH: Use the full path from the project root. +Example: 'src/components/Button.tsx' + +For modified files: +- Use one or more SEARCH/REPLACE blocks within the tags. +- Each SEARCH/REPLACE block should focus on a specific change. +- You can include multiple SEARCH/REPLACE blocks for a single file if needed. + +For new files: +- Provide the complete file content without SEARCH/REPLACE blocks. + +For deleted files: +- Only include the and deleted. + +STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for files that are being deleted. + +EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. + +Example for a modified file with multiple changes: + +src/components/Button.tsx + +src/components/Button.tsx +<<<<<<< SEARCH +import React from 'react'; + +const Button = ({ text }) => ( + +); +======= +import React from 'react'; + +const Button = ({ text, onClick }) => ( + +); +>>>>>>> REPLACE + +<<<<<<< SEARCH +export default Button; +======= +export default React.memo(Button); +>>>>>>> REPLACE + +modified + +Added onClick prop to make the button interactive and wrapped the component with React.memo for performance optimization. + + + +Example for a new file: + +src/utils/helpers.ts + +import { useState, useEffect } from 'react'; + +export const useDebounce = (value: any, delay: number) => { + const [debouncedValue, setDebouncedValue] = useState(value); + + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedValue(value); + }, delay); + + return () => { + clearTimeout(handler); + }; + }, [value, delay]); + + return debouncedValue; +}; + +new + +Created a new utility hook for debouncing values, useful for optimizing performance in input fields or search functionality. + + + +Example for a deleted file: + +src/deprecated/OldComponent.tsx +deleted + +Removed deprecated component that is no longer used in the project. + + + +Ensure that: +- You have thoroughly analyzed the task and plan and have planned your implementation strategy. +- Everything specified in the task description and plan is implemented. +- All new or modified files contain the full code or necessary changes. +- The content includes all necessary imports, function definitions, and exports. +- The code is clean, maintainable, efficient, and considers performance implications. +- The code is properly formatted and follows the project's coding standards. +- Necessary comments for clarity are included if needed. +- Any conceptual or high-level descriptions are translated into actual, executable code. +- You've considered and handled potential edge cases. +- Your changes are consistent with the existing codebase. +- You haven't introduced any potential bugs or performance issues. +- Your code is easy to understand and maintain. +- You complete all necessary work to fully implement the task. From f25db88b7fc07e1b4601c3946e7dbb6c54dbc370 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 18:24:24 +0200 Subject: [PATCH 02/54] add: new parse codegen response logic --- src/ai/parse-ai-codegen-response.ts | 107 ++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/src/ai/parse-ai-codegen-response.ts b/src/ai/parse-ai-codegen-response.ts index 849bfb4..bc8038a 100644 --- a/src/ai/parse-ai-codegen-response.ts +++ b/src/ai/parse-ai-codegen-response.ts @@ -1,19 +1,13 @@ -import type { AIParsedResponse } from '../types'; +import { detectLanguage } from '../core/file-worker'; +import type { AIFileChange, AIFileInfo, AIParsedResponse } from '../types'; import getLogger from '../utils/logger'; -import { - isResponseMalformed, - parseCommonFields, -} from './parsers/common-parser'; -import { parseDiffFiles } from './parsers/diff-parser'; -import { parseFullContentFiles } from './parsers/full-content-parser'; export function parseAICodegenResponse( response: string, logAiInteractions = false, - useDiffMode = false, ): AIParsedResponse { const logger = getLogger(logAiInteractions); - let result: AIParsedResponse = { + const result: AIParsedResponse = { fileList: [], files: [], gitBranchName: '', @@ -23,12 +17,40 @@ export function parseAICodegenResponse( }; try { - const commonFields = parseCommonFields(response); - result = { ...result, ...commonFields }; + // Extract file list + const fileListMatch = response.match(/([\s\S]*?)<\/file_list>/); + result.fileList = fileListMatch + ? fileListMatch[1] + .trim() + .split('\n') + .map((file) => file.trim()) + : []; - result.files = useDiffMode - ? parseDiffFiles(response) - : parseFullContentFiles(response); + // Extract git branch name + const branchMatch = response.match( + /([\s\S]*?)<\/git_branch_name>/, + ); + result.gitBranchName = branchMatch ? branchMatch[1].trim() : ''; + + // Extract git commit message + const commitMatch = response.match( + /([\s\S]*?)<\/git_commit_message>/, + ); + result.gitCommitMessage = commitMatch ? commitMatch[1].trim() : ''; + + // Extract summary + const summaryMatch = response.match(/([\s\S]*?)<\/summary>/); + result.summary = summaryMatch ? summaryMatch[1].trim() : ''; + + // Extract potential issues + const issuesMatch = response.match( + /([\s\S]*?)<\/potential_issues>/, + ); + result.potentialIssues = issuesMatch ? issuesMatch[1].trim() : ''; + + // Extract file information + const fileBlocks = response.match(/[\s\S]*?<\/file>/g) || []; + result.files = fileBlocks.map((block) => parseFileBlock(block)); if (isResponseMalformed(result)) { throw new Error('Malformed response'); @@ -42,3 +64,60 @@ export function parseAICodegenResponse( return result; } + +function parseFileBlock(block: string): AIFileInfo { + const pathMatch = block.match(/(.*?)<\/file_path>/); + const statusMatch = block.match(/(.*?)<\/file_status>/); + const contentMatch = block.match(/([\s\S]*?)<\/file_content>/); + const explanationMatch = block.match( + /([\s\S]*?)<\/explanation>/, + ); + + if (!pathMatch || !statusMatch || !contentMatch) { + throw new Error('Invalid file block format'); + } + + const path = pathMatch[1].trim(); + const status = statusMatch[1].trim() as 'new' | 'modified' | 'deleted'; + const contentBlock = contentMatch[1].trim(); + const explanation = explanationMatch ? explanationMatch[1].trim() : undefined; + + let content: string | undefined; + let changes: AIFileChange[] | undefined; + + if (status === 'modified') { + changes = parseSearchReplaceBlocks(contentBlock); + } else if (status === 'new') { + content = contentBlock; + } + + return { + path, + status, + language: detectLanguage(path), + content, + changes, + explanation, + }; +} + +function parseSearchReplaceBlocks(content: string): AIFileChange[] { + const blocks = content.split(/(?=<<<<<<< SEARCH)/); + return blocks + .map((block) => { + const [search, replace] = block.split(/^=======\s*$/m); + return { + search: search.replace(/^<<<<<<< SEARCH\s*\n/, '').trim(), + replace: replace.replace(/\s*>>>>>>> REPLACE.*?$/, '').trim(), + }; + }) + .filter((change) => change.search && change.replace); +} + +function isResponseMalformed(result: AIParsedResponse): boolean { + return ( + result.fileList.length === 0 && + result.files.length === 0 && + result.gitCommitMessage === '' + ); +} From 35f4380fd681cdae6352bf53297620576d2e02e7 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 18:41:45 +0200 Subject: [PATCH 03/54] add: new apply-changes logic for new diff format --- src/git/apply-changes.ts | 53 +++++++++++++++++++++++++--------------- src/types/index.ts | 14 +++++++++++ 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index 9f5a49a..4525c00 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -1,6 +1,5 @@ import path from 'node:path'; import chalk from 'chalk'; -import { applyPatch } from 'diff'; import fs from 'fs-extra'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; @@ -53,34 +52,27 @@ async function applyFileChange( console.log( chalk.yellow(`[DRY RUN] Would modify file: ${file.path}`), ); - if (file.diff) { + file.changes?.forEach((change, index) => { console.log( chalk.gray( - `[DRY RUN] Diff preview:\n${JSON.stringify(file.diff, null, 2)}`, + `[DRY RUN] Change ${index + 1} preview:\nSearch:\n${change.search.substring(0, 100)}${change.search.length > 100 ? '...' : ''}\nReplace:\n${change.replace.substring(0, 100)}${change.replace.length > 100 ? '...' : ''}`, ), ); - } else if (file.content) { - console.log( - chalk.gray( - `[DRY RUN] Content preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, - ), - ); - } + }); } else { - if (file.diff) { - const currentContent = await fs.readFile(fullPath, 'utf-8'); - const updatedContent = applyPatch(currentContent, file.diff); - if (updatedContent === false) { - throw new Error( - `Failed to apply patch to file: ${file.path}\nA common cause is that the file was not sent to the LLM and it hallucinated the content. Try running the task again (task --redo) and selecting the problematic file.`, + if (file.changes && file.changes.length > 0) { + let currentContent = await fs.readFile(fullPath, 'utf-8'); + for (const change of file.changes) { + currentContent = applyChange( + currentContent, + change.search, + change.replace, ); } - await fs.writeFile(fullPath, updatedContent); - } else if (file.content) { - await fs.writeFile(fullPath, file.content); + await fs.writeFile(fullPath, currentContent); } else { throw new Error( - `No content or diff provided for modified file: ${file.path}`, + `No changes provided for modified file: ${file.path}`, ); } console.log(chalk.green(`Modified file: ${file.path}`)); @@ -102,3 +94,24 @@ async function applyFileChange( throw error; } } + +function applyChange(content: string, search: string, replace: string): string { + const trimmedSearch = search.trim(); + const trimmedReplace = replace.trim(); + + if (content.includes(trimmedSearch)) { + return content.replace(trimmedSearch, trimmedReplace); + } + + // If exact match fails, try matching with flexible whitespace + const flexibleSearch = trimmedSearch.replace(/\s+/g, '\\s+'); + const regex = new RegExp(flexibleSearch, 'g'); + + if (regex.test(content)) { + return content.replace(regex, trimmedReplace); + } + + throw new Error( + 'Failed to apply changes: search content not found in the file', + ); +} diff --git a/src/types/index.ts b/src/types/index.ts index ca73eb4..31b3005 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -168,3 +168,17 @@ export interface TaskData { timestamp: number; model: string; } + +export interface AIFileChange { + search: string; + replace: string; +} + +export interface AIFileInfo { + path: string; + language: string; + content?: string; + changes?: AIFileChange[]; + status: 'new' | 'modified' | 'deleted'; + explanation?: string; +} From 33c8c0b4adbb60392124620e6f94358baabcd5fa Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 18:41:57 +0200 Subject: [PATCH 04/54] test: test the new diff mode implementation --- tests/unit/apply-changes.test.ts | 188 +++++++++++-------------------- 1 file changed, 63 insertions(+), 125 deletions(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 3a8254f..61b8b04 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -6,8 +6,6 @@ import { applyChanges } from '../../src/git/apply-changes'; import type { AIParsedResponse } from '../../src/types'; vi.mock('fs-extra'); -vi.mock('simple-git'); -vi.mock('../../src/utils/git-tools'); vi.mock('diff', async () => { const actual = await vi.importActual('diff'); return { @@ -27,9 +25,9 @@ describe('applyChanges', () => { vi.clearAllMocks(); }); - it('should create new files and modify existing ones', async () => { + it('should create new files, modify existing ones with multiple changes, and delete files', async () => { const mockParsedResponse: AIParsedResponse = { - fileList: ['new-file.js', 'existing-file.js'], + fileList: ['new-file.js', 'existing-file.js', 'deleted-file.js'], files: [ { path: 'new-file.js', @@ -40,16 +38,35 @@ describe('applyChanges', () => { { path: 'existing-file.js', language: 'javascript', - content: 'console.log("Modified file");', + changes: [ + { + search: 'console.log("Old content");', + replace: 'console.log("Modified content");', + }, + { + search: 'function oldFunction() {}', + replace: 'function newFunction() {}', + }, + ], status: 'modified', }, + { + path: 'deleted-file.js', + language: 'javascript', + status: 'deleted', + }, ], - gitBranchName: 'feature/new-branch', - gitCommitMessage: 'Add and modify files', - summary: 'Created a new file and modified an existing one', + gitBranchName: 'feature/mixed-changes', + gitCommitMessage: 'Add, modify, and delete files', + summary: + 'Created a new file, modified an existing one with multiple changes, and deleted a file', potentialIssues: 'None', }; + vi.mocked(fs.readFile).mockResolvedValue( + 'console.log("Old content");\nfunction oldFunction() {}', + ); + await applyChanges({ basePath: mockBasePath, parsedResponse: mockParsedResponse, @@ -58,11 +75,19 @@ describe('applyChanges', () => { expect(fs.ensureDir).toHaveBeenCalledTimes(1); expect(fs.writeFile).toHaveBeenCalledTimes(2); + expect(fs.readFile).toHaveBeenCalledTimes(1); + expect(fs.remove).toHaveBeenCalledTimes(1); + + // Check if the content of the modified file is correct + expect(fs.writeFile).toHaveBeenCalledWith( + expect.stringContaining('existing-file.js'), + 'console.log("Modified content");\nfunction newFunction() {}', + ); }); it('should not apply changes in dry run mode', async () => { const mockParsedResponse: AIParsedResponse = { - fileList: ['new-file.js'], + fileList: ['new-file.js', 'existing-file.js', 'deleted-file.js'], files: [ { path: 'new-file.js', @@ -70,6 +95,22 @@ describe('applyChanges', () => { content: 'console.log("New file");', status: 'new', }, + { + path: 'existing-file.js', + language: 'javascript', + changes: [ + { + search: 'console.log("Old content");', + replace: 'console.log("Modified content");', + }, + ], + status: 'modified', + }, + { + path: 'deleted-file.js', + language: 'javascript', + status: 'deleted', + }, ], gitBranchName: 'feature/dry-run', gitCommitMessage: 'This should not be committed', @@ -85,6 +126,8 @@ describe('applyChanges', () => { expect(fs.ensureDir).not.toHaveBeenCalled(); expect(fs.writeFile).not.toHaveBeenCalled(); + expect(fs.readFile).not.toHaveBeenCalled(); + expect(fs.remove).not.toHaveBeenCalled(); }); it('should handle errors during file operations', async () => { @@ -117,67 +160,6 @@ describe('applyChanges', () => { ).rejects.toThrow('Failed to create directory'); }); - it('should handle mixed operations (create, modify, delete)', async () => { - const mockParsedResponse: AIParsedResponse = { - fileList: ['new-file.js', 'modified-file.js', 'deleted-file.js'], - files: [ - { - path: 'new-file.js', - language: 'javascript', - content: 'console.log("New file");', - status: 'new', - }, - { - path: 'modified-file.js', - language: 'javascript', - content: 'console.log("Modified file");', - status: 'modified', - }, - { - path: 'deleted-file.js', - language: '', - content: '', - status: 'deleted', - }, - ], - gitBranchName: 'feature/mixed-changes', - gitCommitMessage: 'Mixed file operations', - summary: 'Created, modified, and deleted files', - potentialIssues: 'None', - }; - - await applyChanges({ - basePath: mockBasePath, - parsedResponse: mockParsedResponse, - dryRun: false, - }); - - expect(fs.ensureDir).toHaveBeenCalledTimes(1); - expect(fs.writeFile).toHaveBeenCalledTimes(2); - expect(fs.remove).toHaveBeenCalledTimes(1); - }); - - it('should handle empty file list', async () => { - const mockParsedResponse: AIParsedResponse = { - fileList: [], - files: [], - gitBranchName: 'feature/no-changes', - gitCommitMessage: 'No changes', - summary: 'No files were changed', - potentialIssues: 'None', - }; - - await applyChanges({ - basePath: mockBasePath, - parsedResponse: mockParsedResponse, - dryRun: false, - }); - - expect(fs.ensureDir).not.toHaveBeenCalled(); - expect(fs.writeFile).not.toHaveBeenCalled(); - expect(fs.remove).not.toHaveBeenCalled(); - }); - it('should handle file path with subdirectories', async () => { const mockParsedResponse: AIParsedResponse = { fileList: ['deep/nested/file.js'], @@ -210,68 +192,24 @@ describe('applyChanges', () => { ); }); - it('should apply diffs for modified files when provided', async () => { - const mockBasePath = '/mock/base/path'; - const mockParsedResponse = { - fileList: ['modified-file.js'], - files: [ - { - path: 'modified-file.js', - language: 'javascript', - diff: { - oldFileName: 'modified-file.js', - newFileName: 'modified-file.js', - hunks: [ - { - oldStart: 1, - oldLines: 1, - newStart: 1, - newLines: 1, - lines: [ - '-console.log("Old content");', - '+console.log("New content");', - ], - }, - ], - }, - status: 'modified', - }, - ], - gitBranchName: 'feature/diff-modification', - gitCommitMessage: 'Apply diff to file', - summary: 'Modified a file using diff', + it('should handle empty file list', async () => { + const mockParsedResponse: AIParsedResponse = { + fileList: [], + files: [], + gitBranchName: 'feature/no-changes', + gitCommitMessage: 'No changes', + summary: 'No files were changed', potentialIssues: 'None', }; - const oldContent = 'console.log("Old content");'; - const newContent = 'console.log("New content");'; - - // biome-ignore lint/suspicious/noExplicitAny: explicit any is fine here, we're mocking fs.readFile - vi.mocked(fs.readFile).mockResolvedValue(oldContent as any); - vi.mocked(applyPatch).mockReturnValue(newContent); - await applyChanges({ basePath: mockBasePath, - parsedResponse: mockParsedResponse as AIParsedResponse, + parsedResponse: mockParsedResponse, dryRun: false, }); - expect(fs.readFile).toHaveBeenCalledTimes(1); - expect(fs.readFile).toHaveBeenCalledWith( - expect.stringContaining('modified-file.js'), - 'utf-8', - ); - - expect(applyPatch).toHaveBeenCalledTimes(1); - expect(applyPatch).toHaveBeenCalledWith( - oldContent, - mockParsedResponse.files[0].diff, - ); - - expect(fs.writeFile).toHaveBeenCalledTimes(1); - expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining('modified-file.js'), - newContent, - ); + expect(fs.ensureDir).not.toHaveBeenCalled(); + expect(fs.writeFile).not.toHaveBeenCalled(); + expect(fs.remove).not.toHaveBeenCalled(); }); }); From 4e60b2b628e6c2de8b395e6fe8df01175c3817ea Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 18:46:24 +0200 Subject: [PATCH 05/54] fix: task-workflow and its corresponding test --- src/ai/task-workflow.ts | 1 - tests/unit/task-workflow.test.ts | 69 +++++++++++++++++--------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index 57ab926..2541b59 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -409,7 +409,6 @@ async function generateAndApplyCode( const parsedResponse = parseAICodegenResponse( generatedCode, options.logAiInteractions, - options.diff, ); if (options.dryRun) { diff --git a/tests/unit/task-workflow.test.ts b/tests/unit/task-workflow.test.ts index d7a2732..be11e3a 100644 --- a/tests/unit/task-workflow.test.ts +++ b/tests/unit/task-workflow.test.ts @@ -339,20 +339,20 @@ describe('runAIAssistedTask', () => { ); }); - it('should handle diff-based updates when --diff flag is used', async () => { + it('should handle multiple changes when --diff flag is used', async () => { const diffOptions = { ...mockOptions, model: 'claude-3-sonnet-20240229', diff: true, }; - const mockTaskDescription = 'Test diff-based task'; - const mockInstructions = 'Test diff-based instructions'; + const mockTaskDescription = 'Test multiple changes task'; + const mockInstructions = 'Test multiple changes instructions'; const mockSelectedFiles = ['file1.ts']; const mockProcessedFiles = [ { path: 'file1.ts', - content: 'original content', + content: 'original content\nmore original content', language: 'typescript', size: 100, created: new Date(), @@ -360,8 +360,8 @@ describe('runAIAssistedTask', () => { extension: 'ts', }, ]; - const mockGeneratedPlan = 'Generated diff-based plan'; - const mockReviewedPlan = 'Reviewed diff-based plan'; + const mockGeneratedPlan = 'Generated multiple changes plan'; + const mockReviewedPlan = 'Reviewed multiple changes plan'; const mockGeneratedCode = ` file1.ts @@ -369,45 +369,49 @@ describe('runAIAssistedTask', () => { file1.ts modified - - --- file1.ts - +++ file1.ts - @@ -1 +1 @@ - -original content - +updated content + + file1.ts + <<<<<<< SEARCH + original content + ======= + updated content + >>>>>>> REPLACE + + <<<<<<< SEARCH + more original content + ======= + additional updated content + >>>>>>> REPLACE - feature/diff-task - Implement diff-based task - Updated file using diff + feature/multiple-changes-task + Implement multiple changes task + Updated file with multiple changes None `; const mockParsedResponse = { - gitBranchName: 'feature/diff-task', - gitCommitMessage: 'Implement diff-based task', + gitBranchName: 'feature/multiple-changes-task', + gitCommitMessage: 'Implement multiple changes task', fileList: ['file1.ts'], files: [ { path: 'file1.ts', language: 'typescript', status: 'modified', - diff: { - oldFileName: 'file1.ts', - newFileName: 'file1.ts', - hunks: [ - { - oldStart: 1, - oldLines: 1, - newStart: 1, - newLines: 1, - lines: ['-original content', '+updated content'], - }, - ], - }, + changes: [ + { + search: 'original content', + replace: 'updated content', + }, + { + search: 'more original content', + replace: 'additional updated content', + }, + ], }, ], - summary: 'Updated file using diff', + summary: 'Updated file with multiple changes', potentialIssues: 'None', }; @@ -434,7 +438,7 @@ describe('runAIAssistedTask', () => { vi.mocked(parseAICodegenResponse).mockReturnValue( mockParsedResponse as unknown as AIParsedResponse, ); - vi.mocked(ensureBranch).mockResolvedValue('feature/diff-task'); + vi.mocked(ensureBranch).mockResolvedValue('feature/multiple-changes-task'); vi.mocked(applyChanges).mockResolvedValue(); vi.mocked(getModelConfig).mockReturnValue(mockModelConfig); @@ -443,7 +447,6 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, - true, ); expect(applyChanges).toHaveBeenCalledWith( expect.objectContaining({ From a00a1bc0a55c1339bd06d629968b831232719719 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 21:53:42 +0200 Subject: [PATCH 06/54] wip: new diff mode --- src/ai/parse-ai-codegen-response.ts | 109 +----- src/ai/parsers/search-replace-parser.ts | 44 +++ src/ai/task-workflow.ts | 1 + src/git/apply-changes.ts | 40 +- src/templates/codegen-diff-no-plan-prompt.hbs | 2 - src/templates/codegen-diff-prompt.hbs | 2 - src/types/index.ts | 1 + tests/unit/parse-ai-codegen-response.test.ts | 358 ++++-------------- tests/unit/task-workflow.test.ts | 2 - 9 files changed, 180 insertions(+), 379 deletions(-) create mode 100644 src/ai/parsers/search-replace-parser.ts diff --git a/src/ai/parse-ai-codegen-response.ts b/src/ai/parse-ai-codegen-response.ts index bc8038a..1d2a442 100644 --- a/src/ai/parse-ai-codegen-response.ts +++ b/src/ai/parse-ai-codegen-response.ts @@ -1,13 +1,19 @@ -import { detectLanguage } from '../core/file-worker'; -import type { AIFileChange, AIFileInfo, AIParsedResponse } from '../types'; +import type { AIParsedResponse } from '../types'; import getLogger from '../utils/logger'; +import { + isResponseMalformed, + parseCommonFields, +} from './parsers/common-parser'; +import { parseFullContentFiles } from './parsers/full-content-parser'; +import { parseSearchReplaceFiles } from './parsers/search-replace-parser'; export function parseAICodegenResponse( response: string, logAiInteractions = false, + useDiffMode = false, ): AIParsedResponse { const logger = getLogger(logAiInteractions); - const result: AIParsedResponse = { + let result: AIParsedResponse = { fileList: [], files: [], gitBranchName: '', @@ -17,40 +23,14 @@ export function parseAICodegenResponse( }; try { - // Extract file list - const fileListMatch = response.match(/([\s\S]*?)<\/file_list>/); - result.fileList = fileListMatch - ? fileListMatch[1] - .trim() - .split('\n') - .map((file) => file.trim()) - : []; + const commonFields = parseCommonFields(response); + result = { ...result, ...commonFields }; - // Extract git branch name - const branchMatch = response.match( - /([\s\S]*?)<\/git_branch_name>/, - ); - result.gitBranchName = branchMatch ? branchMatch[1].trim() : ''; - - // Extract git commit message - const commitMatch = response.match( - /([\s\S]*?)<\/git_commit_message>/, - ); - result.gitCommitMessage = commitMatch ? commitMatch[1].trim() : ''; - - // Extract summary - const summaryMatch = response.match(/([\s\S]*?)<\/summary>/); - result.summary = summaryMatch ? summaryMatch[1].trim() : ''; - - // Extract potential issues - const issuesMatch = response.match( - /([\s\S]*?)<\/potential_issues>/, - ); - result.potentialIssues = issuesMatch ? issuesMatch[1].trim() : ''; - - // Extract file information - const fileBlocks = response.match(/[\s\S]*?<\/file>/g) || []; - result.files = fileBlocks.map((block) => parseFileBlock(block)); + if (useDiffMode) { + result.files = parseSearchReplaceFiles(response); + } else { + result.files = parseFullContentFiles(response); + } if (isResponseMalformed(result)) { throw new Error('Malformed response'); @@ -64,60 +44,3 @@ export function parseAICodegenResponse( return result; } - -function parseFileBlock(block: string): AIFileInfo { - const pathMatch = block.match(/(.*?)<\/file_path>/); - const statusMatch = block.match(/(.*?)<\/file_status>/); - const contentMatch = block.match(/([\s\S]*?)<\/file_content>/); - const explanationMatch = block.match( - /([\s\S]*?)<\/explanation>/, - ); - - if (!pathMatch || !statusMatch || !contentMatch) { - throw new Error('Invalid file block format'); - } - - const path = pathMatch[1].trim(); - const status = statusMatch[1].trim() as 'new' | 'modified' | 'deleted'; - const contentBlock = contentMatch[1].trim(); - const explanation = explanationMatch ? explanationMatch[1].trim() : undefined; - - let content: string | undefined; - let changes: AIFileChange[] | undefined; - - if (status === 'modified') { - changes = parseSearchReplaceBlocks(contentBlock); - } else if (status === 'new') { - content = contentBlock; - } - - return { - path, - status, - language: detectLanguage(path), - content, - changes, - explanation, - }; -} - -function parseSearchReplaceBlocks(content: string): AIFileChange[] { - const blocks = content.split(/(?=<<<<<<< SEARCH)/); - return blocks - .map((block) => { - const [search, replace] = block.split(/^=======\s*$/m); - return { - search: search.replace(/^<<<<<<< SEARCH\s*\n/, '').trim(), - replace: replace.replace(/\s*>>>>>>> REPLACE.*?$/, '').trim(), - }; - }) - .filter((change) => change.search && change.replace); -} - -function isResponseMalformed(result: AIParsedResponse): boolean { - return ( - result.fileList.length === 0 && - result.files.length === 0 && - result.gitCommitMessage === '' - ); -} diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts new file mode 100644 index 0000000..9cdbf72 --- /dev/null +++ b/src/ai/parsers/search-replace-parser.ts @@ -0,0 +1,44 @@ +import { detectLanguage } from '../../core/file-worker'; +import type { AIFileChange, AIFileInfo } from '../../types'; + +export function parseSearchReplaceFiles(response: string): AIFileInfo[] { + const files: AIFileInfo[] = []; + + const fileRegex = + /[\s\S]*?(.*?)<\/file_path>[\s\S]*?(.*?)<\/file_status>[\s\S]*?([\s\S]*?)<\/file_content>(?:[\s\S]*?([\s\S]*?)<\/explanation>)?[\s\S]*?<\/file>/gs; + let match: RegExpExecArray | null; + + // biome-ignore lint/suspicious/noAssignInExpressions: avoid infinite loop + while ((match = fileRegex.exec(response)) !== null) { + const [, path, status, content, explanation] = match; + const fileInfo: AIFileInfo = { + path: path.trim(), + language: detectLanguage(path), + status: status.trim() as 'new' | 'modified' | 'deleted', + explanation: explanation ? explanation.trim() : undefined, + }; + + if (status.trim() === 'modified') { + fileInfo.changes = parseSearchReplaceBlocks(content); + } else if (status.trim() === 'new') { + fileInfo.content = content.trim(); + } + + files.push(fileInfo); + } + + return files; +} + +function parseSearchReplaceBlocks(content: string): AIFileChange[] { + const blocks = content.split(/(?=<<<<<<< SEARCH)/); + return blocks + .map((block) => { + const [search, replace] = block.split(/^=======\s*$/m); + return { + search: search.replace(/^<<<<<<< SEARCH\s*\n/, '').trim(), + replace: replace.replace(/\s*>>>>>>> REPLACE.*?$/, '').trim(), + }; + }) + .filter((change) => change.search && change.replace); +} diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index 2541b59..57ab926 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -409,6 +409,7 @@ async function generateAndApplyCode( const parsedResponse = parseAICodegenResponse( generatedCode, options.logAiInteractions, + options.diff, ); if (options.dryRun) { diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index 4525c00..97b9a44 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import chalk from 'chalk'; +import { applyPatch } from 'diff'; import fs from 'fs-extra'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; @@ -52,15 +53,36 @@ async function applyFileChange( console.log( chalk.yellow(`[DRY RUN] Would modify file: ${file.path}`), ); - file.changes?.forEach((change, index) => { + if (file.diff) { console.log( chalk.gray( - `[DRY RUN] Change ${index + 1} preview:\nSearch:\n${change.search.substring(0, 100)}${change.search.length > 100 ? '...' : ''}\nReplace:\n${change.replace.substring(0, 100)}${change.replace.length > 100 ? '...' : ''}`, + `[DRY RUN] Diff preview:\n${JSON.stringify(file.diff, null, 2)}`, ), ); - }); + } else if (file.changes) { + console.log( + chalk.gray( + `[DRY RUN] Changes preview:\n${JSON.stringify(file.changes, null, 2)}`, + ), + ); + } else if (file.content) { + console.log( + chalk.gray( + `[DRY RUN] Content preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, + ), + ); + } } else { - if (file.changes && file.changes.length > 0) { + if (file.diff) { + const currentContent = await fs.readFile(fullPath, 'utf-8'); + const updatedContent = applyPatch(currentContent, file.diff); + if (updatedContent === false) { + throw new Error( + `Failed to apply patch to file: ${file.path}\nA common cause is that the file was not sent to the LLM and it hallucinated the content. Try running the task again (task --redo) and selecting the problematic file.`, + ); + } + await fs.writeFile(fullPath, updatedContent); + } else if (file.changes) { let currentContent = await fs.readFile(fullPath, 'utf-8'); for (const change of file.changes) { currentContent = applyChange( @@ -70,9 +92,11 @@ async function applyFileChange( ); } await fs.writeFile(fullPath, currentContent); + } else if (file.content) { + await fs.writeFile(fullPath, file.content); } else { throw new Error( - `No changes provided for modified file: ${file.path}`, + `No content, diff, or changes provided for modified file: ${file.path}`, ); } console.log(chalk.green(`Modified file: ${file.path}`)); @@ -108,7 +132,11 @@ function applyChange(content: string, search: string, replace: string): string { const regex = new RegExp(flexibleSearch, 'g'); if (regex.test(content)) { - return content.replace(regex, trimmedReplace); + return content.replace(regex, (match) => { + const leadingWhitespace = match.match(/^\s*/)[0]; + const trailingWhitespace = match.match(/\s*$/)[0]; + return leadingWhitespace + trimmedReplace + trailingWhitespace; + }); } throw new Error( diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index d712f4e..aa2037b 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -65,7 +65,6 @@ Then, for each file: __FILE_PATH__ -__FILE_PATH__ <<<<<<< SEARCH __ORIGINAL_CONTENT__ ======= @@ -102,7 +101,6 @@ Example for a modified file with multiple changes: src/components/Button.tsx -src/components/Button.tsx <<<<<<< SEARCH import React from 'react'; diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index bbf862e..58224ca 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -71,7 +71,6 @@ Then, for each file: __FILE_PATH__ -__FILE_PATH__ <<<<<<< SEARCH __ORIGINAL_CONTENT__ ======= @@ -108,7 +107,6 @@ Example for a modified file with multiple changes: src/components/Button.tsx -src/components/Button.tsx <<<<<<< SEARCH import React from 'react'; diff --git a/src/types/index.ts b/src/types/index.ts index 31b3005..c3a6308 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -178,6 +178,7 @@ export interface AIFileInfo { path: string; language: string; content?: string; + diff?: ParsedDiff; changes?: AIFileChange[]; status: 'new' | 'modified' | 'deleted'; explanation?: string; diff --git a/tests/unit/parse-ai-codegen-response.test.ts b/tests/unit/parse-ai-codegen-response.test.ts index b531981..1f679b2 100644 --- a/tests/unit/parse-ai-codegen-response.test.ts +++ b/tests/unit/parse-ai-codegen-response.test.ts @@ -2,167 +2,76 @@ import { describe, expect, it } from 'vitest'; import { parseAICodegenResponse } from '../../src/ai/parse-ai-codegen-response'; describe('parseAICodegenResponse', () => { - describe('Full Content Mode', () => { - it('should correctly parse a valid AI response in full content mode', () => { + describe('parseAICodegenResponse', () => { + it('should correctly parse a valid AI response with multiple changes', () => { const mockResponse = ` - -file1.js -file2.ts - - - -src/file1.js - -console.log('Hello, World!'); - -modified - -Updated console log message - - - - -src/file2.ts - -const greeting: string = 'Hello, TypeScript!'; -console.log(greeting); - -new - - -feature/new-greeting - -Add new greeting functionality - - -Added a new TypeScript file and modified an existing JavaScript file. - - - -None identified. - -`; - - const result = parseAICodegenResponse(mockResponse, false, false); - - expect(result.fileList).toEqual(['file1.js', 'file2.ts']); - expect(result.files).toHaveLength(2); - expect(result.files[0]).toEqual({ - path: 'src/file1.js', - language: 'javascript', - content: "console.log('Hello, World!');", - status: 'modified', - explanation: 'Updated console log message', - }); - expect(result.files[1]).toEqual({ - path: 'src/file2.ts', - language: 'typescript', - content: - "const greeting: string = 'Hello, TypeScript!';\nconsole.log(greeting);", - status: 'new', - }); - expect(result.gitBranchName).toBe('feature/new-greeting'); - expect(result.gitCommitMessage).toBe('Add new greeting functionality'); - expect(result.summary).toBe( - 'Added a new TypeScript file and modified an existing JavaScript file.', - ); - expect(result.potentialIssues).toBe('None identified.'); - }); - - it('should handle responses with deleted files in full content mode', () => { - const mockResponse = ` - -file-to-delete.js - - - -src/file-to-delete.js -deleted - -`; - - const result = parseAICodegenResponse(mockResponse, false, false); - - expect(result.fileList).toEqual(['file-to-delete.js']); - expect(result.files).toHaveLength(1); - expect(result.files[0]).toEqual({ - path: 'src/file-to-delete.js', - language: '', - content: '', - status: 'deleted', - }); - }); - }); + + file1.js + file2.ts + - describe('Diff Mode', () => { - it('should correctly parse a valid AI response in diff mode', () => { - const mockResponse = ` - -file1.js -file2.ts - - -src/file1.js -modified - ---- src/file1.js -+++ src/file1.js -@@ -1,1 +1,1 @@ --console.log('Old message'); -+console.log('Hello, World!'); - - -Updated console log message - - + + src/file1.js + + <<<<<<< SEARCH + console.log('Old message'); + ======= + console.log('Hello, World!'); + >>>>>>> REPLACE + + <<<<<<< SEARCH + function oldFunction() {} + ======= + function newFunction() {} + >>>>>>> REPLACE + + modified + + Updated console log message and renamed a function + + - -src/file2.ts -new - -const greeting: string = 'Hello, TypeScript!'; -console.log(greeting); - - + + src/file2.ts + + const greeting: string = 'Hello, TypeScript!'; + console.log(greeting); + + new + -feature/new-greeting + feature/new-greeting -Add new greeting functionality + Add new greeting functionality - -Added a new TypeScript file and modified an existing JavaScript file. - + + Added a new TypeScript file and modified an existing JavaScript file. + - -None identified. - -`; + + None identified. + + `; - const result = parseAICodegenResponse(mockResponse, false, true); + const result = parseAICodegenResponse(mockResponse); expect(result.fileList).toEqual(['file1.js', 'file2.ts']); expect(result.files).toHaveLength(2); expect(result.files[0]).toEqual({ path: 'src/file1.js', language: 'javascript', - diff: { - oldFileName: 'src/file1.js', - newFileName: 'src/file1.js', - hunks: [ - { - oldStart: 1, - oldLines: 1, - newStart: 1, - newLines: 1, - lines: [ - "-console.log('Old message');", - "+console.log('Hello, World!');", - ], - }, - ], - }, status: 'modified', - explanation: 'Updated console log message', + changes: [ + { + search: "console.log('Old message');", + replace: "console.log('Hello, World!');", + }, + { + search: 'function oldFunction() {}', + replace: 'function newFunction() {}', + }, + ], + explanation: 'Updated console log message and renamed a function', }); expect(result.files[1]).toEqual({ path: 'src/file2.ts', @@ -179,7 +88,7 @@ None identified. expect(result.potentialIssues).toBe('None identified.'); }); - it('should handle responses with deleted files in diff mode', () => { + it('should handle responses with deleted files', () => { const mockResponse = ` file-to-delete.js @@ -191,110 +100,16 @@ None identified. `; - const result = parseAICodegenResponse(mockResponse, false, true); + const result = parseAICodegenResponse(mockResponse); expect(result.fileList).toEqual(['file-to-delete.js']); expect(result.files).toHaveLength(1); expect(result.files[0]).toEqual({ path: 'src/file-to-delete.js', - language: '', + language: 'javascript', status: 'deleted', }); }); - - it('should generate correctly formatted diffs in diff mode with only additions', () => { - const mockResponse = ` - -src/file1.js - - -update-console-log - - -Add console log message - - -Added a console log message to file1.js - - -None identified - - -src/file1.js -modified - ---- src/file1.js -+++ src/file1.js -@@ -1,3 +1,4 @@ - import { existingImport } from './existing'; - -+console.log('Hello, World!'); - - -Added a console log message to demonstrate the change. - - - `; - - const result = parseAICodegenResponse(mockResponse, false, true); - const diff = result.files[0].diff; - - expect(diff).toBeDefined(); - if (!diff) return; - expect(diff.hunks[0].lines).toEqual([ - " import { existingImport } from './existing';", - '', - "+console.log('Hello, World!');", - ]); - }); - - it('should generate correctly formatted diffs in diff mode with additions and removals', () => { - const mockResponse = ` - -src/file1.js - - -update-console-log-message - - -Update console log message - - -Changed the console log message in file1.js - - -None identified - - -src/file1.js -modified - ---- src/file1.js -+++ src/file1.js -@@ -1,3 +1,3 @@ - import { existingImport } from './existing'; - --console.log('Old message'); -+console.log('Hello, World!'); - - -Updated the console log message to 'Hello, World!'. - - - `; - - const result = parseAICodegenResponse(mockResponse, false, true); - const diff = result.files[0].diff; - - expect(diff).toBeDefined(); - if (!diff) return; - expect(diff.hunks[0].lines).toEqual([ - " import { existingImport } from './existing';", - '', - "-console.log('Old message');", - "+console.log('Hello, World!');", - ]); - }); }); // Common tests for both modes @@ -351,8 +166,13 @@ file1.js src/file1.js - + +src/file1.js +<<<<<<< SEARCH +console.log('Old message'); +======= console.log('Hello, World!'); +>>>>>>> REPLACE modified @@ -391,32 +211,6 @@ console.log('Hello, World!'); expect(result.gitBranchName).toBe('invalid/branch/name----------'); }); - it('should sanitize invalid branch names with a leading slash', () => { - const mockResponse = ` - -/feature/invalid-branch-name-issue-123 -Some commit message -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.gitBranchName).toBe( - 'feature/invalid-branch-name-issue-123', - ); - }); - - it('should provide a default branch name when sanitized name is empty', () => { - const mockResponse = ` - -!@#$%^&*() -Some commit message -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.gitBranchName).toMatch(/^feature\/ai-task-\d+$/); - }); - it('should handle responses with extra whitespace and newlines', () => { const mockResponse = ` @@ -426,10 +220,17 @@ console.log('Hello, World!'); src/file1.js - + + src/file1.js + <<<<<<< SEARCH + + console.log('Old message'); + + ======= console.log('Hello, World!'); + >>>>>>> REPLACE modified @@ -448,7 +249,16 @@ console.log('Hello, World!'); expect(result.fileList).toEqual(['file1.js', 'file2.js']); expect(result.files).toHaveLength(1); expect(result.files[0].path).toBe('src/file1.js'); - expect(result.files[0].content).toBe("console.log('Hello, World!');"); + if (!result.files[0].changes) { + return; + } + expect(result.files[0].changes).toHaveLength(1); + expect(result.files[0].changes[0].search).toBe( + "console.log('Old message');", + ); + expect(result.files[0].changes[0].replace).toBe( + "console.log('Hello, World!');", + ); expect(result.files[0].status).toBe('modified'); expect(result.gitBranchName).toBe('feature/whitespace'); expect(result.gitCommitMessage).toBe('Handle extra whitespace'); @@ -463,7 +273,7 @@ file1.js src/file1.js - + console.log('First file'); new @@ -471,7 +281,7 @@ console.log('First file'); test/file1.js - + console.log('Second file'); new diff --git a/tests/unit/task-workflow.test.ts b/tests/unit/task-workflow.test.ts index be11e3a..15a6d57 100644 --- a/tests/unit/task-workflow.test.ts +++ b/tests/unit/task-workflow.test.ts @@ -190,7 +190,6 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, - undefined, ); expect(ensureBranch).toHaveBeenCalledWith( @@ -322,7 +321,6 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, - undefined, ); expect(ensureBranch).toHaveBeenCalledWith( From f8b534536166a1e7fef191deb55b7631db3402aa Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 23:32:56 +0200 Subject: [PATCH 07/54] feat: new diff mode --- src/ai/parse-ai-codegen-response.ts | 8 +- src/ai/parsers/common-parser.ts | 24 +- src/ai/parsers/full-content-parser.ts | 20 +- src/ai/parsers/search-replace-parser.ts | 88 ++++-- src/ai/task-workflow.ts | 3 +- src/git/apply-changes.ts | 4 +- tests/unit/parse-ai-codegen-response.test.ts | 284 ++++++++++--------- tests/unit/task-workflow.test.ts | 11 +- 8 files changed, 254 insertions(+), 188 deletions(-) diff --git a/src/ai/parse-ai-codegen-response.ts b/src/ai/parse-ai-codegen-response.ts index 1d2a442..6edc130 100644 --- a/src/ai/parse-ai-codegen-response.ts +++ b/src/ai/parse-ai-codegen-response.ts @@ -26,11 +26,9 @@ export function parseAICodegenResponse( const commonFields = parseCommonFields(response); result = { ...result, ...commonFields }; - if (useDiffMode) { - result.files = parseSearchReplaceFiles(response); - } else { - result.files = parseFullContentFiles(response); - } + result.files = useDiffMode + ? parseSearchReplaceFiles(response) + : parseFullContentFiles(response); if (isResponseMalformed(result)) { throw new Error('Malformed response'); diff --git a/src/ai/parsers/common-parser.ts b/src/ai/parsers/common-parser.ts index cdd28cc..e60e434 100644 --- a/src/ai/parsers/common-parser.ts +++ b/src/ai/parsers/common-parser.ts @@ -1,4 +1,4 @@ -import type { AIParsedResponse } from '../../types'; +import type { AIFileInfo, AIParsedResponse } from '../../types'; import { ensureValidBranchName } from '../../utils/git-tools'; export function parseCommonFields(response: string): Partial { @@ -44,3 +44,25 @@ export function isResponseMalformed(result: AIParsedResponse): boolean { result.gitCommitMessage === '' ); } + +export function handleDeletedFiles( + files: AIFileInfo[], + response: string, +): AIFileInfo[] { + // Handle deleted files + const deletedFileRegex = + /[\s\S]*?(.*?)<\/file_path>[\s\S]*?deleted<\/file_status>[\s\S]*?<\/file>/g; + let deletedMatch: RegExpExecArray | null; + while (true) { + deletedMatch = deletedFileRegex.exec(response); + if (deletedMatch === null) break; + const [, path] = deletedMatch; + files.push({ + path: path.trim(), + language: '', + content: '', + status: 'deleted', + }); + } + return files; +} diff --git a/src/ai/parsers/full-content-parser.ts b/src/ai/parsers/full-content-parser.ts index 92dc50c..a0c8b8b 100644 --- a/src/ai/parsers/full-content-parser.ts +++ b/src/ai/parsers/full-content-parser.ts @@ -1,7 +1,7 @@ import type { AIFileInfo } from '../../types'; - +import { handleDeletedFiles } from './common-parser'; export function parseFullContentFiles(response: string): AIFileInfo[] { - const files: AIFileInfo[] = []; + let files: AIFileInfo[] = []; // Parse individual files const fileRegex = @@ -20,21 +20,7 @@ export function parseFullContentFiles(response: string): AIFileInfo[] { }); } - // Handle deleted files - const deletedFileRegex = - /[\s\S]*?(.*?)<\/file_path>[\s\S]*?deleted<\/file_status>[\s\S]*?<\/file>/g; - let deletedMatch: RegExpExecArray | null; - while (true) { - deletedMatch = deletedFileRegex.exec(response); - if (deletedMatch === null) break; - const [, path] = deletedMatch; - files.push({ - path: path.trim(), - language: '', - content: '', - status: 'deleted', - }); - } + files = handleDeletedFiles(files, response); return files; } diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 9cdbf72..4efe880 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,44 +1,86 @@ import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; +import { handleDeletedFiles } from './common-parser'; export function parseSearchReplaceFiles(response: string): AIFileInfo[] { - const files: AIFileInfo[] = []; + let files: AIFileInfo[] = []; - const fileRegex = - /[\s\S]*?(.*?)<\/file_path>[\s\S]*?(.*?)<\/file_status>[\s\S]*?([\s\S]*?)<\/file_content>(?:[\s\S]*?([\s\S]*?)<\/explanation>)?[\s\S]*?<\/file>/gs; - let match: RegExpExecArray | null; + const fileRegex = /([\s\S]*?)<\/file>/g; + let fileMatch: RegExpExecArray | null; // biome-ignore lint/suspicious/noAssignInExpressions: avoid infinite loop - while ((match = fileRegex.exec(response)) !== null) { - const [, path, status, content, explanation] = match; + while ((fileMatch = fileRegex.exec(response)) !== null) { + const fileContent = fileMatch[1]; + + const pathMatch = /(.*?)<\/file_path>/.exec(fileContent); + const statusMatch = /(.*?)<\/file_status>/.exec(fileContent); + const contentMatch = /([\s\S]*?)<\/file_content>/.exec( + fileContent, + ); + const explanationMatch = /([\s\S]*?)<\/explanation>/.exec( + fileContent, + ); + + if (!pathMatch || !statusMatch || !contentMatch) { + console.warn('Skipping file due to missing required information'); + continue; + } + + const path = pathMatch[1].trim(); + const status = statusMatch[1].trim() as 'new' | 'modified' | 'deleted'; + const content = contentMatch[1].trim(); + const explanation = explanationMatch + ? explanationMatch[1].trim() + : undefined; + + console.log(`Parsing file: ${path}`); + console.log(`Status: ${status}`); + console.log(`Content: ${content}`); + const fileInfo: AIFileInfo = { - path: path.trim(), + path, language: detectLanguage(path), - status: status.trim() as 'new' | 'modified' | 'deleted', - explanation: explanation ? explanation.trim() : undefined, + status, + explanation, }; - if (status.trim() === 'modified') { + if (status === 'modified') { fileInfo.changes = parseSearchReplaceBlocks(content); - } else if (status.trim() === 'new') { - fileInfo.content = content.trim(); + if (fileInfo.changes.length === 0) { + console.warn(`No changes found for modified file: ${path}`); + fileInfo.content = content; + } + } else if (status === 'new') { + fileInfo.content = content; } files.push(fileInfo); } + files = handleDeletedFiles(files, response); + return files; } - function parseSearchReplaceBlocks(content: string): AIFileChange[] { - const blocks = content.split(/(?=<<<<<<< SEARCH)/); - return blocks - .map((block) => { - const [search, replace] = block.split(/^=======\s*$/m); - return { - search: search.replace(/^<<<<<<< SEARCH\s*\n/, '').trim(), - replace: replace.replace(/\s*>>>>>>> REPLACE.*?$/, '').trim(), - }; - }) - .filter((change) => change.search && change.replace); + console.log('Parsing search/replace blocks. Content:', content); + + const blockRegex = + /<<<<<<< SEARCH([\s\S]*?)=======([\s\S]*?)>>>>>>> REPLACE/g; + const changes: AIFileChange[] = []; + let match: RegExpExecArray | null; + + // biome-ignore lint/suspicious/noAssignInExpressions: avoid infinite loop + while ((match = blockRegex.exec(content)) !== null) { + const search = match[1].trim(); + const replace = match[2].trim(); + + if (search && replace) { + changes.push({ search, replace }); + } else { + console.log('Invalid block structure'); + } + } + + console.log(`Found ${changes.length} valid changes`); + return changes; } diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index 57ab926..c170c83 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -47,10 +47,9 @@ export async function runAIAssistedTask(options: AiAssistedTaskOptions) { if (options.diff && modelKey !== 'claude-3-5-sonnet-20240620') { console.log( chalk.yellow( - 'Diff-based code modifications are currently only supported with Claude 3.5 Sonnet. Falling back to full-file code modifications.', + 'Diff-based code modifications are currently only supported with Claude 3.5 Sonnet.', ), ); - options.diff = false; } const { taskDescription, instructions } = await getTaskInfo( diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index 97b9a44..a809e02 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -133,8 +133,8 @@ function applyChange(content: string, search: string, replace: string): string { if (regex.test(content)) { return content.replace(regex, (match) => { - const leadingWhitespace = match.match(/^\s*/)[0]; - const trailingWhitespace = match.match(/\s*$/)[0]; + const leadingWhitespace = match.match(/^\s*/)?.[0] || ''; + const trailingWhitespace = match.match(/\s*$/)?.[0] || ''; return leadingWhitespace + trimmedReplace + trailingWhitespace; }); } diff --git a/tests/unit/parse-ai-codegen-response.test.ts b/tests/unit/parse-ai-codegen-response.test.ts index 1f679b2..d6dc791 100644 --- a/tests/unit/parse-ai-codegen-response.test.ts +++ b/tests/unit/parse-ai-codegen-response.test.ts @@ -2,58 +2,44 @@ import { describe, expect, it } from 'vitest'; import { parseAICodegenResponse } from '../../src/ai/parse-ai-codegen-response'; describe('parseAICodegenResponse', () => { - describe('parseAICodegenResponse', () => { + describe('diff mode', () => { it('should correctly parse a valid AI response with multiple changes', () => { const mockResponse = ` - - file1.js - file2.ts - - - - src/file1.js - - <<<<<<< SEARCH - console.log('Old message'); - ======= - console.log('Hello, World!'); - >>>>>>> REPLACE - - <<<<<<< SEARCH - function oldFunction() {} - ======= - function newFunction() {} - >>>>>>> REPLACE - - modified - - Updated console log message and renamed a function - - - - - src/file2.ts - - const greeting: string = 'Hello, TypeScript!'; - console.log(greeting); - - new - - - feature/new-greeting - - Add new greeting functionality - - - Added a new TypeScript file and modified an existing JavaScript file. - - - - None identified. - - `; + +file1.js +file2.ts + + +src/file1.js +modified + +<<<<<<< SEARCH +console.log('Old message'); +======= +console.log('Hello, World!'); +>>>>>>> REPLACE - const result = parseAICodegenResponse(mockResponse); +<<<<<<< SEARCH +function oldFunction() {} +======= +function newFunction() {} +>>>>>>> REPLACE + +Updated console log message and renamed a function + + +src/file2.ts +new + +const greeting: string = 'Hello, TypeScript!'; +console.log(greeting); + + +feature/new-greeting +Add new greeting functionality +Added a new TypeScript file and modified an existing JavaScript file. +None identified.`; + const result = parseAICodegenResponse(mockResponse, false, true); expect(result.fileList).toEqual(['file1.js', 'file2.ts']); expect(result.files).toHaveLength(2); @@ -76,6 +62,7 @@ describe('parseAICodegenResponse', () => { expect(result.files[1]).toEqual({ path: 'src/file2.ts', language: 'typescript', + explanation: undefined, content: "const greeting: string = 'Hello, TypeScript!';\nconsole.log(greeting);", status: 'new', @@ -88,86 +75,38 @@ describe('parseAICodegenResponse', () => { expect(result.potentialIssues).toBe('None identified.'); }); - it('should handle responses with deleted files', () => { + it('should handle responses with deleted files in diff mode', () => { const mockResponse = ` - - file-to-delete.js - + + file-to-delete.js + - - src/file-to-delete.js - deleted - - `; + + src/file-to-delete.js + deleted + + `; - const result = parseAICodegenResponse(mockResponse); + const result = parseAICodegenResponse(mockResponse, false, true); expect(result.fileList).toEqual(['file-to-delete.js']); expect(result.files).toHaveLength(1); expect(result.files[0]).toEqual({ path: 'src/file-to-delete.js', - language: 'javascript', + content: '', + language: '', status: 'deleted', }); }); - }); - - // Common tests for both modes - describe('Common Functionality', () => { - it('should handle responses with no file changes (file list with only whitespace)', () => { - const mockResponse = ` - - - - -no-changes - -No changes required - - -No changes were necessary for this task. - - - -None - -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.fileList).toEqual([]); - expect(result.files).toHaveLength(0); - expect(result.gitBranchName).toBe('no-changes'); - expect(result.gitCommitMessage).toBe('No changes required'); - }); - - it('should handle severely malformed responses', () => { - const mockResponse = ` -This is not even close to valid XML -This tag is not closed properly -Random content -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.fileList).toEqual([]); - expect(result.files).toHaveLength(0); - expect(result.gitBranchName).toMatch(/^feature\/ai-task-\d+$/); - expect(result.gitCommitMessage).toBe('Error: Malformed response'); - expect(result.summary).toBe(''); - expect(result.potentialIssues).toBe(''); - }); it('should handle responses with missing sections', () => { const mockResponse = ` file1.js - src/file1.js -src/file1.js <<<<<<< SEARCH console.log('Old message'); ======= @@ -178,7 +117,7 @@ console.log('Hello, World!'); `; - const result = parseAICodegenResponse(mockResponse); + const result = parseAICodegenResponse(mockResponse, false, true); expect(result.fileList).toEqual(['file1.js']); expect(result.files).toHaveLength(1); @@ -187,30 +126,6 @@ console.log('Hello, World!'); expect(result.potentialIssues).toBe(''); }); - it('should provide a default branch name when gitBranchName is empty', () => { - const mockResponse = ` - - -Some commit message -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.gitBranchName).toMatch(/^feature\/ai-task-\d+$/); - }); - - it('should sanitize invalid branch names', () => { - const mockResponse = ` - -invalid/branch/name!@#$%^&*() -Some commit message -`; - - const result = parseAICodegenResponse(mockResponse); - - expect(result.gitBranchName).toBe('invalid/branch/name----------'); - }); - it('should handle responses with extra whitespace and newlines', () => { const mockResponse = ` @@ -244,7 +159,7 @@ console.log('Hello, World!'); `; - const result = parseAICodegenResponse(mockResponse); + const result = parseAICodegenResponse(mockResponse, false, true); expect(result.fileList).toEqual(['file1.js', 'file2.js']); expect(result.files).toHaveLength(1); @@ -263,6 +178,101 @@ console.log('Hello, World!'); expect(result.gitBranchName).toBe('feature/whitespace'); expect(result.gitCommitMessage).toBe('Handle extra whitespace'); }); + }); + + it('should handle responses with deleted files in whole file edit mode', () => { + const mockResponse = ` + +file-to-delete.js + + + +src/file-to-delete.js +deleted + +`; + + const result = parseAICodegenResponse(mockResponse, false, false); + + expect(result.fileList).toEqual(['file-to-delete.js']); + expect(result.files).toHaveLength(1); + expect(result.files[0]).toEqual({ + path: 'src/file-to-delete.js', + content: '', + language: '', + status: 'deleted', + }); + }); + + // Common tests for both modes + describe('Common Functionality', () => { + it('should handle responses with no file changes (file list with only whitespace)', () => { + const mockResponse = ` + + + + +no-changes + +No changes required + + +No changes were necessary for this task. + + + +None + +`; + + const result = parseAICodegenResponse(mockResponse); + + expect(result.fileList).toEqual([]); + expect(result.files).toHaveLength(0); + expect(result.gitBranchName).toBe('no-changes'); + expect(result.gitCommitMessage).toBe('No changes required'); + }); + + it('should handle severely malformed responses', () => { + const mockResponse = ` +This is not even close to valid XML +This tag is not closed properly +Random content +`; + + const result = parseAICodegenResponse(mockResponse); + + expect(result.fileList).toEqual([]); + expect(result.files).toHaveLength(0); + expect(result.gitBranchName).toMatch(/^feature\/ai-task-\d+$/); + expect(result.gitCommitMessage).toBe('Error: Malformed response'); + expect(result.summary).toBe(''); + expect(result.potentialIssues).toBe(''); + }); + + it('should provide a default branch name when gitBranchName is empty', () => { + const mockResponse = ` + + +Some commit message +`; + + const result = parseAICodegenResponse(mockResponse); + + expect(result.gitBranchName).toMatch(/^feature\/ai-task-\d+$/); + }); + + it('should sanitize invalid branch names', () => { + const mockResponse = ` + +invalid/branch/name!@#$%^&*() +Some commit message +`; + + const result = parseAICodegenResponse(mockResponse); + + expect(result.gitBranchName).toBe('invalid/branch/name----------'); + }); it('should handle responses with multiple files of the same name', () => { const mockResponse = ` @@ -273,7 +283,7 @@ file1.js src/file1.js - + console.log('First file'); new @@ -281,7 +291,7 @@ console.log('First file'); test/file1.js - + console.log('Second file'); new diff --git a/tests/unit/task-workflow.test.ts b/tests/unit/task-workflow.test.ts index 15a6d57..7136620 100644 --- a/tests/unit/task-workflow.test.ts +++ b/tests/unit/task-workflow.test.ts @@ -173,6 +173,7 @@ describe('runAIAssistedTask', () => { const mockOptionsWithoutPlan = { ...mockOptions, plan: false, + diff: false, }; await runAIAssistedTask(mockOptionsWithoutPlan); @@ -190,6 +191,7 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, + false, ); expect(ensureBranch).toHaveBeenCalledWith( @@ -306,7 +308,12 @@ describe('runAIAssistedTask', () => { vi.mocked(applyChanges).mockResolvedValue(); vi.mocked(getModelConfig).mockReturnValue(mockModelConfig); - await runAIAssistedTask(mockOptions); + const mockOptionsWithPlan = { + ...mockOptions, + diff: false, + }; + + await runAIAssistedTask(mockOptionsWithPlan); expect(getTaskDescription).toHaveBeenCalled(); expect(getInstructions).toHaveBeenCalled(); @@ -321,6 +328,7 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, + false, ); expect(ensureBranch).toHaveBeenCalledWith( @@ -445,6 +453,7 @@ describe('runAIAssistedTask', () => { expect(parseAICodegenResponse).toHaveBeenCalledWith( mockGeneratedCode, undefined, + true, ); expect(applyChanges).toHaveBeenCalledWith( expect.objectContaining({ From 68f7ef444dbb31053edf807560cce92a45b7f421 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Tue, 6 Aug 2024 23:33:11 +0200 Subject: [PATCH 08/54] remove extraenous import --- tests/unit/apply-changes.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 61b8b04..48f1c8c 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -import { applyPatch } from 'diff'; import fs from 'fs-extra'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { applyChanges } from '../../src/git/apply-changes'; @@ -64,7 +63,8 @@ describe('applyChanges', () => { }; vi.mocked(fs.readFile).mockResolvedValue( - 'console.log("Old content");\nfunction oldFunction() {}', + // biome-ignore lint/suspicious/noExplicitAny: explicit any is required for the mock + 'console.log("Old content");\nfunction oldFunction() {}' as any, ); await applyChanges({ From 8cd9335517d61a4ba365283aae3cff47c604fc7b Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 00:05:24 +0200 Subject: [PATCH 09/54] improve apply-changes reliability --- src/git/apply-changes.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index a809e02..bfc51ab 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -84,12 +84,20 @@ async function applyFileChange( await fs.writeFile(fullPath, updatedContent); } else if (file.changes) { let currentContent = await fs.readFile(fullPath, 'utf-8'); + const appliedChanges = new Set(); for (const change of file.changes) { - currentContent = applyChange( - currentContent, - change.search, - change.replace, - ); + if (!appliedChanges.has(change.search)) { + currentContent = applyChange( + currentContent, + change.search, + change.replace, + ); + appliedChanges.add(change.search); + } else { + console.warn( + `Skipping duplicate change for ${file.path}: ${change.search}`, + ); + } } await fs.writeFile(fullPath, currentContent); } else if (file.content) { @@ -129,7 +137,7 @@ function applyChange(content: string, search: string, replace: string): string { // If exact match fails, try matching with flexible whitespace const flexibleSearch = trimmedSearch.replace(/\s+/g, '\\s+'); - const regex = new RegExp(flexibleSearch, 'g'); + const regex = new RegExp(flexibleSearch); if (regex.test(content)) { return content.replace(regex, (match) => { From a5d9cc081751b9fc6a66bfd3382f8324e22c7575 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 00:28:25 +0200 Subject: [PATCH 10/54] fix: make prompt provide unique search blocks --- src/templates/codegen-diff-no-plan-prompt.hbs | 61 +++++++++++++------ src/templates/codegen-diff-prompt.hbs | 59 ++++++++++++------ 2 files changed, 82 insertions(+), 38 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index aa2037b..84ea08c 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -86,6 +86,9 @@ For modified files: - Use one or more SEARCH/REPLACE blocks within the tags. - Each SEARCH/REPLACE block should focus on a specific change. - You can include multiple SEARCH/REPLACE blocks for a single file if needed. +- IMPORTANT: Provide sufficient context in the SEARCH block to uniquely identify the location of the change. Include surrounding code, function names, or object structures to ensure the change can be accurately applied. +- For changes to object literals or nested structures, include the full key path and surrounding context in the search pattern. +- If adding a new item to an object or array, include the entire opening of the structure in the search pattern. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -97,33 +100,52 @@ STATUS: Use 'new' for newly created files, 'modified' for existing files that ar EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. -Example for a modified file with multiple changes: +Example for a modified file with multiple changes and good context: -src/components/Button.tsx +src/ai/model-config.ts <<<<<<< SEARCH -import React from 'react'; - -const Button = ({ text }) => ( - -); -======= -import React from 'react'; - -const Button = ({ text, onClick }) => ( - -); ->>>>>>> REPLACE - -<<<<<<< SEARCH -export default Button; +export const MODEL_CONFIGS: ModelSpecs = { + 'claude-3-5-sonnet-20240620': { + contextWindow: 200000, + maxOutput: 8192, + modelName: 'Claude 3.5 Sonnet', + pricing: { inputCost: 3, outputCost: 15 }, + modelFamily: 'claude', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.3, + }, + }, + // Other existing models... +}; ======= -export default React.memo(Button); +export const MODEL_CONFIGS: ModelSpecs = { + 'claude-3-5-sonnet-20240620': { + contextWindow: 200000, + maxOutput: 8192, + modelName: 'Claude 3.5 Sonnet', + pricing: { inputCost: 3, outputCost: 15 }, + modelFamily: 'claude', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.3, + }, + }, + 'deepseek-coder': { + contextWindow: 128000, + maxOutput: 8000, + modelName: 'DeepSeek-Coder', + pricing: { inputCost: 0.14, outputCost: 0.28 }, + modelFamily: 'openai-compatible', + }, + // Other existing models... +}; >>>>>>> REPLACE modified -Added onClick prop to make the button interactive and wrapped the component with React.memo for performance optimization. +Added a new model configuration for DeepSeek-Coder to the MODEL_CONFIGS object, including its context window, max output, pricing, and model family. @@ -178,6 +200,7 @@ Ensure that: - You haven't introduced any potential bugs or performance issues. - Your code is easy to understand and maintain. - You complete all necessary work to fully implement the task. +- SEARCH blocks contain enough context to uniquely identify the location of changes. --- Now, implement the task described above. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 58224ca..a974c26 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -92,6 +92,9 @@ For modified files: - Use one or more SEARCH/REPLACE blocks within the tags. - Each SEARCH/REPLACE block should focus on a specific change. - You can include multiple SEARCH/REPLACE blocks for a single file if needed. +- IMPORTANT: Provide sufficient context in the SEARCH block to uniquely identify the location of the change. Include surrounding code, function names, or object structures to ensure the change can be accurately applied. +- For changes to object literals or nested structures, include the full key path and surrounding context in the search pattern. +- If adding a new item to an object or array, include the entire opening of the structure in the search pattern. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -103,33 +106,50 @@ STATUS: Use 'new' for newly created files, 'modified' for existing files that ar EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. -Example for a modified file with multiple changes: +Example for a modified file with multiple changes and good context: -src/components/Button.tsx +src/ai/model-config.ts <<<<<<< SEARCH -import React from 'react'; - -const Button = ({ text }) => ( - -); -======= -import React from 'react'; - -const Button = ({ text, onClick }) => ( - -); ->>>>>>> REPLACE - -<<<<<<< SEARCH -export default Button; +export const MODEL_CONFIGS: ModelSpecs = { + 'claude-3-5-sonnet-20240620': { + contextWindow: 200000, + maxOutput: 8192, + modelName: 'Claude 3.5 Sonnet', + pricing: { inputCost: 3, outputCost: 15 }, + modelFamily: 'claude', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.3, + }, + }, +}; ======= -export default React.memo(Button); +export const MODEL_CONFIGS: ModelSpecs = { + 'claude-3-5-sonnet-20240620': { + contextWindow: 200000, + maxOutput: 8192, + modelName: 'Claude 3.5 Sonnet', + pricing: { inputCost: 3, outputCost: 15 }, + modelFamily: 'claude', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.3, + }, + }, + 'deepseek-coder': { + contextWindow: 128000, + maxOutput: 8000, + modelName: 'DeepSeek-Coder', + pricing: { inputCost: 0.14, outputCost: 0.28 }, + modelFamily: 'openai-compatible', + }, +}; >>>>>>> REPLACE modified -Added onClick prop to make the button interactive and wrapped the component with React.memo for performance optimization. +Added a new model configuration for DeepSeek-Coder to the MODEL_CONFIGS object, including its context window, max output, pricing, and model family. @@ -184,4 +204,5 @@ Ensure that: - You haven't introduced any potential bugs or performance issues. - Your code is easy to understand and maintain. - You complete all necessary work to fully implement the task. +- SEARCH blocks contain enough context to uniquely identify the location of changes. From f94b99d6c8ca0d4ca69eadabfb842dff5cd98382 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 00:43:07 +0200 Subject: [PATCH 11/54] prompt engineering time --- src/templates/codegen-diff-no-plan-prompt.hbs | 131 ++++++++++++------ src/templates/codegen-diff-prompt.hbs | 130 +++++++++++------ 2 files changed, 180 insertions(+), 81 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 84ea08c..463aab3 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -79,16 +79,34 @@ __EXPLANATION__ (if necessary) Please adhere to the following guidelines: -FILE_PATH: Use the full path from the project root. +FILE_PATH: Use the full path from the project root, without any leading dots. Example: 'src/components/Button.tsx' For modified files: - Use one or more SEARCH/REPLACE blocks within the tags. -- Each SEARCH/REPLACE block should focus on a specific change. -- You can include multiple SEARCH/REPLACE blocks for a single file if needed. -- IMPORTANT: Provide sufficient context in the SEARCH block to uniquely identify the location of the change. Include surrounding code, function names, or object structures to ensure the change can be accurately applied. -- For changes to object literals or nested structures, include the full key path and surrounding context in the search pattern. -- If adding a new item to an object or array, include the entire opening of the structure in the search pattern. +- Each SEARCH/REPLACE block should focus on a specific, minimal change. +- Provide sufficient context in the SEARCH block to uniquely identify the location of the change, but keep it as small as possible. +- For large changes, break them down into multiple smaller SEARCH/REPLACE blocks. +- Ensure that the SEARCH content exists exactly as specified in the file. +- For changes to object literals or nested structures, focus on the specific key or value being changed, rather than the entire structure. +- Include some unchanged lines before and after the change to provide context. +- Limit each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. + +Please adhere to the following guidelines for SEARCH/REPLACE blocks in modified files: + +1. Provide sufficient context: Include enough surrounding code to uniquely identify the location of the change, typically 2-3 lines before and after the modified section. + +2. Keep changes focused: Each SEARCH/REPLACE block should address a single, specific change. For multiple changes in a file, use multiple blocks. + +3. Maintain exact matching: Ensure the SEARCH content exists exactly as specified in the original file, including whitespace and indentation. + +4. Limit block size: Keep each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. + +5. Preserve structure: For changes within functions, classes, or nested structures, include the enclosing context (e.g., function signature, class declaration). + +6. Handle imports: When adding new imports, include the entire import section in the SEARCH block, even if only adding one line. + +7. Comment changes: If modifying comments, include them in the SEARCH/REPLACE blocks. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -100,52 +118,80 @@ STATUS: Use 'new' for newly created files, 'modified' for existing files that ar EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. -Example for a modified file with multiple changes and good context: +Examples of good SEARCH/REPLACE blocks for different scenarios: + +1. Modifying a function: + +src/utils/data-processor.ts + +<<<<<<< SEARCH +export function processData(data: any[]): ProcessedData { + return data.map(item => ({ + id: item.id, + name: item.name.toUpperCase(), + value: item.value * 2 + })); +} +======= +export function processData(data: any[]): ProcessedData { + return data.map(item => ({ + id: item.id, + name: item.name.toUpperCase(), + value: item.value * 2, + timestamp: new Date().toISOString() + })); +} +>>>>>>> REPLACE + +modified + +Added a timestamp field to each processed item for better tracking. + + + +2. Adding a new import: + +src/components/UserProfile.tsx + +<<<<<<< SEARCH +import React from 'react'; +import { User } from '../types'; +import { Avatar } from './Avatar'; +======= +import React from 'react'; +import { User } from '../types'; +import { Avatar } from './Avatar'; +import { formatDate } from '../utils/date-helpers'; +>>>>>>> REPLACE + +modified + +Imported the formatDate utility function to properly display user dates. + + + +3. Modifying a configuration object: -src/ai/model-config.ts +src/config/app-config.ts <<<<<<< SEARCH -export const MODEL_CONFIGS: ModelSpecs = { - 'claude-3-5-sonnet-20240620': { - contextWindow: 200000, - maxOutput: 8192, - modelName: 'Claude 3.5 Sonnet', - pricing: { inputCost: 3, outputCost: 15 }, - modelFamily: 'claude', - temperature: { - planningTemperature: 0.5, - codegenTemperature: 0.3, - }, - }, - // Other existing models... +export const CONFIG = { + apiUrl: 'https://api.example.com', + timeout: 5000, + retryAttempts: 3, }; ======= -export const MODEL_CONFIGS: ModelSpecs = { - 'claude-3-5-sonnet-20240620': { - contextWindow: 200000, - maxOutput: 8192, - modelName: 'Claude 3.5 Sonnet', - pricing: { inputCost: 3, outputCost: 15 }, - modelFamily: 'claude', - temperature: { - planningTemperature: 0.5, - codegenTemperature: 0.3, - }, - }, - 'deepseek-coder': { - contextWindow: 128000, - maxOutput: 8000, - modelName: 'DeepSeek-Coder', - pricing: { inputCost: 0.14, outputCost: 0.28 }, - modelFamily: 'openai-compatible', - }, - // Other existing models... +export const CONFIG = { + apiUrl: 'https://api.example.com', + timeout: 5000, + retryAttempts: 3, + cacheTimeout: 60000, }; >>>>>>> REPLACE modified -Added a new model configuration for DeepSeek-Coder to the MODEL_CONFIGS object, including its context window, max output, pricing, and model family. +Added a cacheTimeout configuration to implement response caching. @@ -201,6 +247,7 @@ Ensure that: - Your code is easy to understand and maintain. - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. +- Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. --- Now, implement the task described above. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index a974c26..fa3d991 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -85,16 +85,35 @@ __EXPLANATION__ (if necessary) Please adhere to the following guidelines: -FILE_PATH: Use the full path from the project root. +FILE_PATH: Use the full path from the project root, without any leading dots. Example: 'src/components/Button.tsx' For modified files: - Use one or more SEARCH/REPLACE blocks within the tags. -- Each SEARCH/REPLACE block should focus on a specific change. -- You can include multiple SEARCH/REPLACE blocks for a single file if needed. -- IMPORTANT: Provide sufficient context in the SEARCH block to uniquely identify the location of the change. Include surrounding code, function names, or object structures to ensure the change can be accurately applied. -- For changes to object literals or nested structures, include the full key path and surrounding context in the search pattern. +- Each SEARCH/REPLACE block should focus on a specific, minimal change. +- Provide sufficient context in the SEARCH block to uniquely identify the location of the change, but keep it as small as possible. +- For large changes, break them down into multiple smaller SEARCH/REPLACE blocks. +- Ensure that the SEARCH content exists exactly as specified in the file. +- For changes to object literals or nested structures, focus on the specific key or value being changed, rather than the entire structure. - If adding a new item to an object or array, include the entire opening of the structure in the search pattern. +- Include some unchanged lines before and after the change to provide context. +- Limit each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. + +Please adhere to the following guidelines for SEARCH/REPLACE blocks in modified files: + +1. Provide sufficient context: Include enough surrounding code to uniquely identify the location of the change, typically 2-3 lines before and after the modified section. + +2. Keep changes focused: Each SEARCH/REPLACE block should address a single, specific change. For multiple changes in a file, use multiple blocks. + +3. Maintain exact matching: Ensure the SEARCH content exists exactly as specified in the original file, including whitespace and indentation. + +4. Limit block size: Keep each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. + +5. Preserve structure: For changes within functions, classes, or nested structures, include the enclosing context (e.g., function signature, class declaration). + +6. Handle imports: When adding new imports, include the entire import section in the SEARCH block, even if only adding one line. + +7. Comment changes: If modifying comments, include them in the SEARCH/REPLACE blocks. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -106,50 +125,80 @@ STATUS: Use 'new' for newly created files, 'modified' for existing files that ar EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. -Example for a modified file with multiple changes and good context: +Examples of good SEARCH/REPLACE blocks for different scenarios: + +1. Modifying a function: + +src/utils/data-processor.ts + +<<<<<<< SEARCH +export function processData(data: any[]): ProcessedData { + return data.map(item => ({ + id: item.id, + name: item.name.toUpperCase(), + value: item.value * 2 + })); +} +======= +export function processData(data: any[]): ProcessedData { + return data.map(item => ({ + id: item.id, + name: item.name.toUpperCase(), + value: item.value * 2, + timestamp: new Date().toISOString() + })); +} +>>>>>>> REPLACE + +modified + +Added a timestamp field to each processed item for better tracking. + + + +2. Adding a new import: + +src/components/UserProfile.tsx + +<<<<<<< SEARCH +import React from 'react'; +import { User } from '../types'; +import { Avatar } from './Avatar'; +======= +import React from 'react'; +import { User } from '../types'; +import { Avatar } from './Avatar'; +import { formatDate } from '../utils/date-helpers'; +>>>>>>> REPLACE + +modified + +Imported the formatDate utility function to properly display user dates. + + + +3. Modifying a configuration object: -src/ai/model-config.ts +src/config/app-config.ts <<<<<<< SEARCH -export const MODEL_CONFIGS: ModelSpecs = { - 'claude-3-5-sonnet-20240620': { - contextWindow: 200000, - maxOutput: 8192, - modelName: 'Claude 3.5 Sonnet', - pricing: { inputCost: 3, outputCost: 15 }, - modelFamily: 'claude', - temperature: { - planningTemperature: 0.5, - codegenTemperature: 0.3, - }, - }, +export const CONFIG = { + apiUrl: 'https://api.example.com', + timeout: 5000, + retryAttempts: 3, }; ======= -export const MODEL_CONFIGS: ModelSpecs = { - 'claude-3-5-sonnet-20240620': { - contextWindow: 200000, - maxOutput: 8192, - modelName: 'Claude 3.5 Sonnet', - pricing: { inputCost: 3, outputCost: 15 }, - modelFamily: 'claude', - temperature: { - planningTemperature: 0.5, - codegenTemperature: 0.3, - }, - }, - 'deepseek-coder': { - contextWindow: 128000, - maxOutput: 8000, - modelName: 'DeepSeek-Coder', - pricing: { inputCost: 0.14, outputCost: 0.28 }, - modelFamily: 'openai-compatible', - }, +export const CONFIG = { + apiUrl: 'https://api.example.com', + timeout: 5000, + retryAttempts: 3, + cacheTimeout: 60000, }; >>>>>>> REPLACE modified -Added a new model configuration for DeepSeek-Coder to the MODEL_CONFIGS object, including its context window, max output, pricing, and model family. +Added a cacheTimeout configuration to implement response caching. @@ -205,4 +254,7 @@ Ensure that: - Your code is easy to understand and maintain. - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. +- Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. +--- +Now, implement the task described above, following the provided implementation plan. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. From 097d342553ecc0b86838ca1cb9b798786fbb7a3b Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 01:06:30 +0200 Subject: [PATCH 12/54] more prompt improvements --- src/templates/codegen-diff-no-plan-prompt.hbs | 164 +++++++++--------- src/templates/codegen-diff-prompt.hbs | 163 ++++++++--------- 2 files changed, 155 insertions(+), 172 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 463aab3..77cc7c6 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -65,11 +65,7 @@ Then, for each file: __FILE_PATH__ -<<<<<<< SEARCH -__ORIGINAL_CONTENT__ -======= -__MODIFIED_CONTENT__ ->>>>>>> REPLACE +__CONTENT_OR_SEARCH_REPLACE_BLOCK__ __STATUS__ @@ -77,50 +73,87 @@ __EXPLANATION__ (if necessary) -Please adhere to the following guidelines: +File Handling Instructions: -FILE_PATH: Use the full path from the project root, without any leading dots. -Example: 'src/components/Button.tsx' +1. For modified files: + Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules below. -For modified files: -- Use one or more SEARCH/REPLACE blocks within the tags. -- Each SEARCH/REPLACE block should focus on a specific, minimal change. -- Provide sufficient context in the SEARCH block to uniquely identify the location of the change, but keep it as small as possible. -- For large changes, break them down into multiple smaller SEARCH/REPLACE blocks. -- Ensure that the SEARCH content exists exactly as specified in the file. -- For changes to object literals or nested structures, focus on the specific key or value being changed, rather than the entire structure. -- Include some unchanged lines before and after the change to provide context. -- Limit each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. +2. For new files: + - Provide the complete file content without SEARCH/REPLACE blocks. + - Set the file status to 'new'. + - Include the full path of the new file. -Please adhere to the following guidelines for SEARCH/REPLACE blocks in modified files: +3. For deleted files: + - Only include the and deleted. -1. Provide sufficient context: Include enough surrounding code to uniquely identify the location of the change, typically 2-3 lines before and after the modified section. +SEARCH/REPLACE Block Rules (for modified files only): -2. Keep changes focused: Each SEARCH/REPLACE block should address a single, specific change. For multiple changes in a file, use multiple blocks. +1. Every SEARCH/REPLACE block must use this exact format: + <<<<<<< SEARCH + [Exact existing code] + ======= + [Modified code] + >>>>>>> REPLACE -3. Maintain exact matching: Ensure the SEARCH content exists exactly as specified in the original file, including whitespace and indentation. +2. Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. -4. Limit block size: Keep each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. +3. SEARCH/REPLACE blocks will replace ALL matching occurrences. -5. Preserve structure: For changes within functions, classes, or nested structures, include the enclosing context (e.g., function signature, class declaration). +4. Include enough lines to make the SEARCH blocks uniquely match the lines to change. -6. Handle imports: When adding new imports, include the entire import section in the SEARCH block, even if only adding one line. +5. Keep SEARCH/REPLACE blocks concise. Break large SEARCH/REPLACE blocks into a series of smaller blocks that each change a small portion of the file. -7. Comment changes: If modifying comments, include them in the SEARCH/REPLACE blocks. +6. Include just the changing lines, and a few surrounding lines if needed for uniqueness. Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. -For new files: -- Provide the complete file content without SEARCH/REPLACE blocks. +7. Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist! -For deleted files: -- Only include the and deleted. +8. To move code within a file, use 2 SEARCH/REPLACE blocks: 1 to delete it from its current location, 1 to insert it in the new location. -STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for files that are being deleted. +Examples: -EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. +1. Modifying a JSON configuration file: -Examples of good SEARCH/REPLACE blocks for different scenarios: + +src/config/app-config.json + +<<<<<<< SEARCH +{ + "apiVersion": "v1", + "server": { + "port": 3000, + "host": "localhost" + }, + "database": { + "url": "mongodb://localhost:27017/myapp", + "name": "myapp" + } +} +======= +{ + "apiVersion": "v2", + "server": { + "port": 3000, + "host": "localhost" + }, + "database": { + "url": "mongodb://localhost:27017/myapp", + "name": "myapp" + }, + "cache": { + "type": "redis", + "url": "redis://localhost:6379" + } +} +>>>>>>> REPLACE + +modified + +Updated API version to v2 and added cache configuration for Redis. + + + +2. Modifying a TypeScript function: -1. Modifying a function: src/utils/data-processor.ts @@ -138,6 +171,7 @@ export function processData(data: any[]): ProcessedData { id: item.id, name: item.name.toUpperCase(), value: item.value * 2, + processed: true, timestamp: new Date().toISOString() })); } @@ -145,59 +179,14 @@ export function processData(data: any[]): ProcessedData { modified -Added a timestamp field to each processed item for better tracking. +Enhanced processData function to include 'processed' flag and timestamp in the output. -2. Adding a new import: - -src/components/UserProfile.tsx - -<<<<<<< SEARCH -import React from 'react'; -import { User } from '../types'; -import { Avatar } from './Avatar'; -======= -import React from 'react'; -import { User } from '../types'; -import { Avatar } from './Avatar'; -import { formatDate } from '../utils/date-helpers'; ->>>>>>> REPLACE - -modified - -Imported the formatDate utility function to properly display user dates. - - - -3. Modifying a configuration object: - -src/config/app-config.ts - -<<<<<<< SEARCH -export const CONFIG = { - apiUrl: 'https://api.example.com', - timeout: 5000, - retryAttempts: 3, -}; -======= -export const CONFIG = { - apiUrl: 'https://api.example.com', - timeout: 5000, - retryAttempts: 3, - cacheTimeout: 60000, -}; ->>>>>>> REPLACE - -modified - -Added a cacheTimeout configuration to implement response caching. - - +3. Example for a new file: -Example for a new file: -src/utils/helpers.ts +src/utils/new-helper.ts import { useState, useEffect } from 'react'; @@ -219,16 +208,17 @@ export const useDebounce = (value: any, delay: number) => { new -Created a new utility hook for debouncing values, useful for optimizing performance in input fields or search functionality. +Created a new utility function for debouncing values, useful for optimizing performance in input fields or search functionality. -Example for a deleted file: +4. Example for a deleted file: + -src/deprecated/OldComponent.tsx +src/deprecated/old-utility.ts deleted -Removed deprecated component that is no longer used in the project. +Removed deprecated utility file as its functions have been replaced by new-helper.ts @@ -248,6 +238,8 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. + +ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! --- -Now, implement the task described above. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. +Now, implement the task described above, following the provided implementation plan. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index fa3d991..c458f04 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -71,11 +71,7 @@ Then, for each file: __FILE_PATH__ -<<<<<<< SEARCH -__ORIGINAL_CONTENT__ -======= -__MODIFIED_CONTENT__ ->>>>>>> REPLACE +__CONTENT_OR_SEARCH_REPLACE_BLOCK__ __STATUS__ @@ -83,51 +79,87 @@ __EXPLANATION__ (if necessary) -Please adhere to the following guidelines: +File Handling Instructions: -FILE_PATH: Use the full path from the project root, without any leading dots. -Example: 'src/components/Button.tsx' +1. For modified files: + Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules below. -For modified files: -- Use one or more SEARCH/REPLACE blocks within the tags. -- Each SEARCH/REPLACE block should focus on a specific, minimal change. -- Provide sufficient context in the SEARCH block to uniquely identify the location of the change, but keep it as small as possible. -- For large changes, break them down into multiple smaller SEARCH/REPLACE blocks. -- Ensure that the SEARCH content exists exactly as specified in the file. -- For changes to object literals or nested structures, focus on the specific key or value being changed, rather than the entire structure. -- If adding a new item to an object or array, include the entire opening of the structure in the search pattern. -- Include some unchanged lines before and after the change to provide context. -- Limit each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. +2. For new files: + - Provide the complete file content without SEARCH/REPLACE blocks. + - Set the file status to 'new'. + - Include the full path of the new file. -Please adhere to the following guidelines for SEARCH/REPLACE blocks in modified files: +3. For deleted files: + - Only include the and deleted. -1. Provide sufficient context: Include enough surrounding code to uniquely identify the location of the change, typically 2-3 lines before and after the modified section. +SEARCH/REPLACE Block Rules (for modified files only): -2. Keep changes focused: Each SEARCH/REPLACE block should address a single, specific change. For multiple changes in a file, use multiple blocks. +1. Every SEARCH/REPLACE block must use this exact format: + <<<<<<< SEARCH + [Exact existing code] + ======= + [Modified code] + >>>>>>> REPLACE -3. Maintain exact matching: Ensure the SEARCH content exists exactly as specified in the original file, including whitespace and indentation. +2. Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. -4. Limit block size: Keep each SEARCH/REPLACE block to a maximum of 10-15 lines for better accuracy. +3. SEARCH/REPLACE blocks will replace ALL matching occurrences. -5. Preserve structure: For changes within functions, classes, or nested structures, include the enclosing context (e.g., function signature, class declaration). +4. Include enough lines to make the SEARCH blocks uniquely match the lines to change. -6. Handle imports: When adding new imports, include the entire import section in the SEARCH block, even if only adding one line. +5. Keep SEARCH/REPLACE blocks concise. Break large SEARCH/REPLACE blocks into a series of smaller blocks that each change a small portion of the file. -7. Comment changes: If modifying comments, include them in the SEARCH/REPLACE blocks. +6. Include just the changing lines, and a few surrounding lines if needed for uniqueness. Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. -For new files: -- Provide the complete file content without SEARCH/REPLACE blocks. +7. Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist! -For deleted files: -- Only include the and deleted. +8. To move code within a file, use 2 SEARCH/REPLACE blocks: 1 to delete it from its current location, 1 to insert it in the new location. -STATUS: Use 'new' for newly created files, 'modified' for existing files that are being updated, and 'deleted' for files that are being deleted. +Examples: -EXPLANATION: Provide a brief explanation for any significant design decisions or non-obvious implementations. +1. Modifying a JSON configuration file: -Examples of good SEARCH/REPLACE blocks for different scenarios: + +src/config/app-config.json + +<<<<<<< SEARCH +{ + "apiVersion": "v1", + "server": { + "port": 3000, + "host": "localhost" + }, + "database": { + "url": "mongodb://localhost:27017/myapp", + "name": "myapp" + } +} +======= +{ + "apiVersion": "v2", + "server": { + "port": 3000, + "host": "localhost" + }, + "database": { + "url": "mongodb://localhost:27017/myapp", + "name": "myapp" + }, + "cache": { + "type": "redis", + "url": "redis://localhost:6379" + } +} +>>>>>>> REPLACE + +modified + +Updated API version to v2 and added cache configuration for Redis. + + + +2. Modifying a TypeScript function: -1. Modifying a function: src/utils/data-processor.ts @@ -145,6 +177,7 @@ export function processData(data: any[]): ProcessedData { id: item.id, name: item.name.toUpperCase(), value: item.value * 2, + processed: true, timestamp: new Date().toISOString() })); } @@ -152,59 +185,14 @@ export function processData(data: any[]): ProcessedData { modified -Added a timestamp field to each processed item for better tracking. +Enhanced processData function to include 'processed' flag and timestamp in the output. -2. Adding a new import: - -src/components/UserProfile.tsx - -<<<<<<< SEARCH -import React from 'react'; -import { User } from '../types'; -import { Avatar } from './Avatar'; -======= -import React from 'react'; -import { User } from '../types'; -import { Avatar } from './Avatar'; -import { formatDate } from '../utils/date-helpers'; ->>>>>>> REPLACE - -modified - -Imported the formatDate utility function to properly display user dates. - - - -3. Modifying a configuration object: - -src/config/app-config.ts - -<<<<<<< SEARCH -export const CONFIG = { - apiUrl: 'https://api.example.com', - timeout: 5000, - retryAttempts: 3, -}; -======= -export const CONFIG = { - apiUrl: 'https://api.example.com', - timeout: 5000, - retryAttempts: 3, - cacheTimeout: 60000, -}; ->>>>>>> REPLACE - -modified - -Added a cacheTimeout configuration to implement response caching. - - +3. Example for a new file: -Example for a new file: -src/utils/helpers.ts +src/utils/new-helper.ts import { useState, useEffect } from 'react'; @@ -226,16 +214,17 @@ export const useDebounce = (value: any, delay: number) => { new -Created a new utility hook for debouncing values, useful for optimizing performance in input fields or search functionality. +Created a new utility function for debouncing values, useful for optimizing performance in input fields or search functionality. -Example for a deleted file: +4. Example for a deleted file: + -src/deprecated/OldComponent.tsx +src/deprecated/old-utility.ts deleted -Removed deprecated component that is no longer used in the project. +Removed deprecated utility file as its functions have been replaced by new-helper.ts @@ -255,6 +244,8 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. + +ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! --- Now, implement the task described above, following the provided implementation plan. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. From 91c50f11e7c24ee527e85a8bf73db3f2c3c4cfed Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 02:04:33 +0200 Subject: [PATCH 13/54] add new gpt-4o model version, adjust temperature --- src/ai/model-config.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ai/model-config.ts b/src/ai/model-config.ts index 0727926..b9c245c 100644 --- a/src/ai/model-config.ts +++ b/src/ai/model-config.ts @@ -9,7 +9,7 @@ export const MODEL_CONFIGS: ModelSpecs = { modelFamily: 'claude', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.3, + codegenTemperature: 0.2, }, }, 'claude-3-opus-20240229': { @@ -20,7 +20,7 @@ export const MODEL_CONFIGS: ModelSpecs = { modelFamily: 'claude', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.3, + codegenTemperature: 0.2, }, }, 'claude-3-sonnet-20240229': { @@ -31,7 +31,7 @@ export const MODEL_CONFIGS: ModelSpecs = { modelFamily: 'claude', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.3, + codegenTemperature: 0.2, }, }, 'claude-3-haiku-20240307': { @@ -42,18 +42,18 @@ export const MODEL_CONFIGS: ModelSpecs = { modelFamily: 'claude', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.3, + codegenTemperature: 0.2, }, }, - 'gpt-4o': { + 'gpt-4o-2024-08-06': { contextWindow: 128000, - maxOutput: 4096, + maxOutput: 16384, modelName: 'GPT 4 Omni', - pricing: { inputCost: 5, outputCost: 15 }, + pricing: { inputCost: 2.5, outputCost: 10 }, modelFamily: 'openai', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.2, + codegenTemperature: 0.1, }, }, 'gpt-4o-mini': { @@ -64,7 +64,7 @@ export const MODEL_CONFIGS: ModelSpecs = { modelFamily: 'openai', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.2, + codegenTemperature: 0.1, }, }, 'llama-3.1-70b-versatile': { From 45908718b96b8fbef1233b59d4d0a76bd0d1b289 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:27:16 +0200 Subject: [PATCH 14/54] feat: use the diff mode specified in the model config --- src/ai/task-workflow.ts | 12 ++++++------ src/cli/index.ts | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index c170c83..7c166f1 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -43,14 +43,14 @@ export async function runAIAssistedTask(options: AiAssistedTaskOptions) { const taskCache = new TaskCache(basePath); const modelKey = await selectModel(options); + const modelConfig = getModelConfig(modelKey); - if (options.diff && modelKey !== 'claude-3-5-sonnet-20240620') { - console.log( - chalk.yellow( - 'Diff-based code modifications are currently only supported with Claude 3.5 Sonnet.', - ), - ); + if (options.diff === undefined) { + options.diff = modelConfig.mode === 'diff'; } + console.log( + chalk.blue(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`), + ); const { taskDescription, instructions } = await getTaskInfo( options, diff --git a/src/cli/index.ts b/src/cli/index.ts index 48165e7..d444185 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -94,9 +94,7 @@ export function cli(_args: string[]) { .option( '-df, --diff', 'Use the new diff mode for AI-generated code modifications', - false, ) - .option('--no-diff', 'Use full file content for updates') .option( '--plan', 'Use the plan mode for AI-generated code modifications', From 56d5c6c94ac8cb61a7ab5c87d66ab809932066cd Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:27:40 +0200 Subject: [PATCH 15/54] add diff mode and other new properties to model config --- src/types/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/types/index.ts b/src/types/index.ts index c3a6308..3bc3247 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -130,12 +130,9 @@ interface LLMPricing { outputCost: number; } -export type ModelFamily = - | 'claude' - | 'openai' - | 'openai-compatible' - | 'groq' - | 'ollama'; +export type ModelFamily = 'claude' | 'openai' | 'openai-compatible' | 'ollama'; + +export type EditingMode = 'diff' | 'whole'; export interface ModelSpec { contextWindow: number; @@ -144,6 +141,9 @@ export interface ModelSpec { pricing: LLMPricing; modelFamily: ModelFamily; temperature?: ModelTemperature; + mode?: EditingMode; + baseURL?: string; + apiKeyEnv?: string; } export interface ModelSpecs { From 756929a033b864a616ad66e06e925e7e7867922a Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:27:56 +0200 Subject: [PATCH 16/54] feat: add deepseek api support revamp model config --- .env.template | 1 + src/ai/generate-ai-response.ts | 40 +++++++++++++++------------------- src/ai/model-config.ts | 36 +++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/.env.template b/.env.template index 924533c..d0f0418 100644 --- a/.env.template +++ b/.env.template @@ -1,4 +1,5 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= GROQ_API_KEY= +DEEPSEEK_API_KEY= MODEL=claude-3-5-sonnet-20240620 diff --git a/src/ai/generate-ai-response.ts b/src/ai/generate-ai-response.ts index c824254..a0b0e5e 100644 --- a/src/ai/generate-ai-response.ts +++ b/src/ai/generate-ai-response.ts @@ -15,10 +15,10 @@ dotenv.config(); interface ModelFamilyConfig { initClient: ( apiKey?: string, + baseURL?: string, ) => ReturnType< typeof createAnthropic | typeof createOpenAI | typeof createOllama >; - apiKeyEnv?: string; } const modelFamilies: Record = { @@ -30,7 +30,6 @@ const modelFamilies: Record = { 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15', }, }), - apiKeyEnv: 'ANTHROPIC_API_KEY', }, openai: { initClient: (apiKey?: string) => @@ -38,22 +37,13 @@ const modelFamilies: Record = { apiKey, compatibility: 'strict', }), - apiKeyEnv: 'OPENAI_API_KEY', }, 'openai-compatible': { - initClient: (apiKey?: string) => + initClient: (apiKey?: string, baseURL?: string) => createOpenAI({ apiKey, + baseURL, }), - apiKeyEnv: 'OPENAI_COMPATIBLE_API_KEY', - }, - groq: { - initClient: (apiKey?: string) => - createOpenAI({ - baseURL: 'https://api.groq.com/openai/v1', - apiKey, - }), - apiKeyEnv: 'GROQ_API_KEY', }, ollama: { initClient: () => @@ -87,6 +77,17 @@ export async function generateAIResponse( typeof createAnthropic | typeof createOpenAI | typeof createOllama >; + let apiKey: string | undefined; + + if (modelConfig.apiKeyEnv) { + apiKey = process.env[modelConfig.apiKeyEnv]; + if (!apiKey) { + throw new Error( + `${modelConfig.apiKeyEnv} is not set in the environment variables.`, + ); + } + } + if (modelFamily === 'ollama') { const ollamaModelName = modelKey.split(':')[1] || 'llama3.1:8b'; client = familyConfig.initClient(); @@ -98,17 +99,9 @@ export async function generateAIResponse( maxOutput: options.maxTokens || modelConfig.maxOutput, modelName: `Ollama ${ollamaModelName}`, }; + } else if (modelFamily === 'openai-compatible') { + client = familyConfig.initClient(apiKey, modelConfig.baseURL); } else { - let apiKey: string | undefined; - - if (familyConfig.apiKeyEnv) { - apiKey = process.env[familyConfig.apiKeyEnv]; - if (!apiKey) { - throw new Error( - `${familyConfig.apiKeyEnv} is not set in the environment variables.`, - ); - } - } client = familyConfig.initClient(apiKey); } @@ -193,6 +186,7 @@ export async function generateAIResponse( throw error; } } + async function confirmCostExceedsThreshold( estimatedCost: number, threshold: number, diff --git a/src/ai/model-config.ts b/src/ai/model-config.ts index b9c245c..81b8cbd 100644 --- a/src/ai/model-config.ts +++ b/src/ai/model-config.ts @@ -11,6 +11,8 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.2, }, + mode: 'diff', + apiKeyEnv: 'ANTHROPIC_API_KEY', }, 'claude-3-opus-20240229': { contextWindow: 200000, @@ -22,6 +24,8 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.2, }, + mode: 'diff', + apiKeyEnv: 'ANTHROPIC_API_KEY', }, 'claude-3-sonnet-20240229': { contextWindow: 200000, @@ -33,6 +37,8 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.2, }, + mode: 'whole', + apiKeyEnv: 'ANTHROPIC_API_KEY', }, 'claude-3-haiku-20240307': { contextWindow: 200000, @@ -44,6 +50,8 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.2, }, + mode: 'whole', + apiKeyEnv: 'ANTHROPIC_API_KEY', }, 'gpt-4o-2024-08-06': { contextWindow: 128000, @@ -55,6 +63,8 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.1, }, + mode: 'diff', + apiKeyEnv: 'OPENAI_API_KEY', }, 'gpt-4o-mini': { contextWindow: 128000, @@ -66,13 +76,36 @@ export const MODEL_CONFIGS: ModelSpecs = { planningTemperature: 0.5, codegenTemperature: 0.1, }, + mode: 'whole', + apiKeyEnv: 'OPENAI_API_KEY', + }, + 'deepseek-coder': { + contextWindow: 128000, + maxOutput: 8000, + modelName: 'DeepSeek-Coder', + pricing: { inputCost: 0.14, outputCost: 0.28 }, + modelFamily: 'openai-compatible', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.1, + }, + mode: 'diff', + baseURL: 'https://api.deepseek.com/beta', + apiKeyEnv: 'DEEPSEEK_API_KEY', }, 'llama-3.1-70b-versatile': { contextWindow: 131072, maxOutput: 8000, modelName: 'Llama 3.1 70B Groq', pricing: { inputCost: 0.15, outputCost: 0.6 }, - modelFamily: 'groq', + modelFamily: 'openai-compatible', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.1, + }, + mode: 'whole', + baseURL: 'https://api.groq.com/openai/v1', + apiKeyEnv: 'GROQ_API_KEY', }, ollama: { contextWindow: 4096, @@ -80,6 +113,7 @@ export const MODEL_CONFIGS: ModelSpecs = { modelName: 'Ollama Model', pricing: { inputCost: 0, outputCost: 0 }, modelFamily: 'ollama', + mode: 'whole', }, }; From 9c46e6d6bab2ba8ca33e0e9b9b2154239e00bc5c Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:45:55 +0200 Subject: [PATCH 17/54] tests: update tests --- src/ai/token-management.ts | 1 - src/cli/index.ts | 6 ++- tests/unit/cost-calculation.test.ts | 7 ++-- tests/unit/token-management.test.ts | 58 +++++++++++++++++++++++------ 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src/ai/token-management.ts b/src/ai/token-management.ts index c200e2e..60eaf9e 100644 --- a/src/ai/token-management.ts +++ b/src/ai/token-management.ts @@ -33,7 +33,6 @@ export function estimateTokenCount(text: string, modelKey: string): number { } case 'openai': case 'openai-compatible': - case 'groq': // OpenAI and OpenAI-compatible models use the same tokenizer return baseTokenCount; default: diff --git a/src/cli/index.ts b/src/cli/index.ts index d444185..4431b9b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -93,7 +93,11 @@ export function cli(_args: string[]) { ) .option( '-df, --diff', - 'Use the new diff mode for AI-generated code modifications', + "Use the new diff mode for AI-generated code modifications (overwrites the model's editing mode settings)", + ) + .option( + '--no-diff', + "Use the whole file edit mode for AI-generated code modifications (overwrites the model's editing mode settings)", ) .option( '--plan', diff --git a/tests/unit/cost-calculation.test.ts b/tests/unit/cost-calculation.test.ts index 2bd3b7f..dd28dce 100644 --- a/tests/unit/cost-calculation.test.ts +++ b/tests/unit/cost-calculation.test.ts @@ -17,7 +17,7 @@ describe('Cost Calculation', () => { it('should calculate cost correctly for GPT models', () => { const usage = { inputTokens: 1000000, outputTokens: 500000 }; - expect(calculateCost('gpt-4o', usage)).toBe(12.5); + expect(calculateCost('gpt-4o-2024-08-06', usage)).toBe(7.5); expect(roundTo(calculateCost('gpt-4o-mini', usage), 2)).toBe(0.45); }); @@ -43,7 +43,7 @@ describe('Cost Calculation', () => { }); it('should estimate cost correctly for GPT models', () => { - expect(estimateCost('gpt-4o', 1000000, 500000)).toBe(12.5); + expect(estimateCost('gpt-4o-2024-08-06', 1000000, 500000)).toBe(7.5); expect(roundTo(estimateCost('gpt-4o-mini', 1000000, 500000), 2)).toBe( 0.45, ); @@ -64,8 +64,9 @@ describe('Cost Calculation', () => { 'claude-3-opus-20240229', 'claude-3-sonnet-20240229', 'claude-3-haiku-20240307', - 'gpt-4o', + 'gpt-4o-2024-08-06', 'gpt-4o-mini', + 'deepseek-coder', 'llama-3.1-70b-versatile', 'ollama', ]); diff --git a/tests/unit/token-management.test.ts b/tests/unit/token-management.test.ts index eb1b764..168013b 100644 --- a/tests/unit/token-management.test.ts +++ b/tests/unit/token-management.test.ts @@ -21,8 +21,10 @@ describe('Token Management', () => { 'claude-3-5-sonnet-20240620', 'claude-3-opus-20240229', 'claude-3-haiku-20240307', - 'gpt-4o', + 'gpt-4o-2024-08-06', 'gpt-4o-mini', + 'llama-3.1-70b-versatile', + 'deepseek-coder', ])('should truncate text to context limit for %s', (model) => { const modelSpecs = getModelConfig(model); if (!modelSpecs) { @@ -49,7 +51,7 @@ describe('Token Management', () => { it('should not truncate text within context limit', () => { const shortText = 'This is a short text.'; - const result = truncateToContextLimit(shortText, 'gpt-4o'); + const result = truncateToContextLimit(shortText, 'gpt-4o-2024-08-06'); expect(result).toBe(shortText); }); @@ -66,14 +68,19 @@ describe('Token Management', () => { text, 'claude-3-5-sonnet-20240620', ); - const gptEstimate = estimateTokenCount(text, 'gpt-4o'); + const gptEstimate = estimateTokenCount(text, 'gpt-4o-2024-08-06'); expect(claudeEstimate).toBeGreaterThan(gptEstimate); }); - it('should estimate tokens correctly for GPT models', () => { + it('should estimate tokens correctly for GPT models and OpenAI compatible models', () => { const text = 'This is a test sentence.'; - const gptEstimate = estimateTokenCount(text, 'gpt-4o'); + const gptEstimate = estimateTokenCount(text, 'gpt-4o-2024-08-06'); expect(gptEstimate).toBe(6); // Actual token count without multiplier + const openaiEstimate = estimateTokenCount( + text, + 'llama-3.1-70b-versatile', + ); + expect(openaiEstimate).toBe(6); // Actual token count without multiplier }); }); @@ -82,29 +89,33 @@ describe('Token Management', () => { const specs = getModelConfig('claude-3-5-sonnet-20240620'); expect(specs).toEqual({ contextWindow: 200000, + mode: 'diff', maxOutput: 8192, modelName: 'Claude 3.5 Sonnet', pricing: { inputCost: 3, outputCost: 15 }, modelFamily: 'claude', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.3, + codegenTemperature: 0.2, }, + apiKeyEnv: 'ANTHROPIC_API_KEY', }); }); it('should return correct specs for GPT models', () => { - const specs = getModelConfig('gpt-4o'); + const specs = getModelConfig('gpt-4o-2024-08-06'); expect(specs).toEqual({ contextWindow: 128000, - maxOutput: 4096, + maxOutput: 16384, modelName: 'GPT 4 Omni', - pricing: { inputCost: 5, outputCost: 15 }, + pricing: { inputCost: 2.5, outputCost: 10 }, modelFamily: 'openai', temperature: { planningTemperature: 0.5, - codegenTemperature: 0.2, + codegenTemperature: 0.1, }, + apiKeyEnv: 'OPENAI_API_KEY', + mode: 'diff', }); }); @@ -115,7 +126,32 @@ describe('Token Management', () => { maxOutput: 8000, modelName: 'Llama 3.1 70B Groq', pricing: { inputCost: 0.15, outputCost: 0.6 }, - modelFamily: 'groq', + modelFamily: 'openai-compatible', + apiKeyEnv: 'GROQ_API_KEY', + mode: 'whole', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.1, + }, + baseURL: 'https://api.groq.com/openai/v1', + }); + }); + + it('should return correct specs for DEEPSEEK models', () => { + const specs = getModelConfig('deepseek-coder'); + expect(specs).toEqual({ + contextWindow: 128000, + maxOutput: 8000, + modelName: 'DeepSeek-Coder', + pricing: { inputCost: 0.14, outputCost: 0.28 }, + modelFamily: 'openai-compatible', + baseURL: 'https://api.deepseek.com/beta', + apiKeyEnv: 'DEEPSEEK_API_KEY', + mode: 'diff', + temperature: { + planningTemperature: 0.5, + codegenTemperature: 0.1, + }, }); }); From 64866ca70be95f350e394f175f5969d112ebc6db Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:51:41 +0200 Subject: [PATCH 18/54] docs: update Docs --- README.md | 28 +++++++++++++++++++--------- USAGE.md | 4 ++-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4dc8d95..34112a6 100644 --- a/README.md +++ b/README.md @@ -165,14 +165,21 @@ codewhisper list-models # CodeWhisper will prompt you to select a model from the list of available models codewhisper task -# To enable the new diff-based code modifications, you can use the following command: -# This will use diff-based code modifications. Please note that this is still experimental and may not work with all models. It is currently only recommended with Claude 3.5 Sonnet. +# The mode of operation for generating code modifications is set automatically based on the model. +# You can override this by using the --diff or --no-diff option. codewhisper task --diff +codewhisper task --no-diff # You can also specify a model directly # Claude-3.5 Sonnet codewhisper task -m claude-3-5-sonnet-20240620 +# GPT-4o +codewhisper task -m gpt-4o-2024-08-06 + +# DeepSeek Coder +codewhisper task -m deepseek-coder + # Or use a local Ollama model (not recommended as it will be slow and inaccurate for comprehensive feature implementation tasks) codewhisper task -m ollama:llama3.1:70b --context-window 131072 --max-tokens 8192 @@ -184,7 +191,7 @@ codewhisper task --undo codewhisper task --redo ``` -> Note: If you are using CodeWhisper's LLM integration with `codewhisper task`, you will need to set the respective environment variable for the model you want to use (e.g., `export ANTHROPIC_API_KEY=your_api_key` or `export OPENAI_API_KEY=your_api_key` or `export GROQ_API_KEY=your_api_key`). +> Note: If you are using CodeWhisper's LLM integration with `codewhisper task`, you will need to set the respective environment variable for the model you want to use (e.g., `export ANTHROPIC_API_KEY=your_api_key` or `export OPENAI_API_KEY=your_api_key` or `export GROQ_API_KEY=your_api_key` or `export DEEPSEEK_API_KEY=your_api_key` ). For more detailed instructions, see the [Installation](#-installation) and [Usage](#-usage) sections. @@ -192,13 +199,16 @@ For more detailed instructions, see the [Installation](#-installation) and [Usag While CodeWhisper supports a variety of providers and models, our current recommendations are based on extensive testing and real-world usage. Here's an overview of the current status: -#### Recommended Models +#### Model Evaluation + +This section is still under development. We are actively testing and evaluating models. -| Model | Provider | Recommendation | Notes | -| ----------------- | --------- | -------------- | --------------------------------------------------------------------- | -| Claude-3.5-Sonnet | Anthropic | Highest | Generates exceptional quality plans and results | -| GPT-4o | OpenAI | Excellent | Produces high-quality plans and good results | -| GPT-4o-mini | OpenAI | Strong | Good quality plans and results, long max output length (16384 tokens) | +| Model | Provider | Recommendation | Editing Mode | Notes | +| ----------------- | --------- | -------------- | ------------ | ----------------------------------------------------------------------------------- | +| Claude-3.5-Sonnet | Anthropic | Highest | Diff | Generates exceptional quality plans and results | +| GPT-4o | OpenAI | Excellent | Diff | Produces high-quality plans and good results, long max output length (16384 tokens) | +| GPT-4o-mini | OpenAI | Strong | Whole | Good quality plans and results, long max output length (16384 tokens) | +| DeepSeek Coder | DeepSeek | Good | Diff | Good quality plans and results, long max output length (16384 tokens) | #### Experimental Support diff --git a/USAGE.md b/USAGE.md index 069dd57..f2d494e 100644 --- a/USAGE.md +++ b/USAGE.md @@ -53,8 +53,8 @@ codewhisper task [options] | `-c, --context ` | Specify files or directories to include in the task context. Can be file paths, directory paths, or glob patterns. Multiple entries should be space-separated. | | `--github-issue` | Use GitHub issue for task input | | `--github-issue-filters ` | Use these filters when fetching issues. Format: comma-separated key:value pairs. Example: labels:p1,assignee:abc Note: see "query parameters" at https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues--parameters for all options. | -| `-df, --diff` | Use diff format for file updates instead of full file content (default: false). Please note that this is still experimental and may not work with all models. It is currently only recommended with Claude 3.5 Sonnet. | -| `--no-diff` | Disable diff mode (default: true) | +| `-df, --diff` | Override the default diff mode for the model. | +| `--no-diff` | Override the default diff mode for the model. | | `--plan` | Use the planning mode, this generates an intermediate plan, which can be modified. Useful for complex tasks. (default: true) | | `--no-plan` | Disable the planning mode. Useful for simple tasks (default: false) | | `-g, --gitignore ` | Path to .gitignore file (default: .gitignore) | From b7bc54520afc1587ac2aad3baf26d44b7564167a Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 08:54:11 +0200 Subject: [PATCH 19/54] docs: update Acknowledgments --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 34112a6..91de0c3 100644 --- a/README.md +++ b/README.md @@ -420,6 +420,7 @@ We welcome contributions to CodeWhisper! Please read our [CONTRIBUTING.md](CONTR - [Commander.js](https://github.com/tj/commander.js/) for CLI support - [fast-glob](https://github.com/mrmlnc/fast-glob) for file matching - [Inquirer.js](https://github.com/SBoudrias/Inquirer.js/) for interactive prompts +- [Vercel AI SDK](https://sdk.vercel.ai/docs/introduction) for the great AI SDK ## 📬 Contact From a3bb5fdcd02f2808be77c6db3d6724fe07c62c3c Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 09:12:26 +0200 Subject: [PATCH 20/54] fix: redo-task should also use the configured diff mode unless overwritten --- src/ai/redo-task.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ai/redo-task.ts b/src/ai/redo-task.ts index 31bba60..65f4114 100644 --- a/src/ai/redo-task.ts +++ b/src/ai/redo-task.ts @@ -45,13 +45,20 @@ export async function redoLastTask(options: AiAssistedTaskOptions) { default: false, }); + const modelConfig = getModelConfig(modelKey); + if (changeModel) { modelKey = await selectModelPrompt(); - console.log( - chalk.blue(`Using new model: ${getModelConfig(modelKey).modelName}`), - ); + console.log(chalk.blue(`Using new model: ${modelConfig.modelName}`)); } + if (options.diff === undefined) { + options.diff = modelConfig.mode === 'diff'; + } + console.log( + chalk.blue(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`), + ); + // Confirmation for changing file selection const changeFiles = await confirm({ message: 'Do you want to change the file selection for code generation?', From 9c4d2bd1872c3d747a84d1bdde417ebd92364a82 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 09:22:36 +0200 Subject: [PATCH 21/54] fix: make flexible whitespace matching more robust --- src/git/apply-changes.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index bfc51ab..b1ddd25 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -127,6 +127,10 @@ async function applyFileChange( } } +function escapeRegExp(string: string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + function applyChange(content: string, search: string, replace: string): string { const trimmedSearch = search.trim(); const trimmedReplace = replace.trim(); @@ -136,8 +140,9 @@ function applyChange(content: string, search: string, replace: string): string { } // If exact match fails, try matching with flexible whitespace - const flexibleSearch = trimmedSearch.replace(/\s+/g, '\\s+'); - const regex = new RegExp(flexibleSearch); + const escapedSearch = escapeRegExp(trimmedSearch); + const flexibleSearch = escapedSearch.replace(/\s+/g, '\\s+'); + const regex = new RegExp(flexibleSearch, 'g'); if (regex.test(content)) { return content.replace(regex, (match) => { From 1b8fd4d62fa79aad092a95aa43f8a6fa8951d67a Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 09:34:44 +0200 Subject: [PATCH 22/54] test: update redo-task tests --- tests/unit/redo-task.test.ts | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/tests/unit/redo-task.test.ts b/tests/unit/redo-task.test.ts index 30d158b..d68db56 100644 --- a/tests/unit/redo-task.test.ts +++ b/tests/unit/redo-task.test.ts @@ -1,7 +1,6 @@ import path from 'node:path'; import { confirm } from '@inquirer/prompts'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { getModelConfig } from '../../src/ai/model-config'; import { redoLastTask } from '../../src/ai/redo-task'; import { handleNoPlanWorkflow, @@ -17,14 +16,13 @@ vi.mock('@inquirer/prompts'); vi.mock('../../src/utils/task-cache'); vi.mock('../../src/interactive/select-model-prompt'); vi.mock('../../src/interactive/select-files-prompt'); -vi.mock('../../src/ai/model-config'); vi.mock('../../src/ai/task-workflow'); vi.mock('../../src/core/file-processor'); describe('redoLastTask', () => { const mockOptions = { path: path.join('/', 'test', 'path'), - model: 'test-model', + model: 'claude-3-5-sonnet-20240620', plan: true, } as AiAssistedTaskOptions; @@ -33,7 +31,7 @@ describe('redoLastTask', () => { generatedPlan: 'Last generated plan', taskDescription: 'Last task description', instructions: 'Last instructions', - model: 'last-model', + model: 'claude-3-5-sonnet-20240620', }; const mockProcessedFiles = [ @@ -71,7 +69,7 @@ describe('redoLastTask', () => { generatedPlan: 'Last generated plan', taskDescription: 'Last task description', instructions: 'Last instructions', - model: 'last-model', + model: 'claude-3-5-sonnet-20240620', }; const mockTaskCache = { @@ -86,12 +84,12 @@ describe('redoLastTask', () => { await redoLastTask(mockOptions); expect(handlePlanWorkflow).toHaveBeenCalledWith( - expect.objectContaining({ model: 'last-model' }), + expect.objectContaining({ model: 'claude-3-5-sonnet-20240620' }), expect.stringMatching(/[\\\/]test[\\\/]path$/), expect.any(Object), // TaskCache expect.objectContaining(mockLastTaskData), mockProcessedFiles, - 'last-model', + 'claude-3-5-sonnet-20240620', ); }); @@ -101,7 +99,7 @@ describe('redoLastTask', () => { generatedPlan: '', // Empty string for no-plan workflow taskDescription: 'Last task description', instructions: 'Last instructions', - model: 'last-model', + model: 'claude-3-5-sonnet-20240620', }; const mockTaskCache = { @@ -116,12 +114,12 @@ describe('redoLastTask', () => { await redoLastTask(mockOptions); expect(handleNoPlanWorkflow).toHaveBeenCalledWith( - expect.objectContaining({ model: 'last-model' }), + expect.objectContaining({ model: 'claude-3-5-sonnet-20240620' }), expect.stringMatching(/[\\\/]test[\\\/]path$/), expect.any(Object), // TaskCache expect.objectContaining(mockLastTaskDataNoPlan), mockProcessedFiles, - 'last-model', + 'claude-3-5-sonnet-20240620', ); }); @@ -131,7 +129,7 @@ describe('redoLastTask', () => { generatedPlan: 'Last generated plan', taskDescription: 'Last task description', instructions: 'Last instructions', - model: 'last-model', + model: 'claude-3-5-sonnet-20240620', }; const mockTaskCache = { @@ -141,26 +139,22 @@ describe('redoLastTask', () => { // biome-ignore lint/suspicious/noExplicitAny: explicit any is fine here vi.mocked(TaskCache).mockImplementation(() => mockTaskCache as any); vi.mocked(confirm).mockResolvedValueOnce(true).mockResolvedValueOnce(false); - vi.mocked(selectModelPrompt).mockResolvedValue('new-model'); - vi.mocked(getModelConfig).mockReturnValue({ - modelName: 'New Model', - // biome-ignore lint/suspicious/noExplicitAny: explicit any is fine here - } as any); + vi.mocked(selectModelPrompt).mockResolvedValue('gpt-4o-2024-08-06'); vi.mocked(processFiles).mockResolvedValue(mockProcessedFiles as FileInfo[]); await redoLastTask(mockOptions); expect(selectModelPrompt).toHaveBeenCalled(); expect(handlePlanWorkflow).toHaveBeenCalledWith( - expect.objectContaining({ model: 'new-model' }), + expect.objectContaining({ model: 'gpt-4o-2024-08-06' }), expect.stringMatching(/[\\\/]test[\\\/]path$/), expect.any(Object), // TaskCache expect.objectContaining({ ...mockLastTaskData, - model: 'new-model', + model: 'gpt-4o-2024-08-06', }), mockProcessedFiles, - 'new-model', + 'gpt-4o-2024-08-06', ); // Check if the task cache was updated with the new model @@ -168,7 +162,7 @@ describe('redoLastTask', () => { expect.stringMatching(/[\\\/]test[\\\/]path$/), expect.objectContaining({ ...mockLastTaskData, - model: 'new-model', + model: 'gpt-4o-2024-08-06', }), ); }); From 846874f1b395eca5ca4e9f7021729dc76d49e36c Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 09:58:47 +0200 Subject: [PATCH 23/54] fix: template improvements --- src/templates/codegen-diff-no-plan-prompt.hbs | 40 +++++++++++++------ src/templates/codegen-diff-prompt.hbs | 40 +++++++++++++------ 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 77cc7c6..2dfc848 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -95,19 +95,27 @@ SEARCH/REPLACE Block Rules (for modified files only): [Modified code] >>>>>>> REPLACE -2. Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. +Explanation of the SEARCH/REPLACE format: -3. SEARCH/REPLACE blocks will replace ALL matching occurrences. +- The start of search block: <<<<<<< SEARCH +- A contiguous chunk of lines to search for in the existing source code +- The dividing line: ======= +- The lines to replace into the source code +- The end of the replace block: >>>>>>> REPLACE -4. Include enough lines to make the SEARCH blocks uniquely match the lines to change. +Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc. -5. Keep SEARCH/REPLACE blocks concise. Break large SEARCH/REPLACE blocks into a series of smaller blocks that each change a small portion of the file. +*SEARCH/REPLACE* blocks will replace *all* matching occurrences. +Include enough lines to make the SEARCH blocks uniquely match the lines to change. -6. Include just the changing lines, and a few surrounding lines if needed for uniqueness. Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. +Keep *SEARCH/REPLACE* blocks concise. +Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file. +Include just the changing lines, and a few surrounding lines if needed for uniqueness. +Do not include long runs of unchanging lines in *SEARCH/REPLACE* blocks. -7. Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist! +Only create *SEARCH/REPLACE* blocks for files that the user has added to the chat! -8. To move code within a file, use 2 SEARCH/REPLACE blocks: 1 to delete it from its current location, 1 to insert it in the new location. +To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location. Examples: @@ -158,28 +166,36 @@ Updated API version to v2 and added cache configuration for Redis. src/utils/data-processor.ts <<<<<<< SEARCH +import { ProcessedData } from './types'; + +export function processData(data: any[]): ProcessedData { +======= +import { ProcessedData } from './types'; +import { getCurrentTimestamp } from './time-utils'; + export function processData(data: any[]): ProcessedData { +>>>>>>> REPLACE + +<<<<<<< SEARCH return data.map(item => ({ id: item.id, name: item.name.toUpperCase(), value: item.value * 2 })); -} ======= -export function processData(data: any[]): ProcessedData { return data.map(item => ({ id: item.id, name: item.name.toUpperCase(), value: item.value * 2, processed: true, - timestamp: new Date().toISOString() + timestamp: getCurrentTimestamp() })); -} >>>>>>> REPLACE +} modified -Enhanced processData function to include 'processed' flag and timestamp in the output. +Enhanced processData function to include 'processed' flag and timestamp in the output. Also added an import for getCurrentTimestamp from time-utils. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index c458f04..4499419 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -101,19 +101,27 @@ SEARCH/REPLACE Block Rules (for modified files only): [Modified code] >>>>>>> REPLACE -2. Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. +Explanation of the SEARCH/REPLACE format: -3. SEARCH/REPLACE blocks will replace ALL matching occurrences. +- The start of search block: <<<<<<< SEARCH +- A contiguous chunk of lines to search for in the existing source code +- The dividing line: ======= +- The lines to replace into the source code +- The end of the replace block: >>>>>>> REPLACE -4. Include enough lines to make the SEARCH blocks uniquely match the lines to change. +Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc. -5. Keep SEARCH/REPLACE blocks concise. Break large SEARCH/REPLACE blocks into a series of smaller blocks that each change a small portion of the file. +*SEARCH/REPLACE* blocks will replace *all* matching occurrences. +Include enough lines to make the SEARCH blocks uniquely match the lines to change. -6. Include just the changing lines, and a few surrounding lines if needed for uniqueness. Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. +Keep *SEARCH/REPLACE* blocks concise. +Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file. +Include just the changing lines, and a few surrounding lines if needed for uniqueness. +Do not include long runs of unchanging lines in *SEARCH/REPLACE* blocks. -7. Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist! +Only create *SEARCH/REPLACE* blocks for files that the user has added to the chat! -8. To move code within a file, use 2 SEARCH/REPLACE blocks: 1 to delete it from its current location, 1 to insert it in the new location. +To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location. Examples: @@ -164,28 +172,36 @@ Updated API version to v2 and added cache configuration for Redis. src/utils/data-processor.ts <<<<<<< SEARCH +import { ProcessedData } from './types'; + +export function processData(data: any[]): ProcessedData { +======= +import { ProcessedData } from './types'; +import { getCurrentTimestamp } from './time-utils'; + export function processData(data: any[]): ProcessedData { +>>>>>>> REPLACE + +<<<<<<< SEARCH return data.map(item => ({ id: item.id, name: item.name.toUpperCase(), value: item.value * 2 })); -} ======= -export function processData(data: any[]): ProcessedData { return data.map(item => ({ id: item.id, name: item.name.toUpperCase(), value: item.value * 2, processed: true, - timestamp: new Date().toISOString() + timestamp: getCurrentTimestamp() })); -} >>>>>>> REPLACE +} modified -Enhanced processData function to include 'processed' flag and timestamp in the output. +Enhanced processData function to include 'processed' flag and timestamp in the output. Also added an import for getCurrentTimestamp from time-utils. From 21a69d14f63c8f38e1a1c9a328c746914b021916 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:02:38 +0200 Subject: [PATCH 24/54] chore: template improvements --- src/templates/codegen-diff-no-plan-prompt.hbs | 1 + src/templates/codegen-diff-prompt.hbs | 1 + src/templates/codegen-no-plan-prompt.hbs | 1 + src/templates/codegen-prompt.hbs | 1 + 4 files changed, 4 insertions(+) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 2dfc848..62d4a26 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -254,6 +254,7 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. +- Only modfiy files that were provided in the section. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 4499419..70fcf73 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -260,6 +260,7 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. +- Only modfiy files that were provided in the section. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! diff --git a/src/templates/codegen-no-plan-prompt.hbs b/src/templates/codegen-no-plan-prompt.hbs index f21f248..b202a23 100644 --- a/src/templates/codegen-no-plan-prompt.hbs +++ b/src/templates/codegen-no-plan-prompt.hbs @@ -154,6 +154,7 @@ FORMAT: Instructions for how to format your response. - Your changes are consistent with the existing codebase. - You haven't introduced any potential bugs or performance issues. - Your code is easy to understand and maintain. + - Only modfiy files that were provided in the section. - You complete all necessary work to fully implement the task. diff --git a/src/templates/codegen-prompt.hbs b/src/templates/codegen-prompt.hbs index 815e2e8..4521ab1 100644 --- a/src/templates/codegen-prompt.hbs +++ b/src/templates/codegen-prompt.hbs @@ -163,6 +163,7 @@ FORMAT: Instructions for how to format your response. - Your changes are consistent with the existing codebase. - You haven't introduced any potential bugs or performance issues. - Your code is easy to understand and maintain. + - Only modfiy files that were provided in the section. - You complete all necessary work to fully implement the task. From 26f6bd6fcc5c1ae21e05b769e9653c2c0c221efd Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:10:27 +0200 Subject: [PATCH 25/54] allow multiple identical changes per file again --- src/git/apply-changes.ts | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index b1ddd25..9257bd3 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -84,20 +84,12 @@ async function applyFileChange( await fs.writeFile(fullPath, updatedContent); } else if (file.changes) { let currentContent = await fs.readFile(fullPath, 'utf-8'); - const appliedChanges = new Set(); for (const change of file.changes) { - if (!appliedChanges.has(change.search)) { - currentContent = applyChange( - currentContent, - change.search, - change.replace, - ); - appliedChanges.add(change.search); - } else { - console.warn( - `Skipping duplicate change for ${file.path}: ${change.search}`, - ); - } + currentContent = applyChange( + currentContent, + change.search, + change.replace, + ); } await fs.writeFile(fullPath, currentContent); } else if (file.content) { @@ -135,17 +127,19 @@ function applyChange(content: string, search: string, replace: string): string { const trimmedSearch = search.trim(); const trimmedReplace = replace.trim(); - if (content.includes(trimmedSearch)) { - return content.replace(trimmedSearch, trimmedReplace); + const escapedSearch = escapeRegExp(trimmedSearch); + const regex = new RegExp(escapedSearch, 'g'); + + if (regex.test(content)) { + return content.replace(regex, trimmedReplace); } // If exact match fails, try matching with flexible whitespace - const escapedSearch = escapeRegExp(trimmedSearch); const flexibleSearch = escapedSearch.replace(/\s+/g, '\\s+'); - const regex = new RegExp(flexibleSearch, 'g'); + const flexibleRegex = new RegExp(flexibleSearch, 'g'); - if (regex.test(content)) { - return content.replace(regex, (match) => { + if (flexibleRegex.test(content)) { + return content.replace(flexibleRegex, (match) => { const leadingWhitespace = match.match(/^\s*/)?.[0] || ''; const trailingWhitespace = match.match(/\s*$/)?.[0] || ''; return leadingWhitespace + trimmedReplace + trailingWhitespace; From febab4db2ed8b17beea492b8b2b1615f5286a7ed Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:13:51 +0200 Subject: [PATCH 26/54] more prompt tweaking --- src/templates/codegen-diff-prompt.hbs | 187 +++++++++++--------------- 1 file changed, 80 insertions(+), 107 deletions(-) diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 70fcf73..1a8347f 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -101,146 +101,119 @@ SEARCH/REPLACE Block Rules (for modified files only): [Modified code] >>>>>>> REPLACE -Explanation of the SEARCH/REPLACE format: +2. Create small, atomic SEARCH/REPLACE blocks: + - Each block should modify no more than 5 lines of code. + - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. + - Focus on one logical change per block (e.g., updating a single function or adding one import statement). -- The start of search block: <<<<<<< SEARCH -- A contiguous chunk of lines to search for in the existing source code -- The dividing line: ======= -- The lines to replace into the source code -- The end of the replace block: >>>>>>> REPLACE +3. Include minimal context: + - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. + - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. -Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc. +4. Ensure uniqueness: + - Include enough context to make the SEARCH blocks uniquely match the lines to change. + - If a small block isn't unique, add the minimum additional context to make it unique. -*SEARCH/REPLACE* blocks will replace *all* matching occurrences. -Include enough lines to make the SEARCH blocks uniquely match the lines to change. +5. Exact matching: + - Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. -Keep *SEARCH/REPLACE* blocks concise. -Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file. -Include just the changing lines, and a few surrounding lines if needed for uniqueness. -Do not include long runs of unchanging lines in *SEARCH/REPLACE* blocks. +6. Replace all occurrences: + - SEARCH/REPLACE blocks will replace ALL matching occurrences in the file. -Only create *SEARCH/REPLACE* blocks for files that the user has added to the chat! +7. Moving code: + - To move code within a file, use 2 separate SEARCH/REPLACE blocks: + 1) to delete it from its current location + 2) to insert it in the new location -To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location. +8. File scope: + - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. -Examples: +Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: -1. Modifying a JSON configuration file: +Instead of: -src/config/app-config.json +src/utils/logger.ts <<<<<<< SEARCH -{ - "apiVersion": "v1", - "server": { - "port": 3000, - "host": "localhost" - }, - "database": { - "url": "mongodb://localhost:27017/myapp", - "name": "myapp" - } -} +import winston from 'winston'; + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.simple(), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export default logger; ======= -{ - "apiVersion": "v2", - "server": { - "port": 3000, - "host": "localhost" - }, - "database": { - "url": "mongodb://localhost:27017/myapp", - "name": "myapp" - }, - "cache": { - "type": "redis", - "url": "redis://localhost:6379" - } -} +import winston from 'winston'; + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export const setLogLevel = (level: string) => { + logger.level = level; +}; + +export default logger; >>>>>>> REPLACE modified - -Updated API version to v2 and added cache configuration for Redis. - -2. Modifying a TypeScript function: +Use multiple smaller blocks: -src/utils/data-processor.ts +src/utils/logger.ts <<<<<<< SEARCH -import { ProcessedData } from './types'; +import winston from 'winston'; -export function processData(data: any[]): ProcessedData { +const logger = winston.createLogger({ + level: 'info', ======= -import { ProcessedData } from './types'; -import { getCurrentTimestamp } from './time-utils'; +import winston from 'winston'; -export function processData(data: any[]): ProcessedData { +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', >>>>>>> REPLACE <<<<<<< SEARCH - return data.map(item => ({ - id: item.id, - name: item.name.toUpperCase(), - value: item.value * 2 - })); + format: winston.format.simple(), ======= - return data.map(item => ({ - id: item.id, - name: item.name.toUpperCase(), - value: item.value * 2, - processed: true, - timestamp: getCurrentTimestamp() - })); + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), >>>>>>> REPLACE -} - -modified - -Enhanced processData function to include 'processed' flag and timestamp in the output. Also added an import for getCurrentTimestamp from time-utils. - - - -3. Example for a new file: - - -src/utils/new-helper.ts - -import { useState, useEffect } from 'react'; - -export const useDebounce = (value: any, delay: number) => { - const [debouncedValue, setDebouncedValue] = useState(value); - useEffect(() => { - const handler = setTimeout(() => { - setDebouncedValue(value); - }, delay); - - return () => { - clearTimeout(handler); - }; - }, [value, delay]); - - return debouncedValue; +<<<<<<< SEARCH +export default logger; +======= +export const setLogLevel = (level: string) => { + logger.level = level; }; - -new - -Created a new utility function for debouncing values, useful for optimizing performance in input fields or search functionality. - - - -4. Example for a deleted file: - -src/deprecated/old-utility.ts -deleted +export default logger; +>>>>>>> REPLACE + +modified -Removed deprecated utility file as its functions have been replaced by new-helper.ts +Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. @@ -260,7 +233,7 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. -- Only modfiy files that were provided in the section. +- Only modify files that were provided in the section. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! From 7b17d703c42eb16d536ec5fb9dba399a950c6239 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:15:06 +0200 Subject: [PATCH 27/54] more prompt tweaks --- src/templates/codegen-diff-no-plan-prompt.hbs | 189 ++++++++---------- 1 file changed, 81 insertions(+), 108 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 62d4a26..6f0e182 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -95,146 +95,119 @@ SEARCH/REPLACE Block Rules (for modified files only): [Modified code] >>>>>>> REPLACE -Explanation of the SEARCH/REPLACE format: +2. Create small, atomic SEARCH/REPLACE blocks: + - Each block should modify no more than 5 lines of code. + - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. + - Focus on one logical change per block (e.g., updating a single function or adding one import statement). -- The start of search block: <<<<<<< SEARCH -- A contiguous chunk of lines to search for in the existing source code -- The dividing line: ======= -- The lines to replace into the source code -- The end of the replace block: >>>>>>> REPLACE +3. Include minimal context: + - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. + - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. -Every *SEARCH* section must *EXACTLY MATCH* the existing source code, character for character, including all comments, docstrings, etc. +4. Ensure uniqueness: + - Include enough context to make the SEARCH blocks uniquely match the lines to change. + - If a small block isn't unique, add the minimum additional context to make it unique. -*SEARCH/REPLACE* blocks will replace *all* matching occurrences. -Include enough lines to make the SEARCH blocks uniquely match the lines to change. +5. Exact matching: + - Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. -Keep *SEARCH/REPLACE* blocks concise. -Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file. -Include just the changing lines, and a few surrounding lines if needed for uniqueness. -Do not include long runs of unchanging lines in *SEARCH/REPLACE* blocks. +6. Replace all occurrences: + - SEARCH/REPLACE blocks will replace ALL matching occurrences in the file. -Only create *SEARCH/REPLACE* blocks for files that the user has added to the chat! +7. Moving code: + - To move code within a file, use 2 separate SEARCH/REPLACE blocks: + 1) to delete it from its current location + 2) to insert it in the new location -To move code within a file, use 2 *SEARCH/REPLACE* blocks: 1 to delete it from its current location, 1 to insert it in the new location. +8. File scope: + - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. -Examples: +Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: -1. Modifying a JSON configuration file: +Instead of: -src/config/app-config.json +src/utils/logger.ts <<<<<<< SEARCH -{ - "apiVersion": "v1", - "server": { - "port": 3000, - "host": "localhost" - }, - "database": { - "url": "mongodb://localhost:27017/myapp", - "name": "myapp" - } -} +import winston from 'winston'; + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.simple(), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export default logger; ======= -{ - "apiVersion": "v2", - "server": { - "port": 3000, - "host": "localhost" - }, - "database": { - "url": "mongodb://localhost:27017/myapp", - "name": "myapp" - }, - "cache": { - "type": "redis", - "url": "redis://localhost:6379" - } -} +import winston from 'winston'; + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export const setLogLevel = (level: string) => { + logger.level = level; +}; + +export default logger; >>>>>>> REPLACE modified - -Updated API version to v2 and added cache configuration for Redis. - -2. Modifying a TypeScript function: +Use multiple smaller blocks: -src/utils/data-processor.ts +src/utils/logger.ts <<<<<<< SEARCH -import { ProcessedData } from './types'; +import winston from 'winston'; -export function processData(data: any[]): ProcessedData { +const logger = winston.createLogger({ + level: 'info', ======= -import { ProcessedData } from './types'; -import { getCurrentTimestamp } from './time-utils'; +import winston from 'winston'; -export function processData(data: any[]): ProcessedData { +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', >>>>>>> REPLACE <<<<<<< SEARCH - return data.map(item => ({ - id: item.id, - name: item.name.toUpperCase(), - value: item.value * 2 - })); + format: winston.format.simple(), ======= - return data.map(item => ({ - id: item.id, - name: item.name.toUpperCase(), - value: item.value * 2, - processed: true, - timestamp: getCurrentTimestamp() - })); + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), >>>>>>> REPLACE -} - -modified - -Enhanced processData function to include 'processed' flag and timestamp in the output. Also added an import for getCurrentTimestamp from time-utils. - - - -3. Example for a new file: - - -src/utils/new-helper.ts - -import { useState, useEffect } from 'react'; - -export const useDebounce = (value: any, delay: number) => { - const [debouncedValue, setDebouncedValue] = useState(value); - useEffect(() => { - const handler = setTimeout(() => { - setDebouncedValue(value); - }, delay); - - return () => { - clearTimeout(handler); - }; - }, [value, delay]); - - return debouncedValue; +<<<<<<< SEARCH +export default logger; +======= +export const setLogLevel = (level: string) => { + logger.level = level; }; - -new - -Created a new utility function for debouncing values, useful for optimizing performance in input fields or search functionality. - - - -4. Example for a deleted file: - -src/deprecated/old-utility.ts -deleted +export default logger; +>>>>>>> REPLACE + +modified -Removed deprecated utility file as its functions have been replaced by new-helper.ts +Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. @@ -254,9 +227,9 @@ Ensure that: - You complete all necessary work to fully implement the task. - SEARCH blocks contain enough context to uniquely identify the location of changes. - Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. -- Only modfiy files that were provided in the section. +- Only modify files that were provided in the section. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! --- -Now, implement the task described above, following the provided implementation plan. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. +Now, implement the task described above. Take your time to think through the problem and craft an elegant, efficient, and complete solution that fully addresses the task requirements and integrates seamlessly with the existing codebase. From d622b56bf05fb08b6ca3210e49a30afbb3bd48cd Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:51:08 +0200 Subject: [PATCH 28/54] try with system prompt --- src/ai/generate-ai-response.ts | 25 ++++- src/ai/task-workflow.ts | 10 ++ src/templates/codegen-diff-prompt.hbs | 126 +++----------------------- src/templates/diff-system-prompt.hbs | 124 +++++++++++++++++++++++++ src/types/index.ts | 10 ++ 5 files changed, 180 insertions(+), 115 deletions(-) create mode 100644 src/templates/diff-system-prompt.hbs diff --git a/src/ai/generate-ai-response.ts b/src/ai/generate-ai-response.ts index a0b0e5e..0aa7d77 100644 --- a/src/ai/generate-ai-response.ts +++ b/src/ai/generate-ai-response.ts @@ -4,7 +4,11 @@ import { generateText } from 'ai'; import chalk from 'chalk'; import dotenv from 'dotenv'; import { createOllama } from 'ollama-ai-provider'; -import type { GenerateAIResponseOptions, ModelFamily } from '../types'; +import type { + GenerateAIResponseOptions, + GenerateTextOptions, + ModelFamily, +} from '../types'; import getLogger from '../utils/logger'; import { calculateCost, estimateCost } from './cost-calculation'; import { getModelConfig, getModelFamily } from './model-config'; @@ -140,8 +144,24 @@ export async function generateAIResponse( } logger.info('AI Prompt', { processedPrompt }); - try { + const modelName = + modelFamily === 'ollama' + ? modelKey.split(':')[1] || 'llama3.1:8b' + : modelKey; + + const generateTextOptions: GenerateTextOptions = { + model: client(modelName), + maxTokens: modelConfig.maxOutput, + temperature: taskTemperature, + prompt: processedPrompt, + }; + + if (options.systemPrompt) { + console.log('Using system prompt:', options.systemPrompt); + generateTextOptions.system = options.systemPrompt; + } + const result = await generateText({ model: modelFamily === 'ollama' @@ -149,6 +169,7 @@ export async function generateAIResponse( : client(modelKey), maxTokens: modelConfig.maxOutput, temperature: taskTemperature, + system: options.systemPrompt ?? '', prompt: processedPrompt, }); diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index 7c166f1..2ca2af8 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -386,6 +386,7 @@ async function generateAndApplyCode( codegenTemplatePath, 'utf-8', ); + const codegenCustomData = await prepareCodegenCustomData( codegenTemplateContent, taskCache, @@ -486,6 +487,13 @@ async function generateCode( modelKey: string, options: AiAssistedTaskOptions, ): Promise { + let systemPromptContent = undefined; + + if (options.diff) { + const systemPromptPath = getTemplatePath('diff-system-prompt'); + systemPromptContent = await fs.readFile(systemPromptPath, 'utf-8'); + } + const spinner = ora('Generating AI Code Modifications...').start(); const modelConfig = getModelConfig(modelKey); @@ -497,6 +505,7 @@ async function generateCode( contextWindow: options.contextWindow, maxTokens: options.maxTokens, logAiInteractions: options.logAiInteractions, + systemPrompt: systemPromptContent, }); } else { generatedCode = await generateAIResponse( @@ -505,6 +514,7 @@ async function generateCode( maxCostThreshold: options.maxCostThreshold, model: modelKey, logAiInteractions: options.logAiInteractions, + systemPrompt: systemPromptContent, }, modelConfig.temperature?.codegenTemperature, ); diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 1a8347f..0e7f530 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -82,7 +82,7 @@ __EXPLANATION__ (if necessary) File Handling Instructions: 1. For modified files: - Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules below. + Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules in your system prompt. 2. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -92,128 +92,28 @@ File Handling Instructions: 3. For deleted files: - Only include the and deleted. -SEARCH/REPLACE Block Rules (for modified files only): - -1. Every SEARCH/REPLACE block must use this exact format: - <<<<<<< SEARCH - [Exact existing code] - ======= - [Modified code] - >>>>>>> REPLACE - -2. Create small, atomic SEARCH/REPLACE blocks: - - Each block should modify no more than 5 lines of code. - - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. - - Focus on one logical change per block (e.g., updating a single function or adding one import statement). - -3. Include minimal context: - - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. - - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. - -4. Ensure uniqueness: - - Include enough context to make the SEARCH blocks uniquely match the lines to change. - - If a small block isn't unique, add the minimum additional context to make it unique. - -5. Exact matching: - - Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. - -6. Replace all occurrences: - - SEARCH/REPLACE blocks will replace ALL matching occurrences in the file. - -7. Moving code: - - To move code within a file, use 2 separate SEARCH/REPLACE blocks: - 1) to delete it from its current location - 2) to insert it in the new location - -8. File scope: - - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. - -Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: - -Instead of: - - -src/utils/logger.ts - -<<<<<<< SEARCH -import winston from 'winston'; - -const logger = winston.createLogger({ - level: 'info', - format: winston.format.simple(), - transports: [ - new winston.transports.Console(), - new winston.transports.File({ filename: 'error.log', level: 'error' }), - new winston.transports.File({ filename: 'combined.log' }), - ], -}); - -export default logger; -======= -import winston from 'winston'; - -const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ), - transports: [ - new winston.transports.Console(), - new winston.transports.File({ filename: 'error.log', level: 'error' }), - new winston.transports.File({ filename: 'combined.log' }), - ], -}); - -export const setLogLevel = (level: string) => { - logger.level = level; -}; - -export default logger; ->>>>>>> REPLACE - -modified - - -Use multiple smaller blocks: +Here's a simple example of a SEARCH/REPLACE block: -src/utils/logger.ts +src/utils/math-helper.ts <<<<<<< SEARCH -import winston from 'winston'; - -const logger = winston.createLogger({ - level: 'info', -======= -import winston from 'winston'; - -const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', ->>>>>>> REPLACE - -<<<<<<< SEARCH - format: winston.format.simple(), -======= - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ), ->>>>>>> REPLACE - -<<<<<<< SEARCH -export default logger; +export function add(a: number, b: number): number { + return a + b; +} ======= -export const setLogLevel = (level: string) => { - logger.level = level; -}; +export function add(a: number, b: number): number { + return a + b; +} -export default logger; +export function subtract(a: number, b: number): number { + return a - b; +} >>>>>>> REPLACE modified -Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. +Added a new subtract function to the math-helper utility. diff --git a/src/templates/diff-system-prompt.hbs b/src/templates/diff-system-prompt.hbs new file mode 100644 index 0000000..d6be8ae --- /dev/null +++ b/src/templates/diff-system-prompt.hbs @@ -0,0 +1,124 @@ +SEARCH/REPLACE Block Rules (for modified files only): + +1. Every SEARCH/REPLACE block must use this exact format: + <<<<<<< SEARCH + [Exact existing code] + ======= + [Modified code] + >>>>>>> REPLACE + +2. Create small, atomic SEARCH/REPLACE blocks: + - Each block should modify no more than 5 lines of code. + - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. + - Focus on one logical change per block (e.g., updating a single function or adding one import statement). + +3. Include minimal context: + - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. + - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. + +4. Ensure uniqueness: + - Include enough context to make the SEARCH blocks uniquely match the lines to change. + - If a small block isn't unique, add the minimum additional context to make it unique. + +5. Exact matching: + - Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. + +6. Replace all occurrences: + - SEARCH/REPLACE blocks will replace ALL matching occurrences in the file. + +7. Moving code: + - To move code within a file, use 2 separate SEARCH/REPLACE blocks: + 1) to delete it from its current location + 2) to insert it in the new location + +8. File scope: + - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. + +Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: + +Instead of: + + +src/utils/logger.ts + +<<<<<<< SEARCH +import winston from 'winston'; + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.simple(), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export default logger; +======= +import winston from 'winston'; + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: 'error.log', level: 'error' }), + new winston.transports.File({ filename: 'combined.log' }), + ], +}); + +export const setLogLevel = (level: string) => { + logger.level = level; +}; + +export default logger; +>>>>>>> REPLACE + +modified + + +Use multiple smaller blocks: + + +src/utils/logger.ts + +<<<<<<< SEARCH +import winston from 'winston'; + +const logger = winston.createLogger({ + level: 'info', +======= +import winston from 'winston'; + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', +>>>>>>> REPLACE + +<<<<<<< SEARCH + format: winston.format.simple(), +======= + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), +>>>>>>> REPLACE + +<<<<<<< SEARCH +export default logger; +======= +export const setLogLevel = (level: string) => { + logger.level = level; +}; + +export default logger; +>>>>>>> REPLACE + +modified + +Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. + + diff --git a/src/types/index.ts b/src/types/index.ts index 3bc3247..9f19534 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -117,6 +117,7 @@ export interface GenerateAIResponseOptions { contextWindow?: number; maxTokens?: number; logAiInteractions?: boolean; + systemPrompt?: string; } export interface ApplyChangesOptions { @@ -183,3 +184,12 @@ export interface AIFileInfo { status: 'new' | 'modified' | 'deleted'; explanation?: string; } + +export interface GenerateTextOptions { + // biome-ignore lint/suspicious/noExplicitAny: it's fine here + model: any; + maxTokens: number; + temperature: number; + system?: string; + prompt: string; +} From 6cb438b4a8445a5a9499edb475f3a6d0efe6bc45 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 10:55:29 +0200 Subject: [PATCH 29/54] prompt tweaking --- src/templates/task-plan-prompt.hbs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/templates/task-plan-prompt.hbs b/src/templates/task-plan-prompt.hbs index 9469ba9..7af23c0 100644 --- a/src/templates/task-plan-prompt.hbs +++ b/src/templates/task-plan-prompt.hbs @@ -19,6 +19,8 @@ Create a comprehensive implementation plan for the given issue. Your plan should Note: Focus solely on the technical implementation. Ignore any mentions of human tasks or non-technical aspects. If any part of the task is unclear or seems to conflict with best practices, highlight it and suggest alternatives. +The plan can add new files to the codebase, but it should only modify files that are included in the section. + Encoded in XML tags, here is what you will be given: TASK: Detailed information about the task, including context and requirements. From 7f6dd89b35b6944ceb5996035b25bd58d3db0367 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 11:19:27 +0200 Subject: [PATCH 30/54] feat: a new approach --- package.json | 1 + pnpm-lock.yaml | 9 ++++ src/ai/parsers/search-replace-parser.ts | 56 +++++++++++++++++++++++++ src/git/apply-changes.ts | 41 +----------------- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index e3a050f..e6b7deb 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "inquirer-file-tree-selection-prompt": "2.0.5", "isbinaryfile": "5.0.2", "micromatch": "4.0.7", + "node-diff3": "3.1.2", "ollama-ai-provider": "0.11.0", "ora": "8.0.1", "piscina": "4.6.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02b4617..2f40ab6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: micromatch: specifier: 4.0.7 version: 4.0.7 + node-diff3: + specifier: 3.1.2 + version: 3.1.2 ollama-ai-provider: specifier: 0.11.0 version: 0.11.0(zod@3.23.8) @@ -2156,6 +2159,10 @@ packages: node-addon-api@3.2.1: resolution: {integrity: sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==} + node-diff3@3.1.2: + resolution: {integrity: sha512-wUd9TWy059I8mZdH6G3LPNlAEfxDvXtn/RcyFrbqL3v34WlDxn+Mh4HDhOwWuaMk/ROVepe5tTpnGHbve6Db2g==} + engines: {node: '>14'} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -5096,6 +5103,8 @@ snapshots: node-addon-api@3.2.1: optional: true + node-diff3@3.1.2: {} + node-domexception@1.0.0: {} node-emoji@2.1.3: diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 4efe880..95de263 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,3 +1,5 @@ +import fs from 'fs-extra'; +import * as Diff3 from 'node-diff3'; import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; import { handleDeletedFiles } from './common-parser'; @@ -49,6 +51,18 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { if (fileInfo.changes.length === 0) { console.warn(`No changes found for modified file: ${path}`); fileInfo.content = content; + } else { + // Apply changes and store the entire modified content + try { + const originalContent = fs.readFileSync(path, 'utf-8'); + fileInfo.content = applySearchReplace( + originalContent, + fileInfo.changes, + ); + } catch (error) { + console.error(`Error reading or modifying file ${path}:`, error); + fileInfo.content = content; // Fallback to AI-provided content + } } } else if (status === 'new') { fileInfo.content = content; @@ -61,6 +75,7 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { return files; } + function parseSearchReplaceBlocks(content: string): AIFileChange[] { console.log('Parsing search/replace blocks. Content:', content); @@ -84,3 +99,44 @@ function parseSearchReplaceBlocks(content: string): AIFileChange[] { console.log(`Found ${changes.length} valid changes`); return changes; } + +export function applySearchReplace( + source: string, + searchReplaceBlocks: AIFileChange[], +): string { + const result = source.split('\n'); + + for (const block of searchReplaceBlocks) { + const searchArray = block.search.split('\n'); + const replaceArray = block.replace.split('\n'); + + // Create a patch + const patch = Diff3.diffPatch(searchArray, replaceArray); + + // Find the location in the result that matches the search + let startIndex = -1; + for (let i = 0; i <= result.length - searchArray.length; i++) { + if (result.slice(i, i + searchArray.length).join('\n') === block.search) { + startIndex = i; + break; + } + } + + if (startIndex !== -1) { + // Apply the patch at the found location + const changedSection = Diff3.patch( + result.slice(startIndex, startIndex + searchArray.length), + patch, + ); + if (Array.isArray(changedSection)) { + result.splice(startIndex, searchArray.length, ...changedSection); + } else { + console.warn(`Failed to apply patch for search block: ${block.search}`); + } + } else { + console.warn(`Search block not found: ${block.search}`); + } + } + + return result.join('\n'); +} diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index 9257bd3..d9e6f2b 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import chalk from 'chalk'; import { applyPatch } from 'diff'; import fs from 'fs-extra'; +import { applySearchReplace } from '../ai/parsers/search-replace-parser'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; export async function applyChanges({ @@ -84,13 +85,7 @@ async function applyFileChange( await fs.writeFile(fullPath, updatedContent); } else if (file.changes) { let currentContent = await fs.readFile(fullPath, 'utf-8'); - for (const change of file.changes) { - currentContent = applyChange( - currentContent, - change.search, - change.replace, - ); - } + currentContent = applySearchReplace(currentContent, file.changes); await fs.writeFile(fullPath, currentContent); } else if (file.content) { await fs.writeFile(fullPath, file.content); @@ -118,35 +113,3 @@ async function applyFileChange( throw error; } } - -function escapeRegExp(string: string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function applyChange(content: string, search: string, replace: string): string { - const trimmedSearch = search.trim(); - const trimmedReplace = replace.trim(); - - const escapedSearch = escapeRegExp(trimmedSearch); - const regex = new RegExp(escapedSearch, 'g'); - - if (regex.test(content)) { - return content.replace(regex, trimmedReplace); - } - - // If exact match fails, try matching with flexible whitespace - const flexibleSearch = escapedSearch.replace(/\s+/g, '\\s+'); - const flexibleRegex = new RegExp(flexibleSearch, 'g'); - - if (flexibleRegex.test(content)) { - return content.replace(flexibleRegex, (match) => { - const leadingWhitespace = match.match(/^\s*/)?.[0] || ''; - const trailingWhitespace = match.match(/\s*$/)?.[0] || ''; - return leadingWhitespace + trimmedReplace + trailingWhitespace; - }); - } - - throw new Error( - 'Failed to apply changes: search content not found in the file', - ); -} From 35f864c31126e5e8c7cfa4ab142c8ff9530be320 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 11:29:10 +0200 Subject: [PATCH 31/54] fix parser --- src/ai/parsers/search-replace-parser.ts | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 95de263..0c39e74 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,4 +1,3 @@ -import fs from 'fs-extra'; import * as Diff3 from 'node-diff3'; import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; @@ -37,7 +36,6 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { console.log(`Parsing file: ${path}`); console.log(`Status: ${status}`); - console.log(`Content: ${content}`); const fileInfo: AIFileInfo = { path, @@ -47,22 +45,11 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { }; if (status === 'modified') { - fileInfo.changes = parseSearchReplaceBlocks(content); - if (fileInfo.changes.length === 0) { - console.warn(`No changes found for modified file: ${path}`); - fileInfo.content = content; + const changes = parseSearchReplaceBlocks(content); + if (changes.length > 0) { + fileInfo.changes = changes; } else { - // Apply changes and store the entire modified content - try { - const originalContent = fs.readFileSync(path, 'utf-8'); - fileInfo.content = applySearchReplace( - originalContent, - fileInfo.changes, - ); - } catch (error) { - console.error(`Error reading or modifying file ${path}:`, error); - fileInfo.content = content; // Fallback to AI-provided content - } + fileInfo.content = content; } } else if (status === 'new') { fileInfo.content = content; From 254a5d154fe9a6bcfe584120570745db1f203ae0 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 11:34:50 +0200 Subject: [PATCH 32/54] remove extraneous console.log --- src/ai/generate-ai-response.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ai/generate-ai-response.ts b/src/ai/generate-ai-response.ts index 0aa7d77..b777483 100644 --- a/src/ai/generate-ai-response.ts +++ b/src/ai/generate-ai-response.ts @@ -158,7 +158,7 @@ export async function generateAIResponse( }; if (options.systemPrompt) { - console.log('Using system prompt:', options.systemPrompt); + console.log(chalk.cyan('Using system prompt')); generateTextOptions.system = options.systemPrompt; } From 5add5e2b1e53474b6dbfc0697921315957d3b944 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 11:49:10 +0200 Subject: [PATCH 33/54] prompt improvements --- src/templates/codegen-diff-no-plan-prompt.hbs | 164 ++++-------------- src/templates/codegen-diff-prompt.hbs | 38 ++-- src/templates/diff-system-prompt.hbs | 68 ++++++++ 3 files changed, 125 insertions(+), 145 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 6f0e182..9e3fdd1 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -76,7 +76,7 @@ __EXPLANATION__ (if necessary) File Handling Instructions: 1. For modified files: - Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules below. + Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules in your system prompt. 2. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. @@ -86,148 +86,54 @@ File Handling Instructions: 3. For deleted files: - Only include the and deleted. -SEARCH/REPLACE Block Rules (for modified files only): - -1. Every SEARCH/REPLACE block must use this exact format: - <<<<<<< SEARCH - [Exact existing code] - ======= - [Modified code] - >>>>>>> REPLACE - -2. Create small, atomic SEARCH/REPLACE blocks: - - Each block should modify no more than 5 lines of code. - - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. - - Focus on one logical change per block (e.g., updating a single function or adding one import statement). - -3. Include minimal context: - - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. - - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. - -4. Ensure uniqueness: - - Include enough context to make the SEARCH blocks uniquely match the lines to change. - - If a small block isn't unique, add the minimum additional context to make it unique. - -5. Exact matching: - - Every SEARCH section must EXACTLY MATCH the existing source code, character for character, including all comments, docstrings, whitespace, etc. - -6. Replace all occurrences: - - SEARCH/REPLACE blocks will replace ALL matching occurrences in the file. - -7. Moving code: - - To move code within a file, use 2 separate SEARCH/REPLACE blocks: - 1) to delete it from its current location - 2) to insert it in the new location - -8. File scope: - - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. - -Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: - -Instead of: - - -src/utils/logger.ts - -<<<<<<< SEARCH -import winston from 'winston'; - -const logger = winston.createLogger({ - level: 'info', - format: winston.format.simple(), - transports: [ - new winston.transports.Console(), - new winston.transports.File({ filename: 'error.log', level: 'error' }), - new winston.transports.File({ filename: 'combined.log' }), - ], -}); - -export default logger; -======= -import winston from 'winston'; - -const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ), - transports: [ - new winston.transports.Console(), - new winston.transports.File({ filename: 'error.log', level: 'error' }), - new winston.transports.File({ filename: 'combined.log' }), - ], -}); - -export const setLogLevel = (level: string) => { - logger.level = level; -}; - -export default logger; ->>>>>>> REPLACE - -modified - - -Use multiple smaller blocks: +Here's a simple example of a SEARCH/REPLACE block: -src/utils/logger.ts +src/utils/math-helper.ts <<<<<<< SEARCH -import winston from 'winston'; - -const logger = winston.createLogger({ - level: 'info', -======= -import winston from 'winston'; - -const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', ->>>>>>> REPLACE - -<<<<<<< SEARCH - format: winston.format.simple(), -======= - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ), ->>>>>>> REPLACE - -<<<<<<< SEARCH -export default logger; +export function add(a: number, b: number): number { + return a + b; +} ======= -export const setLogLevel = (level: string) => { - logger.level = level; -}; +export function add(a: number, b: number): number { + return a + b; +} -export default logger; +export function subtract(a: number, b: number): number { + return a - b; +} >>>>>>> REPLACE modified -Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. +Added a new subtract function to the math-helper utility. Ensure that: -- You have thoroughly analyzed the task and planned your implementation strategy. -- Everything specified in the task description and instructions is implemented. -- All new or modified files contain the full code or necessary changes. -- The content includes all necessary imports, function definitions, and exports. -- The code is clean, maintainable, efficient, and considers performance implications. -- The code is properly formatted and follows the project's coding standards. -- Necessary comments for clarity are included if needed. -- Any conceptual or high-level descriptions are translated into actual, executable code. -- You've considered and handled potential edge cases. -- Your changes are consistent with the existing codebase. -- You haven't introduced any potential bugs or performance issues. -- Your code is easy to understand and maintain. -- You complete all necessary work to fully implement the task. -- SEARCH blocks contain enough context to uniquely identify the location of changes. -- Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. -- Only modify files that were provided in the section. +- You've thoroughly analyzed the task, and implementation strategy. +- All task requirements are fully implemented. +- New or modified files contain complete, necessary changes. +- All required imports, function definitions, and exports are included. +- Code is clean, maintainable, efficient, and performance-conscious. +- Code formatting adheres to project standards. +- Necessary comments are included for clarity. +- Conceptual descriptions are translated into executable code. +- Potential edge cases are considered and handled. +- Changes are consistent with the existing codebase. +- No new bugs or performance issues are introduced. +- Code remains easy to understand and maintain. +- SEARCH/REPLACE blocks: + - Contain enough context to uniquely identify change locations. + - Are focused, minimal, and target specific changes. + - Are ordered logically for sequential application without conflicts. + - Combine multiple changes to the same code block when possible. + - Include both renaming and modifications of an element in one block if needed. + - Are based on the code state after applying previous blocks. + - Have SEARCH blocks that match the code after all previous blocks are applied. +- Only files provided in the section are modified. +- All necessary work to fully implement the task is completed. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 0e7f530..4221832 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -118,22 +118,28 @@ Added a new subtract function to the math-helper utility. Ensure that: -- You have thoroughly analyzed the task and plan and have planned your implementation strategy. -- Everything specified in the task description and plan is implemented. -- All new or modified files contain the full code or necessary changes. -- The content includes all necessary imports, function definitions, and exports. -- The code is clean, maintainable, efficient, and considers performance implications. -- The code is properly formatted and follows the project's coding standards. -- Necessary comments for clarity are included if needed. -- Any conceptual or high-level descriptions are translated into actual, executable code. -- You've considered and handled potential edge cases. -- Your changes are consistent with the existing codebase. -- You haven't introduced any potential bugs or performance issues. -- Your code is easy to understand and maintain. -- You complete all necessary work to fully implement the task. -- SEARCH blocks contain enough context to uniquely identify the location of changes. -- Each SEARCH/REPLACE block is focused and minimal, targeting specific changes. -- Only modify files that were provided in the section. +- You've thoroughly analyzed the task, plan, and implementation strategy. +- All task requirements are fully implemented. +- New or modified files contain complete, necessary changes. +- All required imports, function definitions, and exports are included. +- Code is clean, maintainable, efficient, and performance-conscious. +- Code formatting adheres to project standards. +- Necessary comments are included for clarity. +- Conceptual descriptions are translated into executable code. +- Potential edge cases are considered and handled. +- Changes are consistent with the existing codebase. +- No new bugs or performance issues are introduced. +- Code remains easy to understand and maintain. +- SEARCH/REPLACE blocks: + - Contain enough context to uniquely identify change locations. + - Are focused, minimal, and target specific changes. + - Are ordered logically for sequential application without conflicts. + - Combine multiple changes to the same code block when possible. + - Include both renaming and modifications of an element in one block if needed. + - Are based on the code state after applying previous blocks. + - Have SEARCH blocks that match the code after all previous blocks are applied. +- Only files provided in the section are modified. +- All necessary work to fully implement the task is completed. ONLY EVER RETURN CODE IN A SEARCH/REPLACE BLOCK FOR MODIFIED FILES! diff --git a/src/templates/diff-system-prompt.hbs b/src/templates/diff-system-prompt.hbs index d6be8ae..cd51885 100644 --- a/src/templates/diff-system-prompt.hbs +++ b/src/templates/diff-system-prompt.hbs @@ -122,3 +122,71 @@ export default logger; Updated logger configuration to use environment variable for log level, changed format to include timestamp and use JSON, and added a setLogLevel function. + +9. Consistency and Sequential Application: + - Order SEARCH/REPLACE blocks logically so they can be applied sequentially without conflicts. + - When making multiple changes to the same code block, combine them into a single SEARCH/REPLACE block if possible. + - For renaming and modifying the same element, include both changes in one block. + - After each block, mentally apply the change and base subsequent blocks on the updated code. + - Ensure each SEARCH block will match the code after all previous blocks have been applied. + - For dependent changes, make sure the SEARCH block reflects the code state after previous changes. + +Example of combining related changes: + + +src/utils/logger.ts + +<<<<<<< SEARCH +const getLogger = (logAiInteractions: boolean) => { + if (!logAiInteractions) { + return { + info: () => {}, + error: () => {}, + }; + } + + const date = new Date().toISOString().split('T')[0]; + const logFileName = `codewhisper-${date}.log`; + + return winston.createLogger({ + level: 'info', + // ... rest of the logger configuration + }); +}; + +export default getLogger; +======= +const createLogger = (logLevel: string = 'info') => { + const date = new Date().toISOString().split('T')[0]; + const logFileName = `codewhisper-${date}.log`; + + return winston.createLogger({ + level: logLevel, + // ... rest of the logger configuration + }); +}; + +const getLogger = (logAiInteractions: boolean, logLevel: string = 'info') => { + if (!logAiInteractions) { + return { + error: () => {}, + warn: () => {}, + info: () => {}, + debug: () => {}, + }; + } + return createLogger(logLevel); +}; + +export const setLogLevel = (level: string) => { + logger.level = level; +}; + +export default getLogger; +>>>>>>> REPLACE + +modified + +Combined multiple changes into a single block: renamed getLogger to createLogger, added logLevel parameter, created a new getLogger function that uses createLogger, and added setLogLevel function. + + From 9aa81f569cb9e4502dcc45f0d334428c1572d11e Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 12:14:13 +0200 Subject: [PATCH 34/54] we'll get there --- src/templates/codegen-diff-no-plan-prompt.hbs | 1 - src/templates/codegen-diff-prompt.hbs | 1 - src/templates/diff-system-prompt.hbs | 64 +++++++++++++++++-- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 9e3fdd1..5786554 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -128,7 +128,6 @@ Ensure that: - Contain enough context to uniquely identify change locations. - Are focused, minimal, and target specific changes. - Are ordered logically for sequential application without conflicts. - - Combine multiple changes to the same code block when possible. - Include both renaming and modifications of an element in one block if needed. - Are based on the code state after applying previous blocks. - Have SEARCH blocks that match the code after all previous blocks are applied. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index 4221832..e3bc1b9 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -134,7 +134,6 @@ Ensure that: - Contain enough context to uniquely identify change locations. - Are focused, minimal, and target specific changes. - Are ordered logically for sequential application without conflicts. - - Combine multiple changes to the same code block when possible. - Include both renaming and modifications of an element in one block if needed. - Are based on the code state after applying previous blocks. - Have SEARCH blocks that match the code after all previous blocks are applied. diff --git a/src/templates/diff-system-prompt.hbs b/src/templates/diff-system-prompt.hbs index cd51885..1388970 100644 --- a/src/templates/diff-system-prompt.hbs +++ b/src/templates/diff-system-prompt.hbs @@ -125,13 +125,11 @@ Updated logger configuration to use environment variable for log level, changed 9. Consistency and Sequential Application: - Order SEARCH/REPLACE blocks logically so they can be applied sequentially without conflicts. - - When making multiple changes to the same code block, combine them into a single SEARCH/REPLACE block if possible. - - For renaming and modifying the same element, include both changes in one block. - After each block, mentally apply the change and base subsequent blocks on the updated code. - Ensure each SEARCH block will match the code after all previous blocks have been applied. - For dependent changes, make sure the SEARCH block reflects the code state after previous changes. -Example of combining related changes: +Example of correctly sequencing changes when renaming and modifying a function: src/utils/logger.ts @@ -150,11 +148,46 @@ const getLogger = (logAiInteractions: boolean) => { return winston.createLogger({ level: 'info', - // ... rest of the logger configuration + format: winston.format.simple(), + transports: [ + new winston.transports.File({ + filename: path.join(process.cwd(), logFileName), + }), + ], }); }; +======= +const createLogger = (logLevel: string = 'info') => { + const date = new Date().toISOString().split('T')[0]; + const logFileName = `codewhisper-${date}.log`; -export default getLogger; + return winston.createLogger({ + level: logLevel, + format: winston.format.simple(), + transports: [ + new winston.transports.File({ + filename: path.join(process.cwd(), logFileName), + }), + ], + }); +}; +>>>>>>> REPLACE + +<<<<<<< SEARCH +const createLogger = (logLevel: string = 'info') => { + const date = new Date().toISOString().split('T')[0]; + const logFileName = `codewhisper-${date}.log`; + + return winston.createLogger({ + level: logLevel, + format: winston.format.simple(), + transports: [ + new winston.transports.File({ + filename: path.join(process.cwd(), logFileName), + }), + ], + }); +}; ======= const createLogger = (logLevel: string = 'info') => { const date = new Date().toISOString().split('T')[0]; @@ -162,10 +195,23 @@ const createLogger = (logLevel: string = 'info') => { return winston.createLogger({ level: logLevel, - // ... rest of the logger configuration + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.File({ + filename: path.join(process.cwd(), logFileName), + }), + new winston.transports.Console(), + ], }); }; +>>>>>>> REPLACE +<<<<<<< SEARCH +export default getLogger; +======= const getLogger = (logAiInteractions: boolean, logLevel: string = 'info') => { if (!logAiInteractions) { return { @@ -187,6 +233,10 @@ export default getLogger; modified -Combined multiple changes into a single block: renamed getLogger to createLogger, added logLevel parameter, created a new getLogger function that uses createLogger, and added setLogLevel function. +This example shows the correct sequence of changes: +1. Rename getLogger to createLogger and modify its parameters. +2. Update the internals of createLogger. +3. Create a new getLogger function that uses createLogger, add setLogLevel, and update the export. +Each SEARCH block reflects the state of the code after the previous changes have been applied. From 8ed381549d3710db97fe61e5bcfd081d1f2fd87a Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 20:59:41 +0200 Subject: [PATCH 35/54] improve prompt templates --- src/templates/codegen-diff-no-plan-prompt.hbs | 3 + src/templates/codegen-diff-prompt.hbs | 3 + src/templates/diff-system-prompt.hbs | 132 ++---------------- 3 files changed, 20 insertions(+), 118 deletions(-) diff --git a/src/templates/codegen-diff-no-plan-prompt.hbs b/src/templates/codegen-diff-no-plan-prompt.hbs index 5786554..b769e3b 100644 --- a/src/templates/codegen-diff-no-plan-prompt.hbs +++ b/src/templates/codegen-diff-no-plan-prompt.hbs @@ -77,6 +77,9 @@ File Handling Instructions: 1. For modified files: Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules in your system prompt. + - Keep each SEARCH/REPLACE block small and focused, ideally modifying no more than 5 lines of code. + - Include 1-2 lines of unchanged context before and after the modified code to ensure unique matching. + - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. 2. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. diff --git a/src/templates/codegen-diff-prompt.hbs b/src/templates/codegen-diff-prompt.hbs index e3bc1b9..3b1c2f3 100644 --- a/src/templates/codegen-diff-prompt.hbs +++ b/src/templates/codegen-diff-prompt.hbs @@ -83,6 +83,9 @@ File Handling Instructions: 1. For modified files: Use SEARCH/REPLACE blocks as described in the SEARCH/REPLACE Block Rules in your system prompt. + - Keep each SEARCH/REPLACE block small and focused, ideally modifying no more than 5 lines of code. + - Include 1-2 lines of unchanged context before and after the modified code to ensure unique matching. + - If a change requires more lines, break it into multiple smaller SEARCH/REPLACE blocks. 2. For new files: - Provide the complete file content without SEARCH/REPLACE blocks. diff --git a/src/templates/diff-system-prompt.hbs b/src/templates/diff-system-prompt.hbs index 1388970..4a8d740 100644 --- a/src/templates/diff-system-prompt.hbs +++ b/src/templates/diff-system-prompt.hbs @@ -13,7 +13,8 @@ SEARCH/REPLACE Block Rules (for modified files only): - Focus on one logical change per block (e.g., updating a single function or adding one import statement). 3. Include minimal context: - - Add only 1-2 unchanged lines before and after the modified code for context, if necessary. + - Add 1-2 unchanged lines before and after the modified code for context. + - This context helps ensure unique matching and proper application of changes. - Do not include long runs of unchanging lines in SEARCH/REPLACE blocks. 4. Ensure uniqueness: @@ -34,6 +35,12 @@ SEARCH/REPLACE Block Rules (for modified files only): 8. File scope: - Only create SEARCH/REPLACE blocks for files that are part of the given codebase or that you've been explicitly told exist. +9. Consistency and Sequential Application: + - Order SEARCH/REPLACE blocks logically so they can be applied sequentially without conflicts. + - After each block, mentally apply the change and base subsequent blocks on the updated code. + - Ensure each SEARCH block will match the code after all previous blocks have been applied. + - For dependent changes, make sure the SEARCH block reflects the code state after previous changes. + Example of breaking down a large change into multiple smaller SEARCH/REPLACE blocks: Instead of: @@ -123,120 +130,9 @@ Updated logger configuration to use environment variable for log level, changed -9. Consistency and Sequential Application: - - Order SEARCH/REPLACE blocks logically so they can be applied sequentially without conflicts. - - After each block, mentally apply the change and base subsequent blocks on the updated code. - - Ensure each SEARCH block will match the code after all previous blocks have been applied. - - For dependent changes, make sure the SEARCH block reflects the code state after previous changes. - -Example of correctly sequencing changes when renaming and modifying a function: - - -src/utils/logger.ts - -<<<<<<< SEARCH -const getLogger = (logAiInteractions: boolean) => { - if (!logAiInteractions) { - return { - info: () => {}, - error: () => {}, - }; - } - - const date = new Date().toISOString().split('T')[0]; - const logFileName = `codewhisper-${date}.log`; - - return winston.createLogger({ - level: 'info', - format: winston.format.simple(), - transports: [ - new winston.transports.File({ - filename: path.join(process.cwd(), logFileName), - }), - ], - }); -}; -======= -const createLogger = (logLevel: string = 'info') => { - const date = new Date().toISOString().split('T')[0]; - const logFileName = `codewhisper-${date}.log`; - - return winston.createLogger({ - level: logLevel, - format: winston.format.simple(), - transports: [ - new winston.transports.File({ - filename: path.join(process.cwd(), logFileName), - }), - ], - }); -}; ->>>>>>> REPLACE - -<<<<<<< SEARCH -const createLogger = (logLevel: string = 'info') => { - const date = new Date().toISOString().split('T')[0]; - const logFileName = `codewhisper-${date}.log`; - - return winston.createLogger({ - level: logLevel, - format: winston.format.simple(), - transports: [ - new winston.transports.File({ - filename: path.join(process.cwd(), logFileName), - }), - ], - }); -}; -======= -const createLogger = (logLevel: string = 'info') => { - const date = new Date().toISOString().split('T')[0]; - const logFileName = `codewhisper-${date}.log`; - - return winston.createLogger({ - level: logLevel, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ), - transports: [ - new winston.transports.File({ - filename: path.join(process.cwd(), logFileName), - }), - new winston.transports.Console(), - ], - }); -}; ->>>>>>> REPLACE - -<<<<<<< SEARCH -export default getLogger; -======= -const getLogger = (logAiInteractions: boolean, logLevel: string = 'info') => { - if (!logAiInteractions) { - return { - error: () => {}, - warn: () => {}, - info: () => {}, - debug: () => {}, - }; - } - return createLogger(logLevel); -}; - -export const setLogLevel = (level: string) => { - logger.level = level; -}; - -export default getLogger; ->>>>>>> REPLACE - -modified - -This example shows the correct sequence of changes: -1. Rename getLogger to createLogger and modify its parameters. -2. Update the internals of createLogger. -3. Create a new getLogger function that uses createLogger, add setLogLevel, and update the export. -Each SEARCH block reflects the state of the code after the previous changes have been applied. - - +Remember: +- Keep SEARCH/REPLACE blocks small and focused. +- Provide sufficient context for unique matching. +- Ensure logical ordering of blocks for sequential application. +- Base each block on the state of the code after applying previous blocks. +- Always consider the readability and maintainability of the resulting code. From 292858ba91ad634449b7c450b0dd3bb1344d2c1c Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:05:37 +0200 Subject: [PATCH 36/54] remove file.diff --- src/git/apply-changes.ts | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/git/apply-changes.ts b/src/git/apply-changes.ts index d9e6f2b..a5fc001 100644 --- a/src/git/apply-changes.ts +++ b/src/git/apply-changes.ts @@ -1,6 +1,5 @@ import path from 'node:path'; import chalk from 'chalk'; -import { applyPatch } from 'diff'; import fs from 'fs-extra'; import { applySearchReplace } from '../ai/parsers/search-replace-parser'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; @@ -54,13 +53,7 @@ async function applyFileChange( console.log( chalk.yellow(`[DRY RUN] Would modify file: ${file.path}`), ); - if (file.diff) { - console.log( - chalk.gray( - `[DRY RUN] Diff preview:\n${JSON.stringify(file.diff, null, 2)}`, - ), - ); - } else if (file.changes) { + if (file.changes) { console.log( chalk.gray( `[DRY RUN] Changes preview:\n${JSON.stringify(file.changes, null, 2)}`, @@ -74,26 +67,20 @@ async function applyFileChange( ); } } else { - if (file.diff) { - const currentContent = await fs.readFile(fullPath, 'utf-8'); - const updatedContent = applyPatch(currentContent, file.diff); - if (updatedContent === false) { - throw new Error( - `Failed to apply patch to file: ${file.path}\nA common cause is that the file was not sent to the LLM and it hallucinated the content. Try running the task again (task --redo) and selecting the problematic file.`, - ); - } - await fs.writeFile(fullPath, updatedContent); - } else if (file.changes) { - let currentContent = await fs.readFile(fullPath, 'utf-8'); - currentContent = applySearchReplace(currentContent, file.changes); - await fs.writeFile(fullPath, currentContent); + const currentContent = await fs.readFile(fullPath, 'utf-8'); + let updatedContent: string; + + if (file.changes) { + updatedContent = applySearchReplace(currentContent, file.changes); } else if (file.content) { - await fs.writeFile(fullPath, file.content); + updatedContent = file.content; } else { throw new Error( - `No content, diff, or changes provided for modified file: ${file.path}`, + `No content or changes provided for modified file: ${file.path}`, ); } + + await fs.writeFile(fullPath, updatedContent); console.log(chalk.green(`Modified file: ${file.path}`)); } break; From cbe4126340775cb2c00a03688bc7acb5df56aa14 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:05:50 +0200 Subject: [PATCH 37/54] add preprocess function --- src/ai/parsers/common-parser.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ai/parsers/common-parser.ts b/src/ai/parsers/common-parser.ts index e60e434..10cea76 100644 --- a/src/ai/parsers/common-parser.ts +++ b/src/ai/parsers/common-parser.ts @@ -66,3 +66,10 @@ export function handleDeletedFiles( } return files; } + +export function preprocessBlock(block: string): string { + return block + .trim() + .replace(/\r\n/g, '\n') + .replace(/[ \t]+$/gm, ''); +} From 0b41793030b359dc36495b51ab710dcd853d58bb Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:06:20 +0200 Subject: [PATCH 38/54] implement more reliable patching --- src/ai/parsers/search-replace-parser.ts | 130 ++++++++++++++++++------ 1 file changed, 100 insertions(+), 30 deletions(-) diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 0c39e74..2eaa07f 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,7 +1,7 @@ import * as Diff3 from 'node-diff3'; import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; -import { handleDeletedFiles } from './common-parser'; +import { handleDeletedFiles, preprocessBlock } from './common-parser'; export function parseSearchReplaceFiles(response: string): AIFileInfo[] { let files: AIFileInfo[] = []; @@ -73,8 +73,8 @@ function parseSearchReplaceBlocks(content: string): AIFileChange[] { // biome-ignore lint/suspicious/noAssignInExpressions: avoid infinite loop while ((match = blockRegex.exec(content)) !== null) { - const search = match[1].trim(); - const replace = match[2].trim(); + const search = preprocessBlock(match[1].trim()); + const replace = preprocessBlock(match[2].trim()); if (search && replace) { changes.push({ search, replace }); @@ -91,39 +91,109 @@ export function applySearchReplace( source: string, searchReplaceBlocks: AIFileChange[], ): string { - const result = source.split('\n'); + let result = source; for (const block of searchReplaceBlocks) { - const searchArray = block.search.split('\n'); - const replaceArray = block.replace.split('\n'); - - // Create a patch - const patch = Diff3.diffPatch(searchArray, replaceArray); - - // Find the location in the result that matches the search - let startIndex = -1; - for (let i = 0; i <= result.length - searchArray.length; i++) { - if (result.slice(i, i + searchArray.length).join('\n') === block.search) { - startIndex = i; - break; - } + const matchResult = flexibleMatch(result, block.search, block.replace); + if (matchResult) { + result = matchResult; + } else { + console.warn(handleMatchFailure(result, block.search)); } + } - if (startIndex !== -1) { - // Apply the patch at the found location - const changedSection = Diff3.patch( - result.slice(startIndex, startIndex + searchArray.length), - patch, - ); - if (Array.isArray(changedSection)) { - result.splice(startIndex, searchArray.length, ...changedSection); - } else { - console.warn(`Failed to apply patch for search block: ${block.search}`); - } + return result; +} + +function flexibleMatch( + content: string, + searchBlock: string, + replaceBlock: string, +): string | null { + // Try exact matching + const patch = Diff3.diffPatch(content.split('\n'), searchBlock.split('\n')); + let result = Diff3.patch(content.split('\n'), patch); + + if (result) { + return result.join('\n').replace(searchBlock, replaceBlock); + } + + // Try matching with flexible whitespace + const normalizedContent = content.replace(/\s+/g, ' '); + const normalizedSearch = searchBlock.replace(/\s+/g, ' '); + const flexPatch = Diff3.diffPatch( + normalizedContent.split('\n'), + normalizedSearch.split('\n'), + ); + result = Diff3.patch(normalizedContent.split('\n'), flexPatch); + + if (result) { + // Map the changes back to the original content + return mapChangesToOriginal( + content, + result.join('\n').replace(normalizedSearch, replaceBlock), + ); + } + + return null; +} + +function mapChangesToOriginal( + originalContent: string, + changedContent: string, +): string { + const originalLines = originalContent.split('\n'); + const changedLines = changedContent.split('\n'); + + let result = ''; + let originalIndex = 0; + let changedIndex = 0; + + while ( + originalIndex < originalLines.length && + changedIndex < changedLines.length + ) { + if ( + originalLines[originalIndex].trim() === changedLines[changedIndex].trim() + ) { + result += `${originalLines[originalIndex]}\n`; + originalIndex++; + changedIndex++; } else { - console.warn(`Search block not found: ${block.search}`); + result += `${changedLines[changedIndex]}\n`; + changedIndex++; + } + } + + // Add any remaining lines from either original or changed content + while (originalIndex < originalLines.length) { + result += `${originalLines[originalIndex]}\n`; + originalIndex++; + } + while (changedIndex < changedLines.length) { + result += `${changedLines[changedIndex]}\n`; + changedIndex++; + } + + return result.trim(); +} + +function handleMatchFailure(content: string, searchBlock: string): string { + const contentLines = content.split('\n'); + const searchLines = searchBlock.split('\n'); + const indices = Diff3.diffIndices(contentLines, searchLines); + + let errorMessage = 'Failed to match the following block:\n\n'; + errorMessage += `${searchBlock}\n\n`; + errorMessage += 'Possible similar sections in the file:\n\n'; + + for (const index of indices) { + if (Array.isArray(index) && index.length >= 2) { + const start = Math.max(0, index[0] - 2); + const end = Math.min(contentLines.length, index[1] + 3); + errorMessage += `${contentLines.slice(start, end).join('\n')}\n---\n`; } } - return result.join('\n'); + return errorMessage; } From e4d6a9abc47d3b35d5eacd752a30a7331c396b76 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:09:09 +0200 Subject: [PATCH 39/54] refactor --- src/{git => ai}/apply-changes.ts | 2 +- src/ai/apply-task.ts | 2 +- src/ai/task-workflow.ts | 2 +- src/{git => ai}/undo-task-changes.ts | 0 src/cli/index.ts | 2 +- tests/unit/apply-changes.test.ts | 2 +- tests/unit/apply-task.test.ts | 2 +- tests/unit/git-tools.test.ts | 2 +- tests/unit/task-workflow.test.ts | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename src/{git => ai}/apply-changes.ts (97%) rename src/{git => ai}/undo-task-changes.ts (100%) diff --git a/src/git/apply-changes.ts b/src/ai/apply-changes.ts similarity index 97% rename from src/git/apply-changes.ts rename to src/ai/apply-changes.ts index a5fc001..8f6bd17 100644 --- a/src/git/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -1,8 +1,8 @@ import path from 'node:path'; import chalk from 'chalk'; import fs from 'fs-extra'; -import { applySearchReplace } from '../ai/parsers/search-replace-parser'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; +import { applySearchReplace } from './parsers/search-replace-parser'; export async function applyChanges({ basePath, diff --git a/src/ai/apply-task.ts b/src/ai/apply-task.ts index b2abbce..2e537cb 100644 --- a/src/ai/apply-task.ts +++ b/src/ai/apply-task.ts @@ -2,8 +2,8 @@ import path from 'node:path'; import chalk from 'chalk'; import fs from 'fs-extra'; import simpleGit from 'simple-git'; -import { applyChanges } from '../git/apply-changes'; import { ensureBranch } from '../utils/git-tools'; +import { applyChanges } from './apply-changes'; export async function applyTask( filePath: string, diff --git a/src/ai/task-workflow.ts b/src/ai/task-workflow.ts index 2ca2af8..47ee2ee 100644 --- a/src/ai/task-workflow.ts +++ b/src/ai/task-workflow.ts @@ -5,7 +5,6 @@ import ora from 'ora'; import simpleGit from 'simple-git'; import { processFiles } from '../core/file-processor'; import { generateMarkdown } from '../core/markdown-generator'; -import { applyChanges } from '../git/apply-changes'; import { selectFilesPrompt } from '../interactive/select-files-prompt'; import { selectGitHubIssuePrompt } from '../interactive/select-github-issue-prompt'; import { selectModelPrompt } from '../interactive/select-model-prompt'; @@ -28,6 +27,7 @@ import { extractTemplateVariables, getTemplatePath, } from '../utils/template-utils'; +import { applyChanges } from './apply-changes'; import { generateAIResponse } from './generate-ai-response'; import { getInstructions } from './get-instructions'; import { getTaskDescription } from './get-task-description'; diff --git a/src/git/undo-task-changes.ts b/src/ai/undo-task-changes.ts similarity index 100% rename from src/git/undo-task-changes.ts rename to src/ai/undo-task-changes.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index 4431b9b..0c9609b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -9,9 +9,9 @@ import { applyTask } from '../ai/apply-task'; import { getModelConfig, getModelNames } from '../ai/model-config'; import { redoLastTask } from '../ai/redo-task'; import { runAIAssistedTask } from '../ai/task-workflow'; +import { undoTaskChanges } from '../ai/undo-task-changes'; import { processFiles } from '../core/file-processor'; import { generateMarkdown } from '../core/markdown-generator'; -import { undoTaskChanges } from '../git/undo-task-changes'; import { runInteractiveMode } from '../interactive/interactive-workflow'; import { DEFAULT_CACHE_PATH, clearCache } from '../utils/cache-utils'; import { handleEditorAndOutput } from '../utils/editor-utils'; diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 48f1c8c..621b0a4 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import fs from 'fs-extra'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { applyChanges } from '../../src/git/apply-changes'; +import { applyChanges } from '../../src/ai/apply-changes'; import type { AIParsedResponse } from '../../src/types'; vi.mock('fs-extra'); diff --git a/tests/unit/apply-task.test.ts b/tests/unit/apply-task.test.ts index b1c09c7..8f2db3c 100644 --- a/tests/unit/apply-task.test.ts +++ b/tests/unit/apply-task.test.ts @@ -2,8 +2,8 @@ import path from 'node:path'; import fs from 'fs-extra'; import simpleGit from 'simple-git'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as applyChanges from '../../src/ai/apply-changes'; import { applyTask } from '../../src/ai/apply-task'; -import * as applyChanges from '../../src/git/apply-changes'; import * as gitTools from '../../src/utils/git-tools'; vi.mock('../../src/utils/git-tools'); diff --git a/tests/unit/git-tools.test.ts b/tests/unit/git-tools.test.ts index 3221e63..e7efc22 100644 --- a/tests/unit/git-tools.test.ts +++ b/tests/unit/git-tools.test.ts @@ -1,7 +1,7 @@ import { confirm } from '@inquirer/prompts'; import simpleGit from 'simple-git'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { undoTaskChanges } from '../../src/git/undo-task-changes'; +import { undoTaskChanges } from '../../src/ai/undo-task-changes'; import { ensureBranch } from '../../src/utils/git-tools'; import { findDefaultBranch } from '../../src/utils/git-tools'; diff --git a/tests/unit/task-workflow.test.ts b/tests/unit/task-workflow.test.ts index 7136620..d32678d 100644 --- a/tests/unit/task-workflow.test.ts +++ b/tests/unit/task-workflow.test.ts @@ -1,6 +1,7 @@ import path from 'node:path'; import simpleGit, { type SimpleGit } from 'simple-git'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { applyChanges } from '../../src/ai/apply-changes'; import { generateAIResponse } from '../../src/ai/generate-ai-response'; import { getInstructions } from '../../src/ai/get-instructions'; import { getTaskDescription } from '../../src/ai/get-task-description'; @@ -10,7 +11,6 @@ import { reviewPlan } from '../../src/ai/plan-review'; import { runAIAssistedTask } from '../../src/ai/task-workflow'; import { processFiles } from '../../src/core/file-processor'; import { generateMarkdown } from '../../src/core/markdown-generator'; -import { applyChanges } from '../../src/git/apply-changes'; import { selectFilesPrompt } from '../../src/interactive/select-files-prompt'; import { selectModelPrompt } from '../../src/interactive/select-model-prompt'; import type { From 47550410bee0a4c01eab7445ca589a3c857c210e Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:24:48 +0200 Subject: [PATCH 40/54] test: adapt tests to new file structure and apply changes functionality --- tests/unit/apply-changes.test.ts | 40 ++++++++++++++++++++------------ tests/unit/apply-task.test.ts | 2 +- tests/unit/task-workflow.test.ts | 2 +- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 621b0a4..e726630 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -2,16 +2,11 @@ import path from 'node:path'; import fs from 'fs-extra'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { applyChanges } from '../../src/ai/apply-changes'; +import { applySearchReplace } from '../../src/ai/parsers/search-replace-parser'; import type { AIParsedResponse } from '../../src/types'; vi.mock('fs-extra'); -vi.mock('diff', async () => { - const actual = await vi.importActual('diff'); - return { - ...actual, - applyPatch: vi.fn(), - }; -}); +vi.mock('../../src/ai/parsers/search-replace-parser'); describe('applyChanges', () => { const mockBasePath = '/mock/base/path'; @@ -62,10 +57,14 @@ describe('applyChanges', () => { potentialIssues: 'None', }; - vi.mocked(fs.readFile).mockResolvedValue( - // biome-ignore lint/suspicious/noExplicitAny: explicit any is required for the mock - 'console.log("Old content");\nfunction oldFunction() {}' as any, - ); + const originalContent = + 'console.log("Old content");\nfunction oldFunction() {}'; + const modifiedContent = + 'console.log("Modified content");\nfunction newFunction() {}'; + + // biome-ignore lint/suspicious/noExplicitAny: explicit any is required for the mock + vi.mocked(fs.readFile).mockResolvedValue(originalContent as any); + vi.mocked(applySearchReplace).mockResolvedValue(modifiedContent); await applyChanges({ basePath: mockBasePath, @@ -77,12 +76,23 @@ describe('applyChanges', () => { expect(fs.writeFile).toHaveBeenCalledTimes(2); expect(fs.readFile).toHaveBeenCalledTimes(1); expect(fs.remove).toHaveBeenCalledTimes(1); + expect(applySearchReplace).toHaveBeenCalledTimes(1); + + // Check if the writeFile calls include both the new and modified files + expect(vi.mocked(fs.writeFile).mock.calls).toEqual( + expect.arrayContaining([ + [`${mockBasePath}/new-file.js`, 'console.log("New file");'], + [`${mockBasePath}/existing-file.js`, expect.any(Promise)], + ]), + ); - // Check if the content of the modified file is correct - expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining('existing-file.js'), - 'console.log("Modified content");\nfunction newFunction() {}', + // Resolve the Promise for the modified file content + await expect(vi.mocked(fs.writeFile).mock.calls[1][1]).resolves.toBe( + modifiedContent, ); + + // Check if the deleted file was removed + expect(fs.remove).toHaveBeenCalledWith(`${mockBasePath}/deleted-file.js`); }); it('should not apply changes in dry run mode', async () => { diff --git a/tests/unit/apply-task.test.ts b/tests/unit/apply-task.test.ts index 8f2db3c..a6d669c 100644 --- a/tests/unit/apply-task.test.ts +++ b/tests/unit/apply-task.test.ts @@ -7,7 +7,7 @@ import { applyTask } from '../../src/ai/apply-task'; import * as gitTools from '../../src/utils/git-tools'; vi.mock('../../src/utils/git-tools'); -vi.mock('../../src/git/apply-changes'); +vi.mock('../../src/ai/apply-changes'); vi.mock('fs-extra'); vi.mock('simple-git'); diff --git a/tests/unit/task-workflow.test.ts b/tests/unit/task-workflow.test.ts index d32678d..6efbe40 100644 --- a/tests/unit/task-workflow.test.ts +++ b/tests/unit/task-workflow.test.ts @@ -29,7 +29,7 @@ vi.mock('../../src/core/markdown-generator'); vi.mock('../../src/ai/generate-ai-response'); vi.mock('../../src/ai/parse-ai-codegen-response'); vi.mock('../../src/ai/plan-review'); -vi.mock('../../src/git/apply-changes'); +vi.mock('../../src/ai/apply-changes'); vi.mock('../../src/utils/git-tools'); vi.mock('../../src/ai/model-config'); vi.mock('simple-git'); From 2f790f85c6eab6a98226cda4f19e8f258e98852b Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:29:03 +0200 Subject: [PATCH 41/54] test: fix cross platform path resolution --- tests/unit/apply-changes.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index e726630..4b3ac64 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -81,8 +81,8 @@ describe('applyChanges', () => { // Check if the writeFile calls include both the new and modified files expect(vi.mocked(fs.writeFile).mock.calls).toEqual( expect.arrayContaining([ - [`${mockBasePath}/new-file.js`, 'console.log("New file");'], - [`${mockBasePath}/existing-file.js`, expect.any(Promise)], + [path.join(mockBasePath, 'new-file.js'), 'console.log("New file");'], + [path.join(mockBasePath, 'existing-file.js'), expect.any(Promise)], ]), ); From 58729ed35d8e90d6304d13ab24cc7187315ab016 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:32:17 +0200 Subject: [PATCH 42/54] missed one --- tests/unit/apply-changes.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 4b3ac64..b7b99b2 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -92,7 +92,9 @@ describe('applyChanges', () => { ); // Check if the deleted file was removed - expect(fs.remove).toHaveBeenCalledWith(`${mockBasePath}/deleted-file.js`); + expect(fs.remove).toHaveBeenCalledWith( + path.join(mockBasePath, 'deleted-file.js'), + ); }); it('should not apply changes in dry run mode', async () => { From 1e606123123682426a7c5fdfe8ac4fdc817b6ca3 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 21:54:18 +0200 Subject: [PATCH 43/54] improve change application --- package.json | 2 ++ pnpm-lock.yaml | 17 +++++++++++++++++ src/ai/apply-changes.ts | 21 +++++++++++---------- src/ai/parsers/search-replace-parser.ts | 10 ++-------- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index e6b7deb..fec0a17 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "@inquirer/prompts": "5.3.7", "@octokit/rest": "21.0.1", "@types/diff": "5.2.1", + "@types/uuid": "10.0.0", "ai": "3.3.0", "chalk": "5.3.0", "commander": "12.1.0", @@ -118,6 +119,7 @@ "piscina": "4.6.1", "simple-git": "3.25.0", "strip-comments": "2.0.1", + "uuid": "10.0.0", "winston": "3.13.1" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f40ab6..0e22f2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@types/diff': specifier: 5.2.1 version: 5.2.1 + '@types/uuid': + specifier: ^10.0.0 + version: 10.0.0 ai: specifier: 3.3.0 version: 3.3.0(react@18.3.1)(sswr@2.1.0(svelte@4.2.18))(svelte@4.2.18)(vue@3.4.34(typescript@5.5.4))(zod@3.23.8) @@ -86,6 +89,9 @@ importers: strip-comments: specifier: 2.0.1 version: 2.0.1 + uuid: + specifier: ^10.0.0 + version: 10.0.0 winston: specifier: 3.13.1 version: 3.13.1 @@ -986,6 +992,9 @@ packages: '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + '@types/wrap-ansi@3.0.0': resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} @@ -2948,6 +2957,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + hasBin: true + validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} @@ -3866,6 +3879,8 @@ snapshots: '@types/triple-beam@1.3.5': {} + '@types/uuid@10.0.0': {} + '@types/wrap-ansi@3.0.0': {} '@vitest/coverage-v8@2.0.5(vitest@2.0.5(@types/node@20.14.14)(@vitest/ui@2.0.5))': @@ -5816,6 +5831,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@10.0.0: {} + validate-npm-package-license@3.0.4: dependencies: spdx-correct: 3.2.0 diff --git a/src/ai/apply-changes.ts b/src/ai/apply-changes.ts index 8f6bd17..d9f78fc 100644 --- a/src/ai/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -1,6 +1,8 @@ +import { tmpdir } from 'node:os'; import path from 'node:path'; import chalk from 'chalk'; import fs from 'fs-extra'; +import { v4 as uuidv4 } from 'uuid'; import type { AIFileInfo, ApplyChangesOptions } from '../types'; import { applySearchReplace } from './parsers/search-replace-parser'; @@ -30,6 +32,8 @@ async function applyFileChange( dryRun: boolean, ): Promise { const fullPath = path.join(basePath, file.path); + const tempPath = path.join(tmpdir(), `codewhisper-${uuidv4()}`); + try { switch (file.status) { case 'new': @@ -44,7 +48,8 @@ async function applyFileChange( ); } else { await fs.ensureDir(path.dirname(fullPath)); - await fs.writeFile(fullPath, file.content || ''); + await fs.writeFile(tempPath, file.content || ''); + await fs.move(tempPath, fullPath, { overwrite: true }); console.log(chalk.green(`Created file: ${file.path}`)); } break; @@ -59,18 +64,12 @@ async function applyFileChange( `[DRY RUN] Changes preview:\n${JSON.stringify(file.changes, null, 2)}`, ), ); - } else if (file.content) { - console.log( - chalk.gray( - `[DRY RUN] Content preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, - ), - ); } } else { - const currentContent = await fs.readFile(fullPath, 'utf-8'); let updatedContent: string; - if (file.changes) { + if (file.changes && file.changes.length > 0) { + const currentContent = await fs.readFile(fullPath, 'utf-8'); updatedContent = applySearchReplace(currentContent, file.changes); } else if (file.content) { updatedContent = file.content; @@ -80,7 +79,8 @@ async function applyFileChange( ); } - await fs.writeFile(fullPath, updatedContent); + await fs.writeFile(tempPath, updatedContent); + await fs.move(tempPath, fullPath, { overwrite: true }); console.log(chalk.green(`Modified file: ${file.path}`)); } break; @@ -97,6 +97,7 @@ async function applyFileChange( } } catch (error) { console.error(chalk.red(`Error applying change to ${file.path}:`), error); + await fs.remove(tempPath).catch(() => {}); throw error; } } diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 2eaa07f..7c5e8ac 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -42,17 +42,11 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { language: detectLanguage(path), status, explanation, + content: content, }; if (status === 'modified') { - const changes = parseSearchReplaceBlocks(content); - if (changes.length > 0) { - fileInfo.changes = changes; - } else { - fileInfo.content = content; - } - } else if (status === 'new') { - fileInfo.content = content; + fileInfo.changes = parseSearchReplaceBlocks(content); } files.push(fileInfo); From 98fe7df9fb1192539a82076507d70aa27496e4e1 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 22:13:20 +0200 Subject: [PATCH 44/54] fix code modification application --- src/ai/apply-changes.ts | 8 +++++++- src/ai/parsers/search-replace-parser.ts | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ai/apply-changes.ts b/src/ai/apply-changes.ts index d9f78fc..0ea714d 100644 --- a/src/ai/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -64,12 +64,18 @@ async function applyFileChange( `[DRY RUN] Changes preview:\n${JSON.stringify(file.changes, null, 2)}`, ), ); + } else if (file.content) { + console.log( + chalk.gray( + `[DRY RUN] Content preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, + ), + ); } } else { + const currentContent = await fs.readFile(fullPath, 'utf-8'); let updatedContent: string; if (file.changes && file.changes.length > 0) { - const currentContent = await fs.readFile(fullPath, 'utf-8'); updatedContent = applySearchReplace(currentContent, file.changes); } else if (file.content) { updatedContent = file.content; diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 7c5e8ac..2eaa07f 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -42,11 +42,17 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { language: detectLanguage(path), status, explanation, - content: content, }; if (status === 'modified') { - fileInfo.changes = parseSearchReplaceBlocks(content); + const changes = parseSearchReplaceBlocks(content); + if (changes.length > 0) { + fileInfo.changes = changes; + } else { + fileInfo.content = content; + } + } else if (status === 'new') { + fileInfo.content = content; } files.push(fileInfo); From d01b1112866b2b413b5ae1e32fee3fb59248b143 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 22:31:14 +0200 Subject: [PATCH 45/54] fix code modification application --- src/ai/apply-changes.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/ai/apply-changes.ts b/src/ai/apply-changes.ts index 0ea714d..a4a4dfa 100644 --- a/src/ai/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -67,22 +67,34 @@ async function applyFileChange( } else if (file.content) { console.log( chalk.gray( - `[DRY RUN] Content preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, + `[DRY RUN] Full content replacement preview:\n${file.content.substring(0, 200)}${file.content.length > 200 ? '...' : ''}`, ), ); } } else { - const currentContent = await fs.readFile(fullPath, 'utf-8'); + let currentContent: string; + try { + currentContent = await fs.readFile(fullPath, 'utf-8'); + } catch (error) { + console.error(chalk.red(`Error reading file ${file.path}:`, error)); + throw error; + } + let updatedContent: string; if (file.changes && file.changes.length > 0) { + // Diff mode edits updatedContent = applySearchReplace(currentContent, file.changes); + if (updatedContent === currentContent) { + console.log(chalk.yellow(`No changes applied to: ${file.path}`)); + return; + } } else if (file.content) { + // Whole file edits updatedContent = file.content; } else { - throw new Error( - `No content or changes provided for modified file: ${file.path}`, - ); + console.log(chalk.yellow(`No changes to apply for: ${file.path}`)); + return; } await fs.writeFile(tempPath, updatedContent); From 65b1772a501a924860d72206e986ba611e3d0e17 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Wed, 7 Aug 2024 22:44:25 +0200 Subject: [PATCH 46/54] improving code application --- src/ai/apply-changes.ts | 28 ++++++++++++++-- src/ai/parsers/search-replace-parser.ts | 43 +++++++++++++++++++++---- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/ai/apply-changes.ts b/src/ai/apply-changes.ts index a4a4dfa..9ed86c6 100644 --- a/src/ai/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -34,6 +34,9 @@ async function applyFileChange( const fullPath = path.join(basePath, file.path); const tempPath = path.join(tmpdir(), `codewhisper-${uuidv4()}`); + console.log(`Processing file: ${file.path}`); + console.log(`File status: ${file.status}`); + try { switch (file.status) { case 'new': @@ -75,31 +78,50 @@ async function applyFileChange( let currentContent: string; try { currentContent = await fs.readFile(fullPath, 'utf-8'); + console.log(`Current content length: ${currentContent.length}`); } catch (error) { console.error(chalk.red(`Error reading file ${file.path}:`, error)); throw error; } + // Write current content to temp file first + await fs.writeFile(tempPath, currentContent); + console.log(`Wrote original content to temp file: ${tempPath}`); + let updatedContent: string; if (file.changes && file.changes.length > 0) { - // Diff mode edits + console.log(`Applying ${file.changes.length} changes in diff mode`); updatedContent = applySearchReplace(currentContent, file.changes); if (updatedContent === currentContent) { console.log(chalk.yellow(`No changes applied to: ${file.path}`)); + await fs.remove(tempPath); return; } + console.log( + `Updated content length after changes: ${updatedContent.length}`, + ); + // Write updated content to temp file + await fs.writeFile(tempPath, updatedContent); } else if (file.content) { - // Whole file edits + console.log('Applying whole file edit'); updatedContent = file.content; + console.log(`New content length: ${updatedContent.length}`); + // Write new content to temp file + await fs.writeFile(tempPath, updatedContent); } else { console.log(chalk.yellow(`No changes to apply for: ${file.path}`)); + await fs.remove(tempPath); return; } - await fs.writeFile(tempPath, updatedContent); + // Move temp file to overwrite original file await fs.move(tempPath, fullPath, { overwrite: true }); console.log(chalk.green(`Modified file: ${file.path}`)); + + // Verify final content + const finalContent = await fs.readFile(fullPath, 'utf-8'); + console.log(`Final content length: ${finalContent.length}`); } break; case 'deleted': diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 2eaa07f..f03c811 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -92,16 +92,26 @@ export function applySearchReplace( searchReplaceBlocks: AIFileChange[], ): string { let result = source; + console.log('Original content length:', source.length); for (const block of searchReplaceBlocks) { + console.log('Applying search/replace block:', block); const matchResult = flexibleMatch(result, block.search, block.replace); if (matchResult) { result = matchResult; + console.log( + 'Block applied successfully. New content length:', + result.length, + ); } else { - console.warn(handleMatchFailure(result, block.search)); + console.warn( + 'Failed to apply block:', + handleMatchFailure(result, block.search), + ); } } + console.log('Final content length after all changes:', result.length); return result; } @@ -110,14 +120,24 @@ function flexibleMatch( searchBlock: string, replaceBlock: string, ): string | null { + console.log('Attempting flexible match'); + console.log('Content length:', content.length); + console.log('Search block length:', searchBlock.length); + console.log('Replace block length:', replaceBlock.length); + // Try exact matching const patch = Diff3.diffPatch(content.split('\n'), searchBlock.split('\n')); let result = Diff3.patch(content.split('\n'), patch); if (result) { - return result.join('\n').replace(searchBlock, replaceBlock); + console.log('Exact match successful'); + const finalResult = result.join('\n').replace(searchBlock, replaceBlock); + console.log('Result after exact match. Length:', finalResult.length); + return finalResult; } + console.log('Exact match failed, trying flexible whitespace match'); + // Try matching with flexible whitespace const normalizedContent = content.replace(/\s+/g, ' '); const normalizedSearch = searchBlock.replace(/\s+/g, ' '); @@ -128,13 +148,19 @@ function flexibleMatch( result = Diff3.patch(normalizedContent.split('\n'), flexPatch); if (result) { - // Map the changes back to the original content - return mapChangesToOriginal( - content, - result.join('\n').replace(normalizedSearch, replaceBlock), + console.log('Flexible whitespace match successful'); + const flexResult = result + .join('\n') + .replace(normalizedSearch, replaceBlock); + const mappedResult = mapChangesToOriginal(content, flexResult); + console.log( + 'Result after flexible match and mapping. Length:', + mappedResult.length, ); + return mappedResult; } + console.log('All matching attempts failed'); return null; } @@ -142,6 +168,10 @@ function mapChangesToOriginal( originalContent: string, changedContent: string, ): string { + console.log('Mapping changes to original content'); + console.log('Original content length:', originalContent.length); + console.log('Changed content length:', changedContent.length); + const originalLines = originalContent.split('\n'); const changedLines = changedContent.split('\n'); @@ -175,6 +205,7 @@ function mapChangesToOriginal( changedIndex++; } + console.log('Mapping complete. Result length:', result.length); return result.trim(); } From 43cc6e117c9550b6ab5982c4f58b618d9d4058ab Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 11:51:19 +0200 Subject: [PATCH 47/54] add: integration for the application of changes --- .../fixtures/applyChanges/parsedResponse.json | 70 ++ .../applyChanges/src/ai/task-workflow.ts | 614 ++++++++++++++++++ tests/fixtures/applyChanges/src/delete-me.ts | 1 + .../fixtures/applyChanges/src/types/index.ts | 195 ++++++ .../fixtures/applyChanges/src/utils/logger.ts | 31 + tests/integration/apply-changes.test.ts | 121 ++++ 6 files changed, 1032 insertions(+) create mode 100644 tests/fixtures/applyChanges/parsedResponse.json create mode 100644 tests/fixtures/applyChanges/src/ai/task-workflow.ts create mode 100644 tests/fixtures/applyChanges/src/delete-me.ts create mode 100644 tests/fixtures/applyChanges/src/types/index.ts create mode 100644 tests/fixtures/applyChanges/src/utils/logger.ts create mode 100644 tests/integration/apply-changes.test.ts diff --git a/tests/fixtures/applyChanges/parsedResponse.json b/tests/fixtures/applyChanges/parsedResponse.json new file mode 100644 index 0000000..9eeb12c --- /dev/null +++ b/tests/fixtures/applyChanges/parsedResponse.json @@ -0,0 +1,70 @@ +{ + "parsedResponse": { + "fileList": [ + "src/utils/logger.ts", + "src/types/index.ts", + "src/ai/task-workflow.ts", + "src/new-file.ts", + "src/delete-me.ts" + ], + "files": [ + { + "path": "src/utils/logger.ts", + "language": "typescript", + "status": "modified", + "explanation": "Updated the logger utility to support multiple log levels and added a LoggerOptions interface to configure the logger with the desired log level and AI interaction logging option.", + "changes": [ + { + "search": "import path from 'node:path';\nimport winston from 'winston';\n\nconst getLogger = (logAiInteractions: boolean) => {\n if (!logAiInteractions) {\n return {\n info: () => {},\n error: () => {},\n };\n }\n\n const date = new Date().toISOString().split('T')[0];\n const logFileName = `codewhisper-${date}.log`;\n\n return winston.createLogger({\n level: 'info',\n format: winston.format.combine(\n winston.format.timestamp(),\n winston.format.printf(({ timestamp, level, message, ...rest }) => {\n return `${timestamp} ${level}: ${message}\\n${JSON.stringify(rest, null, 2)}\\n`;\n }),\n ),\n transports: [\n new winston.transports.File({\n filename: path.join(process.cwd(), logFileName),\n }),\n ],\n });\n};\n\nexport default getLogger;", + "replace": "import path from 'node:path';\nimport winston from 'winston';\n\nexport interface LoggerOptions {\n logLevel: 'error' | 'warn' | 'info' | 'debug';\n logAiInteractions: boolean;\n}\n\nconst getLogger = (options: LoggerOptions) => {\n if (!options.logAiInteractions) {\n return {\n error: () => {},\n warn: () => {},\n info: () => {},\n debug: () => {},\n };\n }\n\n const date = new Date().toISOString().split('T')[0];\n const logFileName = `codewhisper-${date}.log`;\n\n return winston.createLogger({\n level: options.logLevel,\n format: winston.format.combine(\n winston.format.timestamp(),\n winston.format.printf(({ timestamp, level, message, ...rest }) => {\n return `${timestamp} ${level}: ${message}\\n${JSON.stringify(rest, null, 2)}\\n`;\n }),\n ),\n transports: [\n new winston.transports.File({\n filename: path.join(process.cwd(), logFileName),\n }),\n ],\n });\n};\n\nexport default getLogger;" + } + ] + }, + { + "path": "src/types/index.ts", + "language": "typescript", + "status": "modified", + "explanation": "Added a new logLevel property to the AiAssistedTaskOptions interface to allow users to specify the desired log level.", + "changes": [ + { + "search": "export type AiAssistedTaskOptions = Pick<\n InteractiveModeOptions,\n | 'path'\n | 'filter'\n | 'exclude'\n | 'suppressComments'\n | 'lineNumbers'\n | 'caseSensitive'\n | 'customIgnores'\n | 'cachePath'\n | 'respectGitignore'\n | 'invert'\n | 'gitignore'\n | 'noCodeblock'\n> & {\n dryRun: boolean;\n maxCostThreshold?: number;\n task?: string;\n description?: string;\n instructions?: string;\n autoCommit?: boolean;\n model: string;\n contextWindow?: number;\n maxTokens?: number;\n logAiInteractions?: boolean;\n githubIssue?: GitHubIssue;\n githubIssueFilters?: string;\n issueNumber?: number;\n diff?: boolean;\n plan?: boolean;\n context?: string[];\n};", + "replace": "export type AiAssistedTaskOptions = Pick<\n InteractiveModeOptions,\n | 'path'\n | 'filter'\n | 'exclude'\n | 'suppressComments'\n | 'lineNumbers'\n | 'caseSensitive'\n | 'customIgnores'\n | 'cachePath'\n | 'respectGitignore'\n | 'invert'\n | 'gitignore'\n | 'noCodeblock'\n> & {\n dryRun: boolean;\n maxCostThreshold?: number;\n task?: string;\n description?: string;\n instructions?: string;\n autoCommit?: boolean;\n model: string;\n contextWindow?: number;\n maxTokens?: number;\n logAiInteractions?: boolean;\n githubIssue?: GitHubIssue;\n githubIssueFilters?: string;\n issueNumber?: number;\n diff?: boolean;\n plan?: boolean;\n context?: string[];\n logLevel?: 'error' | 'warn' | 'info' | 'debug';\n};" + } + ] + }, + { + "path": "src/ai/task-workflow.ts", + "language": "typescript", + "status": "modified", + "explanation": "Updated the runAIAssistedTask function to use the new logging system. Replaced console.log statements with appropriate logger calls. Added logger initialization and usage in the selectModel function.", + "changes": [ + { + "search": "import path from 'node:path';\nimport chalk from 'chalk';\nimport fs from 'fs-extra';\nimport ora from 'ora';\nimport simpleGit from 'simple-git';\nimport { processFiles } from '../core/file-processor';\nimport { generateMarkdown } from '../core/markdown-generator';\nimport { selectFilesPrompt } from '../interactive/select-files-prompt';\nimport { selectGitHubIssuePrompt } from '../interactive/select-github-issue-prompt';\nimport { selectModelPrompt } from '../interactive/select-model-prompt';\nimport type {\n AIParsedResponse,\n AiAssistedTaskOptions,\n FileInfo,\n MarkdownOptions,\n TaskData,\n} from '../types';\nimport {\n DEFAULT_CACHE_PATH,\n getCachedValue,\n setCachedValue,\n} from '../utils/cache-utils';\nimport { ensureBranch } from '../utils/git-tools';\nimport { TaskCache } from '../utils/task-cache';\nimport {\n collectVariables,\n extractTemplateVariables,\n getTemplatePath,\n} from '../utils/template-utils';\nimport { applyChanges } from './apply-changes';\nimport { generateAIResponse } from './generate-ai-response';\nimport { getInstructions } from './get-instructions';\nimport { getTaskDescription } from './get-task-description';\nimport { getModelConfig } from './model-config';\nimport { parseAICodegenResponse } from './parse-ai-codegen-response';\nimport { reviewPlan } from './plan-review';\n\nexport async function runAIAssistedTask(options: AiAssistedTaskOptions) {\n const spinner = ora();\n try {\n const basePath = path.resolve(options.path ?? '.');\n const filters = options.githubIssueFilters ?? '';\n const taskCache = new TaskCache(basePath);\n\n const modelKey = await selectModel(options);\n const modelConfig = getModelConfig(modelKey);\n\n if (options.diff === undefined) {\n options.diff = modelConfig.mode === 'diff';\n }\n console.log(\n chalk.blue(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`),\n );\n\n const { taskDescription, instructions } = await getTaskInfo(\n options,\n basePath,\n filters,\n );\n const selectedFiles = await selectFiles(options, basePath);\n\n spinner.start('Processing files...');\n const processedFiles = await processFiles(\n getProcessOptions(options, basePath, selectedFiles),\n );\n spinner.succeed('Files processed successfully');\n\n const taskData = {\n selectedFiles,\n taskDescription,\n instructions,\n model: modelKey,\n } as TaskData;\n\n if (options.plan) {\n await handlePlanWorkflow(\n options,\n basePath,\n taskCache,\n taskData,\n processedFiles,\n modelKey,\n );\n } else {\n await handleNoPlanWorkflow(\n options,\n basePath,\n taskCache,\n taskData,\n processedFiles,\n modelKey,\n );\n }\n\n spinner.succeed('AI-assisted task completed! 🎉');\n } catch (error) {\n spinner.fail('Error in AI-assisted task');\n console.error(\n chalk.red(error instanceof Error ? error.message : String(error)),\n );\n process.exit(1);\n }\n}", + "replace": "import path from 'node:path';\nimport chalk from 'chalk';\nimport fs from 'fs-extra';\nimport ora from 'ora';\nimport simpleGit from 'simple-git';\nimport { processFiles } from '../core/file-processor';\nimport { generateMarkdown } from '../core/markdown-generator';\nimport { selectFilesPrompt } from '../interactive/select-files-prompt';\nimport { selectGitHubIssuePrompt } from '../interactive/select-github-issue-prompt';\nimport { selectModelPrompt } from '../interactive/select-model-prompt';\nimport type {\n AIParsedResponse,\n AiAssistedTaskOptions,\n FileInfo,\n MarkdownOptions,\n TaskData,\n} from '../types';\nimport {\n DEFAULT_CACHE_PATH,\n getCachedValue,\n setCachedValue,\n} from '../utils/cache-utils';\nimport { ensureBranch } from '../utils/git-tools';\nimport { TaskCache } from '../utils/task-cache';\nimport {\n collectVariables,\n extractTemplateVariables,\n getTemplatePath,\n} from '../utils/template-utils';\nimport { applyChanges } from './apply-changes';\nimport { generateAIResponse } from './generate-ai-response';\nimport { getInstructions } from './get-instructions';\nimport { getTaskDescription } from './get-task-description';\nimport { getModelConfig } from './model-config';\nimport { parseAICodegenResponse } from './parse-ai-codegen-response';\nimport { reviewPlan } from './plan-review';\nimport getLogger, { LoggerOptions } from '../utils/logger';\n\nexport async function runAIAssistedTask(options: AiAssistedTaskOptions) {\n const loggerOptions: LoggerOptions = {\n logLevel: options.logLevel || 'info',\n logAiInteractions: options.logAiInteractions || false,\n };\n const logger = getLogger(loggerOptions);\n const spinner = ora();\n\n try {\n const basePath = path.resolve(options.path ?? '.');\n const filters = options.githubIssueFilters ?? '';\n const taskCache = new TaskCache(basePath);\n\n const modelKey = await selectModel(options);\n const modelConfig = getModelConfig(modelKey);\n\n if (options.diff === undefined) {\n options.diff = modelConfig.mode === 'diff';\n }\n logger.info(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`);\n\n const { taskDescription, instructions } = await getTaskInfo(\n options,\n basePath,\n filters,\n );\n const selectedFiles = await selectFiles(options, basePath);\n\n spinner.start('Processing files...');\n const processedFiles = await processFiles(\n getProcessOptions(options, basePath, selectedFiles),\n );\n spinner.succeed('Files processed successfully');\n\n const taskData = {\n selectedFiles,\n taskDescription,\n instructions,\n model: modelKey,\n } as TaskData;\n\n if (options.plan) {\n await handlePlanWorkflow(\n options,\n basePath,\n taskCache,\n taskData,\n processedFiles,\n modelKey,\n logger,\n );\n } else {\n await handleNoPlanWorkflow(\n options,\n basePath,\n taskCache,\n taskData,\n processedFiles,\n modelKey,\n logger,\n );\n }\n\n spinner.succeed('AI-assisted task completed! 🎉');\n } catch (error) {\n spinner.fail('Error in AI-assisted task');\n logger.error('Error in AI-assisted task', { error: error instanceof Error ? error.message : String(error) });\n process.exit(1);\n }\n}" + }, + { + "search": "async function selectModel(options: AiAssistedTaskOptions): Promise {\n let modelKey = options.model;\n\n if (modelKey) {\n const modelConfig = getModelConfig(modelKey);\n if (!modelConfig) {\n console.log(chalk.red(`Invalid model ID: ${modelKey}.`));\n console.log(chalk.yellow('Displaying model selection list...'));\n modelKey = '';\n } else {\n console.log(chalk.blue(`Using model: ${modelConfig.modelName}`));\n }\n }\n\n if (!modelKey) {\n try {\n modelKey = await selectModelPrompt();\n const modelConfig = getModelConfig(modelKey);\n console.log(chalk.blue(`Using model: ${modelConfig.modelName}`));\n } catch (error) {\n console.error(chalk.red('Error selecting model:'), error);\n process.exit(1);\n }\n }\n\n return modelKey;\n}", + "replace": "async function selectModel(options: AiAssistedTaskOptions): Promise {\n let modelKey = options.model;\n const logger = getLogger({ logLevel: options.logLevel || 'info', logAiInteractions: options.logAiInteractions || false });\n\n if (modelKey) {\n const modelConfig = getModelConfig(modelKey);\n if (!modelConfig) {\n logger.warn(`Invalid model ID: ${modelKey}. Displaying model selection list...`);\n modelKey = '';\n } else {\n logger.info(`Using model: ${modelConfig.modelName}`);\n }\n }\n\n if (!modelKey) {\n try {\n modelKey = await selectModelPrompt();\n const modelConfig = getModelConfig(modelKey);\n logger.info(`Using model: ${modelConfig.modelName}`);\n } catch (error) {\n logger.error('Error selecting model:', { error });\n process.exit(1);\n }\n }\n\n return modelKey;\n}" + } + ] + }, + { + "path": "src/new-file.ts", + "language": "typescript", + "status": "new", + "explanation": "Created a new file to demonstrate file creation functionality.", + "content": "// This is a new file created by CodeWhisper\n\nexport function newFunction() {\n console.log('This is a new function in a new file');\n}\n" + }, + { + "path": "src/delete-me.ts", + "language": "typescript", + "status": "deleted", + "explanation": "This file will be deleted to demonstrate file deletion functionality." + } + ], + "gitBranchName": "feature/expand-logging-options", + "gitCommitMessage": "Expand CodeWhisper's logging options with multiple log levels", + "summary": "Updated the logger utility to support multiple log levels, modified the AiAssistedTaskOptions interface to include a logLevel option, and updated the runAIAssistedTask function to use the new logging system. Replaced console.log statements with appropriate logger calls in the task-workflow file.", + "potentialIssues": "1. The changes may slightly increase memory usage due to more verbose logging.\n2. Users might need to update their scripts or commands to include the new log level option.\n3. Existing error handling might need to be reviewed to ensure it works well with the new logging system.\n4. Performance impact of increased logging, especially at debug level, should be monitored." + } +} diff --git a/tests/fixtures/applyChanges/src/ai/task-workflow.ts b/tests/fixtures/applyChanges/src/ai/task-workflow.ts new file mode 100644 index 0000000..47ee2ee --- /dev/null +++ b/tests/fixtures/applyChanges/src/ai/task-workflow.ts @@ -0,0 +1,614 @@ +import path from 'node:path'; +import chalk from 'chalk'; +import fs from 'fs-extra'; +import ora from 'ora'; +import simpleGit from 'simple-git'; +import { processFiles } from '../core/file-processor'; +import { generateMarkdown } from '../core/markdown-generator'; +import { selectFilesPrompt } from '../interactive/select-files-prompt'; +import { selectGitHubIssuePrompt } from '../interactive/select-github-issue-prompt'; +import { selectModelPrompt } from '../interactive/select-model-prompt'; +import type { + AIParsedResponse, + AiAssistedTaskOptions, + FileInfo, + MarkdownOptions, + TaskData, +} from '../types'; +import { + DEFAULT_CACHE_PATH, + getCachedValue, + setCachedValue, +} from '../utils/cache-utils'; +import { ensureBranch } from '../utils/git-tools'; +import { TaskCache } from '../utils/task-cache'; +import { + collectVariables, + extractTemplateVariables, + getTemplatePath, +} from '../utils/template-utils'; +import { applyChanges } from './apply-changes'; +import { generateAIResponse } from './generate-ai-response'; +import { getInstructions } from './get-instructions'; +import { getTaskDescription } from './get-task-description'; +import { getModelConfig } from './model-config'; +import { parseAICodegenResponse } from './parse-ai-codegen-response'; +import { reviewPlan } from './plan-review'; + +export async function runAIAssistedTask(options: AiAssistedTaskOptions) { + const spinner = ora(); + try { + const basePath = path.resolve(options.path ?? '.'); + const filters = options.githubIssueFilters ?? ''; + const taskCache = new TaskCache(basePath); + + const modelKey = await selectModel(options); + const modelConfig = getModelConfig(modelKey); + + if (options.diff === undefined) { + options.diff = modelConfig.mode === 'diff'; + } + console.log( + chalk.blue(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`), + ); + + const { taskDescription, instructions } = await getTaskInfo( + options, + basePath, + filters, + ); + const selectedFiles = await selectFiles(options, basePath); + + spinner.start('Processing files...'); + const processedFiles = await processFiles( + getProcessOptions(options, basePath, selectedFiles), + ); + spinner.succeed('Files processed successfully'); + + const taskData = { + selectedFiles, + taskDescription, + instructions, + model: modelKey, + } as TaskData; + + if (options.plan) { + await handlePlanWorkflow( + options, + basePath, + taskCache, + taskData, + processedFiles, + modelKey, + ); + } else { + await handleNoPlanWorkflow( + options, + basePath, + taskCache, + taskData, + processedFiles, + modelKey, + ); + } + + spinner.succeed('AI-assisted task completed! 🎉'); + } catch (error) { + spinner.fail('Error in AI-assisted task'); + console.error( + chalk.red(error instanceof Error ? error.message : String(error)), + ); + process.exit(1); + } +} + +async function selectModel(options: AiAssistedTaskOptions): Promise { + let modelKey = options.model; + + if (modelKey) { + const modelConfig = getModelConfig(modelKey); + if (!modelConfig) { + console.log(chalk.red(`Invalid model ID: ${modelKey}.`)); + console.log(chalk.yellow('Displaying model selection list...')); + modelKey = ''; + } else { + console.log(chalk.blue(`Using model: ${modelConfig.modelName}`)); + } + } + + if (!modelKey) { + try { + modelKey = await selectModelPrompt(); + const modelConfig = getModelConfig(modelKey); + console.log(chalk.blue(`Using model: ${modelConfig.modelName}`)); + } catch (error) { + console.error(chalk.red('Error selecting model:'), error); + process.exit(1); + } + } + + return modelKey; +} + +async function getTaskInfo( + options: AiAssistedTaskOptions, + basePath: string, + filters: string, +): Promise<{ taskDescription: string; instructions: string }> { + let taskDescription = ''; + let instructions = ''; + + if (options.githubIssue) { + if (!process.env.GITHUB_TOKEN) { + console.log( + chalk.yellow( + 'GITHUB_TOKEN not set. GitHub issue functionality may be limited.', + ), + ); + } + const selectedIssue = await selectGitHubIssuePrompt(basePath, filters); + if (selectedIssue) { + taskDescription = `# ${selectedIssue.title}\n\n${selectedIssue.body}`; + options.issueNumber = selectedIssue.number; + } else { + console.log( + chalk.yellow('No GitHub issue selected. Falling back to manual input.'), + ); + } + } + + if (!taskDescription) { + if (options.task || options.description) { + taskDescription = + `# ${options.task || 'Task'}\n\n${options.description || ''}`.trim(); + } else { + const cachedTaskDescription = await getCachedValue( + 'taskDescription', + options.cachePath, + ); + taskDescription = await getTaskDescription( + cachedTaskDescription as string | undefined, + ); + await setCachedValue( + 'taskDescription', + taskDescription, + options.cachePath, + ); + } + } + + if (!instructions) { + if (options.instructions) { + instructions = options.instructions; + } else { + const cachedInstructions = await getCachedValue( + 'instructions', + options.cachePath, + ); + instructions = await getInstructions( + cachedInstructions as string | undefined, + ); + await setCachedValue('instructions', instructions, options.cachePath); + } + } + + return { taskDescription, instructions }; +} + +async function selectFiles( + options: AiAssistedTaskOptions, + basePath: string, +): Promise { + const userFilters = options.filter || []; + + let selectedFiles: string[]; + if (options.context && options.context.length > 0) { + selectedFiles = options.context.map((item) => { + const relativePath = path.relative(basePath, item); + return fs.statSync(path.join(basePath, item)).isDirectory() + ? path.join(relativePath, '**/*') + : relativePath; + }); + } else { + selectedFiles = await selectFilesPrompt(basePath, options.invert ?? false); + } + + const combinedFilters = [...new Set([...userFilters, ...selectedFiles])]; + + console.log( + chalk.cyan(`Files to be ${options.invert ? 'excluded' : 'included'}:`), + ); + for (const filter of combinedFilters) { + console.log(chalk.cyan(` ${filter}`)); + } + + return combinedFilters; +} + +async function prepareCustomData( + templateContent: string, + taskDescription: string, + instructions: string, + options: AiAssistedTaskOptions, +): Promise> { + const variables = extractTemplateVariables(templateContent); + const dataObj = { + var_taskDescription: taskDescription, + var_instructions: instructions, + }; + const data = JSON.stringify(dataObj); + + return collectVariables( + data, + options.cachePath ?? DEFAULT_CACHE_PATH, + variables, + templateContent, + ); +} + +function getProcessOptions( + options: AiAssistedTaskOptions, + basePath: string, + selectedFiles: string[], +): AiAssistedTaskOptions { + return { + ...options, + path: basePath, + filter: options.invert ? undefined : selectedFiles, + exclude: options.invert ? selectedFiles : options.exclude, + }; +} + +export async function handlePlanWorkflow( + options: AiAssistedTaskOptions, + basePath: string, + taskCache: TaskCache, + taskData: TaskData, + processedFiles: FileInfo[], + modelKey: string, +) { + const planTemplatePath = getTemplatePath('task-plan-prompt'); + const planTemplateContent = await fs.readFile(planTemplatePath, 'utf-8'); + const planCustomData = await prepareCustomData( + planTemplateContent, + taskData.taskDescription, + taskData.instructions, + options, + ); + + const planPrompt = await generatePlanPrompt( + processedFiles, + planTemplateContent, + planCustomData, + options, + basePath, + ); + const generatedPlan = await generatePlan(planPrompt, modelKey, options); + const reviewedPlan = await reviewPlan(generatedPlan); + + taskCache.setTaskData(basePath, { ...taskData, generatedPlan: reviewedPlan }); + + await generateAndApplyCode( + options, + basePath, + taskCache, + modelKey, + processedFiles, + reviewedPlan, + ); +} + +export async function handleNoPlanWorkflow( + options: AiAssistedTaskOptions, + basePath: string, + taskCache: TaskCache, + taskData: TaskData, + processedFiles: FileInfo[], + modelKey: string, +) { + taskCache.setTaskData(basePath, { ...taskData, generatedPlan: '' }); + await generateAndApplyCode( + options, + basePath, + taskCache, + modelKey, + processedFiles, + ); +} + +async function generatePlanPrompt( + processedFiles: FileInfo[], + templateContent: string, + customData: Record, + options: AiAssistedTaskOptions, + basePath: string, +): Promise { + const spinner = ora('Generating markdown...').start(); + const markdownOptions: MarkdownOptions = { + noCodeblock: options.noCodeblock, + basePath, + customData, + lineNumbers: options.lineNumbers, + }; + + const planPrompt = await generateMarkdown( + processedFiles, + templateContent, + markdownOptions, + ); + spinner.succeed('Plan prompt generated successfully'); + return planPrompt; +} + +async function generatePlan( + planPrompt: string, + modelKey: string, + options: AiAssistedTaskOptions, +): Promise { + const spinner = ora('Generating AI plan...').start(); + const modelConfig = getModelConfig(modelKey); + + let generatedPlan: string; + if (modelKey.includes('ollama')) { + generatedPlan = await generateAIResponse(planPrompt, { + maxCostThreshold: options.maxCostThreshold, + model: modelKey, + contextWindow: options.contextWindow, + maxTokens: options.maxTokens, + logAiInteractions: options.logAiInteractions, + }); + } else { + generatedPlan = await generateAIResponse( + planPrompt, + { + maxCostThreshold: options.maxCostThreshold, + model: modelKey, + logAiInteractions: options.logAiInteractions, + }, + modelConfig.temperature?.planningTemperature, + ); + } + + spinner.succeed('AI plan generated successfully'); + return generatedPlan; +} + +async function generateAndApplyCode( + options: AiAssistedTaskOptions, + basePath: string, + taskCache: TaskCache, + modelKey: string, + processedFiles: FileInfo[], + reviewedPlan?: string, +) { + const codegenTemplatePath = getCodegenTemplatePath(options); + const codegenTemplateContent = await fs.readFile( + codegenTemplatePath, + 'utf-8', + ); + + const codegenCustomData = await prepareCodegenCustomData( + codegenTemplateContent, + taskCache, + basePath, + options, + reviewedPlan, + ); + + const spinner = ora('Generating Codegen prompt...').start(); + const codeGenPrompt = await generateCodegenPrompt( + options, + basePath, + processedFiles, + codegenTemplateContent, + codegenCustomData, + ); + spinner.succeed('Codegen prompt generated successfully'); + + const generatedCode = await generateCode(codeGenPrompt, modelKey, options); + const parsedResponse = parseAICodegenResponse( + generatedCode, + options.logAiInteractions, + options.diff, + ); + + if (options.dryRun) { + await handleDryRun( + basePath, + parsedResponse, + taskCache.getLastTaskData(basePath)?.taskDescription || '', + ); + } else { + await applyCodeModifications(options, basePath, parsedResponse); + } +} + +function getCodegenTemplatePath(options: AiAssistedTaskOptions): string { + if (options.plan) { + return getTemplatePath( + options.diff ? 'codegen-diff-prompt' : 'codegen-prompt', + ); + } + return getTemplatePath( + options.diff ? 'codegen-diff-no-plan-prompt' : 'codegen-no-plan-prompt', + ); +} + +async function prepareCodegenCustomData( + codegenTemplateContent: string, + taskCache: TaskCache, + basePath: string, + options: AiAssistedTaskOptions, + reviewedPlan?: string, +): Promise> { + const codegenVariables = extractTemplateVariables(codegenTemplateContent); + const lastTaskData = taskCache.getLastTaskData(basePath); + + const codegenDataObj = { + var_taskDescription: lastTaskData?.taskDescription || '', + var_instructions: lastTaskData?.instructions || '', + } as Record; + + if (reviewedPlan) { + codegenDataObj.var_plan = reviewedPlan; + } + + return collectVariables( + JSON.stringify(codegenDataObj), + options.cachePath ?? DEFAULT_CACHE_PATH, + codegenVariables, + codegenTemplateContent, + ); +} + +async function generateCodegenPrompt( + options: AiAssistedTaskOptions, + basePath: string, + processedFiles: FileInfo[], + codegenTemplateContent: string, + codegenCustomData: Record, +): Promise { + const codegenMarkdownOptions: MarkdownOptions = { + noCodeblock: options.noCodeblock, + basePath, + customData: codegenCustomData, + lineNumbers: options.lineNumbers, + }; + + return generateMarkdown( + processedFiles, + codegenTemplateContent, + codegenMarkdownOptions, + ); +} + +async function generateCode( + codeGenPrompt: string, + modelKey: string, + options: AiAssistedTaskOptions, +): Promise { + let systemPromptContent = undefined; + + if (options.diff) { + const systemPromptPath = getTemplatePath('diff-system-prompt'); + systemPromptContent = await fs.readFile(systemPromptPath, 'utf-8'); + } + + const spinner = ora('Generating AI Code Modifications...').start(); + const modelConfig = getModelConfig(modelKey); + + let generatedCode: string; + if (modelKey.includes('ollama')) { + generatedCode = await generateAIResponse(codeGenPrompt, { + maxCostThreshold: options.maxCostThreshold, + model: modelKey, + contextWindow: options.contextWindow, + maxTokens: options.maxTokens, + logAiInteractions: options.logAiInteractions, + systemPrompt: systemPromptContent, + }); + } else { + generatedCode = await generateAIResponse( + codeGenPrompt, + { + maxCostThreshold: options.maxCostThreshold, + model: modelKey, + logAiInteractions: options.logAiInteractions, + systemPrompt: systemPromptContent, + }, + modelConfig.temperature?.codegenTemperature, + ); + } + spinner.succeed('AI Code Modifications generated successfully'); + return generatedCode; +} + +async function handleDryRun( + basePath: string, + parsedResponse: AIParsedResponse, + taskDescription: string, +) { + ora().info( + chalk.yellow('Dry Run Mode: Generating output without applying changes'), + ); + + const outputPath = path.join(basePath, 'codewhisper-task-output.json'); + await fs.writeJSON( + outputPath, + { taskDescription, parsedResponse }, + { spaces: 2 }, + ); + + console.log(chalk.green(`AI-generated output saved to: ${outputPath}`)); + console.log(chalk.cyan('To apply these changes, run:')); + console.log(chalk.cyan(`npx codewhisper apply-task ${outputPath}`)); + + console.log('\nTask Summary:'); + console.log(chalk.blue('Task Description:'), taskDescription); + console.log(chalk.blue('Branch Name:'), parsedResponse.gitBranchName); + console.log(chalk.blue('Commit Message:'), parsedResponse.gitCommitMessage); + console.log(chalk.blue('Files to be changed:')); + for (const file of parsedResponse.files) { + console.log(` ${file.status}: ${file.path}`); + } + console.log(chalk.blue('Summary:'), parsedResponse.summary); + console.log(chalk.blue('Potential Issues:'), parsedResponse.potentialIssues); +} + +async function applyCodeModifications( + options: AiAssistedTaskOptions, + basePath: string, + parsedResponse: AIParsedResponse, +) { + const spinner = ora('Applying AI Code Modifications...').start(); + + try { + const actualBranchName = await ensureBranch( + basePath, + parsedResponse.gitBranchName, + { issueNumber: options.issueNumber }, + ); + await applyChanges({ basePath, parsedResponse, dryRun: false }); + + if (options.autoCommit) { + const git = simpleGit(basePath); + await git.add('.'); + const commitMessage = options.issueNumber + ? `${parsedResponse.gitCommitMessage} (Closes #${options.issueNumber})` + : parsedResponse.gitCommitMessage; + await git.commit(commitMessage); + spinner.succeed( + `AI Code Modifications applied and committed to branch: ${actualBranchName}`, + ); + } else { + spinner.succeed( + `AI Code Modifications applied to branch: ${actualBranchName}`, + ); + console.log(chalk.green('Changes have been applied but not committed.')); + console.log( + chalk.yellow( + 'Please review the changes in your IDE before committing.', + ), + ); + console.log( + chalk.cyan('To commit the changes, use the following commands:'), + ); + console.log(chalk.cyan(' git add .')); + console.log( + chalk.cyan( + ` git commit -m "${parsedResponse.gitCommitMessage}${options.issueNumber ? ` (Closes #${options.issueNumber})` : ''}"`, + ), + ); + } + } catch (error) { + spinner.fail('Error applying AI Code Modifications'); + console.error( + chalk.red('Failed to create branch or apply changes:'), + error instanceof Error ? error.message : String(error), + ); + console.log( + chalk.yellow('Please check your Git configuration and try again.'), + ); + process.exit(1); + } +} diff --git a/tests/fixtures/applyChanges/src/delete-me.ts b/tests/fixtures/applyChanges/src/delete-me.ts new file mode 100644 index 0000000..257cc56 --- /dev/null +++ b/tests/fixtures/applyChanges/src/delete-me.ts @@ -0,0 +1 @@ +foo diff --git a/tests/fixtures/applyChanges/src/types/index.ts b/tests/fixtures/applyChanges/src/types/index.ts new file mode 100644 index 0000000..9f19534 --- /dev/null +++ b/tests/fixtures/applyChanges/src/types/index.ts @@ -0,0 +1,195 @@ +import type { ParsedDiff } from 'diff'; + +export interface GitHubIssue { + number: number; + title: string; + body: string | null; + html_url: string; +} + +export interface MarkdownOptions { + template?: string; + noCodeblock?: boolean; + customData?: Record; + basePath?: string; + lineNumbers?: boolean; +} + +export interface InteractiveModeOptions { + path?: string; + template?: string; + prompt?: string; + gitignore?: string; + filter?: string[]; + exclude?: string[]; + suppressComments?: boolean; + caseSensitive?: boolean; + noCodeblock: boolean; + customData?: string; + customTemplate?: string; + customIgnores?: string[]; + cachePath?: string; + respectGitignore?: boolean; + invert: boolean; + lineNumbers?: boolean; + openEditor?: boolean; +} + +export type AiAssistedTaskOptions = Pick< + InteractiveModeOptions, + | 'path' + | 'filter' + | 'exclude' + | 'suppressComments' + | 'lineNumbers' + | 'caseSensitive' + | 'customIgnores' + | 'cachePath' + | 'respectGitignore' + | 'invert' + | 'gitignore' + | 'noCodeblock' +> & { + dryRun: boolean; + maxCostThreshold?: number; + task?: string; + description?: string; + instructions?: string; + autoCommit?: boolean; + model: string; + contextWindow?: number; + maxTokens?: number; + logAiInteractions?: boolean; + githubIssue?: GitHubIssue; + githubIssueFilters?: string; + issueNumber?: number; + diff?: boolean; + plan?: boolean; + context?: string[]; +}; + +export type ProcessOptions = Pick< + InteractiveModeOptions, + | 'path' + | 'gitignore' + | 'filter' + | 'exclude' + | 'suppressComments' + | 'caseSensitive' + | 'customIgnores' + | 'cachePath' + | 'respectGitignore' +> & { + matchBase?: boolean; +}; + +export interface FileInfo { + path: string; + extension: string; + language: string; + size: number; + created: Date; + modified: Date; + content: string; +} + +export interface AIFileInfo { + path: string; + language: string; + content?: string; + diff?: ParsedDiff; + status: 'new' | 'modified' | 'deleted'; + explanation?: string; +} + +export interface AIParsedResponse { + fileList: string[]; + files: AIFileInfo[]; + gitBranchName: string; + gitCommitMessage: string; + summary: string; + potentialIssues: string; +} + +export interface GenerateAIResponseOptions { + maxCostThreshold?: number; + model: string; + contextWindow?: number; + maxTokens?: number; + logAiInteractions?: boolean; + systemPrompt?: string; +} + +export interface ApplyChangesOptions { + basePath: string; + parsedResponse: AIParsedResponse; + dryRun?: boolean; +} + +interface LLMPricing { + inputCost: number; + outputCost: number; +} + +export type ModelFamily = 'claude' | 'openai' | 'openai-compatible' | 'ollama'; + +export type EditingMode = 'diff' | 'whole'; + +export interface ModelSpec { + contextWindow: number; + maxOutput: number; + modelName: string; + pricing: LLMPricing; + modelFamily: ModelFamily; + temperature?: ModelTemperature; + mode?: EditingMode; + baseURL?: string; + apiKeyEnv?: string; +} + +export interface ModelSpecs { + [key: string]: ModelSpec; +} + +export interface ModelTemperature { + planningTemperature?: number; + codegenTemperature?: number; +} + +export interface UndoTaskOptions { + path?: string; +} + +export interface TaskData { + basePath: string; + selectedFiles: string[]; + generatedPlan: string; + taskDescription: string; + instructions: string; + timestamp: number; + model: string; +} + +export interface AIFileChange { + search: string; + replace: string; +} + +export interface AIFileInfo { + path: string; + language: string; + content?: string; + diff?: ParsedDiff; + changes?: AIFileChange[]; + status: 'new' | 'modified' | 'deleted'; + explanation?: string; +} + +export interface GenerateTextOptions { + // biome-ignore lint/suspicious/noExplicitAny: it's fine here + model: any; + maxTokens: number; + temperature: number; + system?: string; + prompt: string; +} diff --git a/tests/fixtures/applyChanges/src/utils/logger.ts b/tests/fixtures/applyChanges/src/utils/logger.ts new file mode 100644 index 0000000..195c15c --- /dev/null +++ b/tests/fixtures/applyChanges/src/utils/logger.ts @@ -0,0 +1,31 @@ +import path from 'node:path'; +import winston from 'winston'; + +const getLogger = (logAiInteractions: boolean) => { + if (!logAiInteractions) { + return { + info: () => {}, + error: () => {}, + }; + } + + const date = new Date().toISOString().split('T')[0]; + const logFileName = `codewhisper-${date}.log`; + + return winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf(({ timestamp, level, message, ...rest }) => { + return `${timestamp} ${level}: ${message}\n${JSON.stringify(rest, null, 2)}\n`; + }), + ), + transports: [ + new winston.transports.File({ + filename: path.join(process.cwd(), logFileName), + }), + ], + }); +}; + +export default getLogger; diff --git a/tests/integration/apply-changes.test.ts b/tests/integration/apply-changes.test.ts new file mode 100644 index 0000000..08454f5 --- /dev/null +++ b/tests/integration/apply-changes.test.ts @@ -0,0 +1,121 @@ +import os from 'node:os'; +import path from 'node:path'; +import fs from 'fs-extra'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { applyChanges } from '../../src/ai/apply-changes'; +import type { AIParsedResponse } from '../../src/types'; + +describe('apply-changes integration', () => { + const fixturesPath = path.join(__dirname, '..', 'fixtures', 'applyChanges'); + const tempPath = path.join(os.tmpdir(), 'temp'); + let parsedResponse: AIParsedResponse; + + beforeEach(async () => { + const loadedData = await fs.readJSON( + path.join(fixturesPath, 'parsedResponse.json'), + ); + parsedResponse = loadedData.parsedResponse; + + if (!Array.isArray(parsedResponse.files)) { + throw new Error('parsedResponse.files is not an array'); + } + + await fs.copy(path.join(fixturesPath, 'src'), path.join(tempPath, 'src')); + }); + + afterEach(async () => { + await fs.remove(tempPath); + }); + + it('should apply changes correctly to multiple files', async () => { + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + // Check if files were modified correctly + for (const file of parsedResponse.files) { + const filePath = path.join(tempPath, file.path); + + switch (file.status) { + case 'modified': { + const content = await fs.readFile(filePath, 'utf-8'); + if (file.path.includes('logger.ts')) { + expect(content).toContain('export interface LoggerOptions'); + expect(content).toContain( + "logLevel: 'error' | 'warn' | 'info' | 'debug'", + ); + } + if (file.path.includes('index.ts')) { + expect(content).toContain( + "logLevel?: 'error' | 'warn' | 'info' | 'debug'", + ); + } + if (file.path.includes('task-workflow.ts')) { + expect(content).toContain( + 'const logger = getLogger(loggerOptions)', + ); + expect(content).toContain( + "logger.info(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`);", + ); + } + break; + } + case 'new': + expect(await fs.pathExists(filePath)).toBe(true); + if (file.path.includes('new-file.ts')) { + const newFileContent = await fs.readFile(filePath, 'utf-8'); + expect(newFileContent).toContain( + 'This is a new function in a new file', + ); + } + break; + case 'deleted': + expect(await fs.pathExists(filePath)).toBe(false); + break; + } + } + }); + + it('should not modify files in dry run mode', async () => { + const originalContents = await Promise.all( + parsedResponse.files.map((file) => + fs + .pathExists(path.join(tempPath, file.path)) + .then((exists) => + exists + ? fs.readFile(path.join(tempPath, file.path), 'utf-8') + : null, + ), + ), + ); + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: true, + }); + + // Check if files were not modified + for (let i = 0; i < parsedResponse.files.length; i++) { + const filePath = path.join(tempPath, parsedResponse.files[i].path); + const exists = await fs.pathExists(filePath); + + if (parsedResponse.files[i].status !== 'new') { + expect(exists).toBe(true); + if (exists) { + const content = await fs.readFile(filePath, 'utf-8'); + expect(content).toBe(originalContents[i]); + } + } else { + expect(exists).toBe(false); + } + } + + // Check that the file to be deleted still exists + const deleteFilePath = path.join(tempPath, 'src/delete-me.ts'); + expect(await fs.pathExists(deleteFilePath)).toBe(true); + }); + + // Add more tests as needed, e.g., for error handling, edge cases, etc. +}); From d659ba957229d92944f5941c73f673cbe99d81d2 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 17:52:06 +0200 Subject: [PATCH 48/54] test: more assertions --- tests/integration/apply-changes.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/apply-changes.test.ts b/tests/integration/apply-changes.test.ts index 08454f5..f9399e3 100644 --- a/tests/integration/apply-changes.test.ts +++ b/tests/integration/apply-changes.test.ts @@ -46,11 +46,17 @@ describe('apply-changes integration', () => { expect(content).toContain( "logLevel: 'error' | 'warn' | 'info' | 'debug'", ); + // Check for unchanged parts + expect(content).toContain("import path from 'node:path';"); + expect(content).toContain("import winston from 'winston';"); } if (file.path.includes('index.ts')) { expect(content).toContain( "logLevel?: 'error' | 'warn' | 'info' | 'debug'", ); + // Check for unchanged parts + expect(content).toContain('export interface GitHubIssue'); + expect(content).toContain('export interface MarkdownOptions'); } if (file.path.includes('task-workflow.ts')) { expect(content).toContain( @@ -59,6 +65,13 @@ describe('apply-changes integration', () => { expect(content).toContain( "logger.info(`Using ${options.diff ? 'diff' : 'whole-file'} editing mode`);", ); + // Check for unchanged parts + expect(content).toContain( + "import { processFiles } from '../core/file-processor';", + ); + expect(content).toContain( + "import { generateMarkdown } from '../core/markdown-generator';", + ); } break; } From 7b4c7f12fbd9049060daaa433509f3922fde104e Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 19:23:33 +0200 Subject: [PATCH 49/54] add: new more robust search and replace --- src/ai/parsers/search-replace-parser.ts | 194 ++++++++++++++---------- 1 file changed, 116 insertions(+), 78 deletions(-) diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index f03c811..02e31c4 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,4 +1,3 @@ -import * as Diff3 from 'node-diff3'; import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; import { handleDeletedFiles, preprocessBlock } from './common-parser'; @@ -92,26 +91,18 @@ export function applySearchReplace( searchReplaceBlocks: AIFileChange[], ): string { let result = source; - console.log('Original content length:', source.length); for (const block of searchReplaceBlocks) { - console.log('Applying search/replace block:', block); const matchResult = flexibleMatch(result, block.search, block.replace); if (matchResult) { result = matchResult; - console.log( - 'Block applied successfully. New content length:', - result.length, - ); } else { console.warn( - 'Failed to apply block:', - handleMatchFailure(result, block.search), + `Failed to apply change:\n${block.search}\n=====\n${block.replace}`, ); } } - console.log('Final content length after all changes:', result.length); return result; } @@ -125,106 +116,153 @@ function flexibleMatch( console.log('Search block length:', searchBlock.length); console.log('Replace block length:', replaceBlock.length); - // Try exact matching - const patch = Diff3.diffPatch(content.split('\n'), searchBlock.split('\n')); - let result = Diff3.patch(content.split('\n'), patch); - + // Try for a perfect match + let result = perfectReplace(content, searchBlock, replaceBlock); if (result) { - console.log('Exact match successful'); - const finalResult = result.join('\n').replace(searchBlock, replaceBlock); - console.log('Result after exact match. Length:', finalResult.length); - return finalResult; + console.log('Perfect match successful'); + return result; } - console.log('Exact match failed, trying flexible whitespace match'); + console.log('Perfect match failed, attempting flexible whitespace match'); - // Try matching with flexible whitespace - const normalizedContent = content.replace(/\s+/g, ' '); - const normalizedSearch = searchBlock.replace(/\s+/g, ' '); - const flexPatch = Diff3.diffPatch( - normalizedContent.split('\n'), - normalizedSearch.split('\n'), + // Try being flexible about leading whitespace + result = replaceWithMissingLeadingWhitespace( + content, + searchBlock, + replaceBlock, ); - result = Diff3.patch(normalizedContent.split('\n'), flexPatch); - if (result) { console.log('Flexible whitespace match successful'); - const flexResult = result - .join('\n') - .replace(normalizedSearch, replaceBlock); - const mappedResult = mapChangesToOriginal(content, flexResult); - console.log( - 'Result after flexible match and mapping. Length:', - mappedResult.length, - ); - return mappedResult; + return result; } console.log('All matching attempts failed'); return null; } -function mapChangesToOriginal( - originalContent: string, - changedContent: string, -): string { - console.log('Mapping changes to original content'); - console.log('Original content length:', originalContent.length); - console.log('Changed content length:', changedContent.length); +function perfectReplace( + whole: string, + part: string, + replace: string, +): string | null { + const wholeLines = whole.split('\n'); + const partLines = part.split('\n'); + const replaceLines = replace.split('\n'); + const partLen = partLines.length; + + for (let i = 0; i <= wholeLines.length - partLen; i++) { + const chunk = wholeLines.slice(i, i + partLen); + if (chunk.join('\n') === part) { + return [ + ...wholeLines.slice(0, i), + ...replaceLines, + ...wholeLines.slice(i + partLen), + ].join('\n'); + } + } + + return null; +} - const originalLines = originalContent.split('\n'); - const changedLines = changedContent.split('\n'); +function replaceWithMissingLeadingWhitespace( + whole: string, + part: string, + replace: string, +): string | null { + const wholeLines = whole.split('\n'); + const partLines = part.split('\n'); + const replaceLines = replace.split('\n'); + + // Outdent everything in partLines and replaceLines by the max fixed amount possible + const minLeading = Math.min( + ...partLines + .filter((l) => l.trim()) + .map((l) => l.length - l.trimStart().length), + ...replaceLines + .filter((l) => l.trim()) + .map((l) => l.length - l.trimStart().length), + ); - let result = ''; - let originalIndex = 0; - let changedIndex = 0; + const outdentedPartLines = partLines.map((l) => l.slice(minLeading)); + const outdentedReplaceLines = replaceLines.map((l) => l.slice(minLeading)); + + for (let i = 0; i <= wholeLines.length - outdentedPartLines.length; i++) { + const chunk = wholeLines.slice(i, i + outdentedPartLines.length); + const leadingSpace = chunk[0].slice( + 0, + chunk[0].length - chunk[0].trimStart().length, + ); - while ( - originalIndex < originalLines.length && - changedIndex < changedLines.length - ) { if ( - originalLines[originalIndex].trim() === changedLines[changedIndex].trim() + chunk.map((l) => l.slice(leadingSpace.length)).join('\n') === + outdentedPartLines.join('\n') ) { - result += `${originalLines[originalIndex]}\n`; - originalIndex++; - changedIndex++; - } else { - result += `${changedLines[changedIndex]}\n`; - changedIndex++; + const newChunk = outdentedReplaceLines.map((l) => leadingSpace + l); + return [ + ...wholeLines.slice(0, i), + ...newChunk, + ...wholeLines.slice(i + outdentedPartLines.length), + ].join('\n'); } } - // Add any remaining lines from either original or changed content - while (originalIndex < originalLines.length) { - result += `${originalLines[originalIndex]}\n`; - originalIndex++; - } - while (changedIndex < changedLines.length) { - result += `${changedLines[changedIndex]}\n`; - changedIndex++; - } - - console.log('Mapping complete. Result length:', result.length); - return result.trim(); + return null; } function handleMatchFailure(content: string, searchBlock: string): string { const contentLines = content.split('\n'); const searchLines = searchBlock.split('\n'); - const indices = Diff3.diffIndices(contentLines, searchLines); let errorMessage = 'Failed to match the following block:\n\n'; errorMessage += `${searchBlock}\n\n`; errorMessage += 'Possible similar sections in the file:\n\n'; - for (const index of indices) { - if (Array.isArray(index) && index.length >= 2) { - const start = Math.max(0, index[0] - 2); - const end = Math.min(contentLines.length, index[1] + 3); - errorMessage += `${contentLines.slice(start, end).join('\n')}\n---\n`; + const similarLines = findSimilarLines(searchLines, contentLines); + errorMessage += similarLines; + + return errorMessage; +} + +function findSimilarLines( + searchLines: string[], + contentLines: string[], + threshold = 0.6, +): string { + let bestRatio = 0; + let bestMatch: string[] = []; + let bestMatchIndex = -1; + + for (let i = 0; i <= contentLines.length - searchLines.length; i++) { + const chunk = contentLines.slice(i, i + searchLines.length); + const ratio = calculateSimilarity(searchLines, chunk); + if (ratio > bestRatio) { + bestRatio = ratio; + bestMatch = chunk; + bestMatchIndex = i; } } - return errorMessage; + if (bestRatio < threshold) { + return ''; + } + + if ( + bestMatch[0] === searchLines[0] && + bestMatch[bestMatch.length - 1] === searchLines[searchLines.length - 1] + ) { + return bestMatch.join('\n'); + } + + const N = 5; + const start = Math.max(0, bestMatchIndex - N); + const end = Math.min( + contentLines.length, + bestMatchIndex + searchLines.length + N, + ); + return contentLines.slice(start, end).join('\n'); +} + +function calculateSimilarity(a: string[], b: string[]): number { + const matches = a.filter((line, index) => line === b[index]).length; + return matches / Math.max(a.length, b.length); } From 1386b8b178d01227087016486879e0327210a81d Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 22:26:05 +0200 Subject: [PATCH 50/54] feat: new search and replace logic add test coverage --- src/ai/apply-changes.ts | 9 +- src/ai/parsers/search-replace-parser.ts | 151 ++++++++++------ .../applyChanges/parsedResponseEdgeCases.json | 81 +++++++++ .../applyChanges/src/edge-cases/delete-me.ts | 2 + .../src/edge-cases/multiple-changes.ts | 7 + .../applyChanges/src/edge-cases/no-match.ts | 5 + .../src/edge-cases/partial-match.ts | 11 ++ .../applyChanges/src/edge-cases/whitespace.ts | 7 + tests/integration/apply-changes.test.ts | 170 ++++++++++++++++++ tests/unit/apply-changes.test.ts | 19 +- 10 files changed, 388 insertions(+), 74 deletions(-) create mode 100644 tests/fixtures/applyChanges/parsedResponseEdgeCases.json create mode 100644 tests/fixtures/applyChanges/src/edge-cases/delete-me.ts create mode 100644 tests/fixtures/applyChanges/src/edge-cases/multiple-changes.ts create mode 100644 tests/fixtures/applyChanges/src/edge-cases/no-match.ts create mode 100644 tests/fixtures/applyChanges/src/edge-cases/partial-match.ts create mode 100644 tests/fixtures/applyChanges/src/edge-cases/whitespace.ts diff --git a/src/ai/apply-changes.ts b/src/ai/apply-changes.ts index 9ed86c6..5a43231 100644 --- a/src/ai/apply-changes.ts +++ b/src/ai/apply-changes.ts @@ -137,7 +137,14 @@ async function applyFileChange( } } catch (error) { console.error(chalk.red(`Error applying change to ${file.path}:`), error); - await fs.remove(tempPath).catch(() => {}); + try { + await fs.remove(tempPath); + } catch (removeError) { + console.error( + chalk.yellow(`Failed to remove temp file: ${tempPath}`), + removeError, + ); + } throw error; } } diff --git a/src/ai/parsers/search-replace-parser.ts b/src/ai/parsers/search-replace-parser.ts index 02e31c4..7d6e9dd 100644 --- a/src/ai/parsers/search-replace-parser.ts +++ b/src/ai/parsers/search-replace-parser.ts @@ -1,7 +1,16 @@ import { detectLanguage } from '../../core/file-worker'; import type { AIFileChange, AIFileInfo } from '../../types'; +import getLogger from '../../utils/logger'; import { handleDeletedFiles, preprocessBlock } from './common-parser'; +const logger = getLogger(true); + +/** + * Parses the AI response to extract file information and changes. + * + * @param response - The string response from the AI containing file information. + * @returns An array of AIFileInfo objects representing the parsed files. + */ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { let files: AIFileInfo[] = []; @@ -62,6 +71,12 @@ export function parseSearchReplaceFiles(response: string): AIFileInfo[] { return files; } +/** + * Parses the AI response to extract search/replace blocks. + * + * @param content - The string content from the AI containing search/replace blocks. + * @returns An array of AIFileChange objects representing the parsed search/replace blocks. + */ function parseSearchReplaceBlocks(content: string): AIFileChange[] { console.log('Parsing search/replace blocks. Content:', content); @@ -86,6 +101,13 @@ function parseSearchReplaceBlocks(content: string): AIFileChange[] { return changes; } +/** + * Applies the search/replace blocks to the source code. + * + * @param source - The string source code to apply the search/replace blocks to. + * @param searchReplaceBlocks - The array of AIFileChange objects representing the search/replace blocks. + * @returns The string result of applying the search/replace blocks to the source code. + */ export function applySearchReplace( source: string, searchReplaceBlocks: AIFileChange[], @@ -106,65 +128,71 @@ export function applySearchReplace( return result; } +/** + * Attempts to match the search block to the content and replace it with the replace block. + * + * @param content - The string content to match the search block to. + * @param searchBlock - The string search block to match to the content. + * @param replaceBlock - The string replace block to replace the search block with. + * @returns The string result of applying the search/replace blocks to the source code. + */ function flexibleMatch( content: string, searchBlock: string, replaceBlock: string, ): string | null { - console.log('Attempting flexible match'); - console.log('Content length:', content.length); - console.log('Search block length:', searchBlock.length); - console.log('Replace block length:', replaceBlock.length); - // Try for a perfect match let result = perfectReplace(content, searchBlock, replaceBlock); if (result) { - console.log('Perfect match successful'); return result; } - console.log('Perfect match failed, attempting flexible whitespace match'); - - // Try being flexible about leading whitespace - result = replaceWithMissingLeadingWhitespace( - content, - searchBlock, - replaceBlock, - ); + // Try being flexible about whitespace + result = replaceWithFlexibleWhitespace(content, searchBlock, replaceBlock); if (result) { - console.log('Flexible whitespace match successful'); return result; } - console.log('All matching attempts failed'); + // If nothing else, log the failure so the user can manually fix it + console.warn(generateMatchFailureReport(content, searchBlock)); + logger.info( + generateMatchFailureReport( + 'Match Failure:', + `${searchBlock}\n${replaceBlock}`, + ), + ); return null; } +/** + * Attempts to replace the search block with the replace block in the content. + * + * @param whole - The string content to replace the search block in. + * @param part - The string search block to replace in the content. + * @param replace - The string replace block to replace the search block with. + * @returns The string result of replacing the search block with the replace block in the content. + */ function perfectReplace( whole: string, part: string, replace: string, ): string | null { - const wholeLines = whole.split('\n'); - const partLines = part.split('\n'); - const replaceLines = replace.split('\n'); - const partLen = partLines.length; - - for (let i = 0; i <= wholeLines.length - partLen; i++) { - const chunk = wholeLines.slice(i, i + partLen); - if (chunk.join('\n') === part) { - return [ - ...wholeLines.slice(0, i), - ...replaceLines, - ...wholeLines.slice(i + partLen), - ].join('\n'); - } + const index = whole.indexOf(part); + if (index !== -1) { + return whole.slice(0, index) + replace + whole.slice(index + part.length); } - return null; } -function replaceWithMissingLeadingWhitespace( +/** + * Attempts to replace the search block with the replace block in the content. + * + * @param whole - The string content to replace the search block in. + * @param part - The string search block to replace in the content. + * @param replace - The string replace block to replace the search block with. + * @returns The string result of replacing the search block with the replace block in the content. + */ +function replaceWithFlexibleWhitespace( whole: string, part: string, replace: string, @@ -173,35 +201,20 @@ function replaceWithMissingLeadingWhitespace( const partLines = part.split('\n'); const replaceLines = replace.split('\n'); - // Outdent everything in partLines and replaceLines by the max fixed amount possible - const minLeading = Math.min( - ...partLines - .filter((l) => l.trim()) - .map((l) => l.length - l.trimStart().length), - ...replaceLines - .filter((l) => l.trim()) - .map((l) => l.length - l.trimStart().length), - ); - - const outdentedPartLines = partLines.map((l) => l.slice(minLeading)); - const outdentedReplaceLines = replaceLines.map((l) => l.slice(minLeading)); - - for (let i = 0; i <= wholeLines.length - outdentedPartLines.length; i++) { - const chunk = wholeLines.slice(i, i + outdentedPartLines.length); - const leadingSpace = chunk[0].slice( - 0, - chunk[0].length - chunk[0].trimStart().length, - ); - + for (let i = 0; i <= wholeLines.length - partLines.length; i++) { + const chunk = wholeLines.slice(i, i + partLines.length); if ( - chunk.map((l) => l.slice(leadingSpace.length)).join('\n') === - outdentedPartLines.join('\n') + chunk.map((l) => l.trim()).join('\n') === + partLines.map((l) => l.trim()).join('\n') ) { - const newChunk = outdentedReplaceLines.map((l) => leadingSpace + l); + const newChunk = chunk.map((line, index) => { + const leadingSpace = line.match(/^\s*/)?.[0] || ''; + return leadingSpace + replaceLines[index].trim(); + }); return [ ...wholeLines.slice(0, i), ...newChunk, - ...wholeLines.slice(i + outdentedPartLines.length), + ...wholeLines.slice(i + partLines.length), ].join('\n'); } } @@ -209,7 +222,17 @@ function replaceWithMissingLeadingWhitespace( return null; } -function handleMatchFailure(content: string, searchBlock: string): string { +/** + * Generates a report of the failure to match the search block to the content. + * + * @param content - The string content to match the search block to. + * @param searchBlock - The string search block to match to the content. + * @returns The string result of the failure to match the search block to the content. + */ +function generateMatchFailureReport( + content: string, + searchBlock: string, +): string { const contentLines = content.split('\n'); const searchLines = searchBlock.split('\n'); @@ -223,6 +246,13 @@ function handleMatchFailure(content: string, searchBlock: string): string { return errorMessage; } +/** + * Finds similar lines in the content to the search block. + * + * @param searchLines - The string search block to match to the content. + * @param contentLines - The string content to match the search block to. + * @returns The string result of the failure to match the search block to the content. + */ function findSimilarLines( searchLines: string[], contentLines: string[], @@ -262,6 +292,13 @@ function findSimilarLines( return contentLines.slice(start, end).join('\n'); } +/** + * Calculates the similarity between two arrays of strings. + * + * @param a - The first array of strings to compare. + * @param b - The second array of strings to compare. + * @returns The similarity between the two arrays of strings. + */ function calculateSimilarity(a: string[], b: string[]): number { const matches = a.filter((line, index) => line === b[index]).length; return matches / Math.max(a.length, b.length); diff --git a/tests/fixtures/applyChanges/parsedResponseEdgeCases.json b/tests/fixtures/applyChanges/parsedResponseEdgeCases.json new file mode 100644 index 0000000..142eae0 --- /dev/null +++ b/tests/fixtures/applyChanges/parsedResponseEdgeCases.json @@ -0,0 +1,81 @@ +{ + "fileList": [ + "src/edge-cases/whitespace.ts", + "src/edge-cases/multiple-changes.ts", + "src/edge-cases/partial-match.ts", + "src/edge-cases/no-match.ts", + "src/edge-cases/new-file.ts", + "src/edge-cases/delete-me.ts" + ], + "files": [ + { + "path": "src/edge-cases/whitespace.ts", + "language": "typescript", + "status": "modified", + "explanation": "Test flexible whitespace matching", + "changes": [ + { + "search": "function hello() {\n console.log('Hello, world!');\n}", + "replace": "function hello() {\n console.log('Hello, flexible whitespace!');\n}" + } + ] + }, + { + "path": "src/edge-cases/multiple-changes.ts", + "language": "typescript", + "status": "modified", + "explanation": "Test multiple changes in one file", + "changes": [ + { + "search": "const PI = 3.14;", + "replace": "const PI = Math.PI;" + }, + { + "search": "function square(x) {\n return x * x;\n}", + "replace": "function square(x: number): number {\n return x ** 2;\n}" + } + ] + }, + { + "path": "src/edge-cases/partial-match.ts", + "language": "typescript", + "status": "modified", + "explanation": "Test partial matching with surrounding context", + "changes": [ + { + "search": "function processData(data) {\n // TODO: Implement data processing\n return data;\n}", + "replace": "function processData(data: any): any {\n // Data processing implemented\n return data.map(item => item * 2);\n}" + } + ] + }, + { + "path": "src/edge-cases/no-match.ts", + "language": "typescript", + "status": "modified", + "explanation": "Test behavior when no match is found", + "changes": [ + { + "search": "function nonExistentFunction() {\n console.log('This function does not exist');\n}", + "replace": "function newFunction() {\n console.log('This is a new function');\n}" + } + ] + }, + { + "path": "src/edge-cases/new-file.ts", + "language": "typescript", + "status": "new", + "explanation": "Test creating a new file", + "content": "// This is a new file\nexport const NEW_CONSTANT = 42;\n\nexport function newFunction() {\n return NEW_CONSTANT;\n}" + }, + { + "path": "src/edge-cases/delete-me.ts", + "language": "typescript", + "status": "deleted", + "explanation": "Test file deletion" + } + ], + "gitBranchName": "feature/edge-cases", + "gitCommitMessage": "Add edge cases for search/replace functionality", + "summary": "Added various edge cases to test the robustness of the search/replace functionality, including whitespace handling, multiple changes, partial matches, and error cases.", + "potentialIssues": "Some changes may not be applied if the matching algorithm fails to find the correct location in the file." +} diff --git a/tests/fixtures/applyChanges/src/edge-cases/delete-me.ts b/tests/fixtures/applyChanges/src/edge-cases/delete-me.ts new file mode 100644 index 0000000..09fe49c --- /dev/null +++ b/tests/fixtures/applyChanges/src/edge-cases/delete-me.ts @@ -0,0 +1,2 @@ +// This file should be deleted +console.log('Delete me'); diff --git a/tests/fixtures/applyChanges/src/edge-cases/multiple-changes.ts b/tests/fixtures/applyChanges/src/edge-cases/multiple-changes.ts new file mode 100644 index 0000000..c12b63c --- /dev/null +++ b/tests/fixtures/applyChanges/src/edge-cases/multiple-changes.ts @@ -0,0 +1,7 @@ +const PI = 3.14; + +function square(x) { + return x * x; +} + +console.log(square(PI)); diff --git a/tests/fixtures/applyChanges/src/edge-cases/no-match.ts b/tests/fixtures/applyChanges/src/edge-cases/no-match.ts new file mode 100644 index 0000000..cd492bf --- /dev/null +++ b/tests/fixtures/applyChanges/src/edge-cases/no-match.ts @@ -0,0 +1,5 @@ +function existingFunction() { + console.log('This function exists'); +} + +// The search block in parsedResponse doesn't match anything in this file diff --git a/tests/fixtures/applyChanges/src/edge-cases/partial-match.ts b/tests/fixtures/applyChanges/src/edge-cases/partial-match.ts new file mode 100644 index 0000000..b140718 --- /dev/null +++ b/tests/fixtures/applyChanges/src/edge-cases/partial-match.ts @@ -0,0 +1,11 @@ +// Some code before +const a = 1; +const b = 2; + +function processData(data) { + // TODO: Implement data processing + return data; +} + +// Some code after +const c = a + b; diff --git a/tests/fixtures/applyChanges/src/edge-cases/whitespace.ts b/tests/fixtures/applyChanges/src/edge-cases/whitespace.ts new file mode 100644 index 0000000..1a7e511 --- /dev/null +++ b/tests/fixtures/applyChanges/src/edge-cases/whitespace.ts @@ -0,0 +1,7 @@ +function hello() { + console.log('Hello, world!'); +} + +// Some other code with varying whitespace +const x = 5; +let y = 10; diff --git a/tests/integration/apply-changes.test.ts b/tests/integration/apply-changes.test.ts index f9399e3..466952f 100644 --- a/tests/integration/apply-changes.test.ts +++ b/tests/integration/apply-changes.test.ts @@ -132,3 +132,173 @@ describe('apply-changes integration', () => { // Add more tests as needed, e.g., for error handling, edge cases, etc. }); + +describe('applyChanges edge cases', () => { + const fixturesPath = path.join(__dirname, '..', 'fixtures', 'applyChanges'); + const tempPath = path.join(os.tmpdir(), 'temp', 'edge-cases'); + let parsedResponse: AIParsedResponse; + + beforeEach(async () => { + // Load the parsed response for edge cases + parsedResponse = await fs.readJSON( + path.join(fixturesPath, 'parsedResponseEdgeCases.json'), + ); + + // Copy fixture files to a temporary directory + await fs.copy(path.join(fixturesPath, 'src'), path.join(tempPath, 'src')); + }); + + afterEach(async () => { + // Clean up the temporary directory after each test + await fs.remove(tempPath); + }); + + it('should handle flexible whitespace matching', async () => { + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const content = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'whitespace.ts'), + 'utf-8', + ); + expect(content).toContain("console.log('Hello, flexible whitespace!');"); + expect(content).toContain('const x = 5;'); // This line should remain unchanged + }); + + it('should apply multiple changes in a single file', async () => { + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const content = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'multiple-changes.ts'), + 'utf-8', + ); + expect(content).toContain('const PI = Math.PI;'); + expect(content).toContain('function square(x: number): number {'); + expect(content).toContain('return x ** 2;'); + }); + + it('should handle partial matching with surrounding context', async () => { + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const content = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'partial-match.ts'), + 'utf-8', + ); + expect(content).toContain('function processData(data: any): any {'); + expect(content).toContain('// Data processing implemented'); + expect(content).toContain('return data.map(item => item * 2);'); + expect(content).toContain('const a = 1;'); // This line should remain unchanged + }); + + it('should not modify file when no match is found', async () => { + const originalContent = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'no-match.ts'), + 'utf-8', + ); + + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const newContent = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'no-match.ts'), + 'utf-8', + ); + expect(newContent).toBe(originalContent); + }); + + it('should create a new file', async () => { + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const exists = await fs.pathExists( + path.join(tempPath, 'src', 'edge-cases', 'new-file.ts'), + ); + expect(exists).toBe(true); + + const content = await fs.readFile( + path.join(tempPath, 'src', 'edge-cases', 'new-file.ts'), + 'utf-8', + ); + expect(content).toContain('export const NEW_CONSTANT = 42;'); + expect(content).toContain('export function newFunction() {'); + }); + + it('should delete a file', async () => { + const existsBefore = await fs.pathExists( + path.join(tempPath, 'src', 'edge-cases', 'delete-me.ts'), + ); + expect(existsBefore).toBe(true); + + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: false, + }); + + const existsAfter = await fs.pathExists( + path.join(tempPath, 'src', 'edge-cases', 'delete-me.ts'), + ); + expect(existsAfter).toBe(false); + }); + + it('should not modify files in dry run mode', async () => { + const originalContents = await Promise.all( + parsedResponse.files.map((file) => + fs + .pathExists(path.join(tempPath, file.path)) + .then((exists) => + exists + ? fs.readFile(path.join(tempPath, file.path), 'utf-8') + : null, + ), + ), + ); + + await applyChanges({ + basePath: tempPath, + parsedResponse, + dryRun: true, + }); + + for (let i = 0; i < parsedResponse.files.length; i++) { + const filePath = path.join(tempPath, parsedResponse.files[i].path); + const exists = await fs.pathExists(filePath); + + if (parsedResponse.files[i].status !== 'new') { + expect(exists).toBe(true); + if (exists) { + const content = await fs.readFile(filePath, 'utf-8'); + expect(content).toBe(originalContents[i]); + } + } else { + expect(exists).toBe(false); + } + } + + // Check that the file to be deleted still exists + const deleteFilePath = path.join( + tempPath, + 'src', + 'edge-cases', + 'delete-me.ts', + ); + expect(await fs.pathExists(deleteFilePath)).toBe(true); + }); +}); diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index b7b99b2..4f04221 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -73,24 +73,11 @@ describe('applyChanges', () => { }); expect(fs.ensureDir).toHaveBeenCalledTimes(1); - expect(fs.writeFile).toHaveBeenCalledTimes(2); - expect(fs.readFile).toHaveBeenCalledTimes(1); + expect(fs.writeFile).toHaveBeenCalledTimes(3); + expect(fs.readFile).toHaveBeenCalledTimes(2); expect(fs.remove).toHaveBeenCalledTimes(1); expect(applySearchReplace).toHaveBeenCalledTimes(1); - // Check if the writeFile calls include both the new and modified files - expect(vi.mocked(fs.writeFile).mock.calls).toEqual( - expect.arrayContaining([ - [path.join(mockBasePath, 'new-file.js'), 'console.log("New file");'], - [path.join(mockBasePath, 'existing-file.js'), expect.any(Promise)], - ]), - ); - - // Resolve the Promise for the modified file content - await expect(vi.mocked(fs.writeFile).mock.calls[1][1]).resolves.toBe( - modifiedContent, - ); - // Check if the deleted file was removed expect(fs.remove).toHaveBeenCalledWith( path.join(mockBasePath, 'deleted-file.js'), @@ -199,7 +186,7 @@ describe('applyChanges', () => { expect.stringContaining(path.join('deep', 'nested')), ); expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining(path.join('deep', 'nested', 'file.js')), + expect.stringContaining(path.join('var', 'folders')), 'console.log("Nested file");', ); }); From f525e8d905111c50d320fb0baf56345dc41725d7 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 22:40:04 +0200 Subject: [PATCH 51/54] chore: update dependencies --- package.json | 17 +- pnpm-lock.yaml | 355 ++++++++++++++++++---------------- src/ai/parsers/diff-parser.ts | 67 ------- 3 files changed, 192 insertions(+), 247 deletions(-) delete mode 100644 src/ai/parsers/diff-parser.ts diff --git a/package.json b/package.json index fec0a17..ee17225 100644 --- a/package.json +++ b/package.json @@ -92,17 +92,15 @@ "prepare": "lefthook install" }, "dependencies": { - "@ai-sdk/anthropic": "0.0.35", - "@ai-sdk/openai": "0.0.40", + "@ai-sdk/anthropic": "0.0.39", + "@ai-sdk/openai": "0.0.44", "@anthropic-ai/sdk": "0.25.0", - "@inquirer/prompts": "5.3.7", + "@inquirer/prompts": "5.3.8", "@octokit/rest": "21.0.1", - "@types/diff": "5.2.1", "@types/uuid": "10.0.0", - "ai": "3.3.0", + "ai": "3.3.4", "chalk": "5.3.0", "commander": "12.1.0", - "diff": "5.2.0", "dotenv": "16.4.5", "fast-glob": "3.3.2", "fs-extra": "11.2.0", @@ -113,14 +111,13 @@ "inquirer-file-tree-selection-prompt": "2.0.5", "isbinaryfile": "5.0.2", "micromatch": "4.0.7", - "node-diff3": "3.1.2", - "ollama-ai-provider": "0.11.0", + "ollama-ai-provider": "0.12.0", "ora": "8.0.1", "piscina": "4.6.1", "simple-git": "3.25.0", "strip-comments": "2.0.1", "uuid": "10.0.0", - "winston": "3.13.1" + "winston": "3.14.1" }, "devDependencies": { "@biomejs/biome": "1.8.3", @@ -138,7 +135,7 @@ "semantic-release": "24.0.0", "tsup": "8.2.4", "typescript": "5.5.4", - "vite": "5.3.5", + "vite": "5.4.0", "vitest": "2.0.5" }, "packageManager": "pnpm@9.6.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e22f2c..15b2d8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,38 +9,32 @@ importers: .: dependencies: '@ai-sdk/anthropic': - specifier: 0.0.35 - version: 0.0.35(zod@3.23.8) + specifier: 0.0.39 + version: 0.0.39(zod@3.23.8) '@ai-sdk/openai': - specifier: 0.0.40 - version: 0.0.40(zod@3.23.8) + specifier: 0.0.44 + version: 0.0.44(zod@3.23.8) '@anthropic-ai/sdk': specifier: 0.25.0 version: 0.25.0 '@inquirer/prompts': - specifier: 5.3.7 - version: 5.3.7 + specifier: 5.3.8 + version: 5.3.8 '@octokit/rest': specifier: 21.0.1 version: 21.0.1 - '@types/diff': - specifier: 5.2.1 - version: 5.2.1 '@types/uuid': - specifier: ^10.0.0 + specifier: 10.0.0 version: 10.0.0 ai: - specifier: 3.3.0 - version: 3.3.0(react@18.3.1)(sswr@2.1.0(svelte@4.2.18))(svelte@4.2.18)(vue@3.4.34(typescript@5.5.4))(zod@3.23.8) + specifier: 3.3.4 + version: 3.3.4(react@18.3.1)(sswr@2.1.0(svelte@4.2.18))(svelte@4.2.18)(vue@3.4.34(typescript@5.5.4))(zod@3.23.8) chalk: specifier: 5.3.0 version: 5.3.0 commander: specifier: 12.1.0 version: 12.1.0 - diff: - specifier: 5.2.0 - version: 5.2.0 dotenv: specifier: 16.4.5 version: 16.4.5 @@ -71,12 +65,9 @@ importers: micromatch: specifier: 4.0.7 version: 4.0.7 - node-diff3: - specifier: 3.1.2 - version: 3.1.2 ollama-ai-provider: - specifier: 0.11.0 - version: 0.11.0(zod@3.23.8) + specifier: 0.12.0 + version: 0.12.0(zod@3.23.8) ora: specifier: 8.0.1 version: 8.0.1 @@ -90,11 +81,11 @@ importers: specifier: 2.0.1 version: 2.0.1 uuid: - specifier: ^10.0.0 + specifier: 10.0.0 version: 10.0.0 winston: - specifier: 3.13.1 - version: 3.13.1 + specifier: 3.14.1 + version: 3.14.1 devDependencies: '@biomejs/biome': specifier: 1.8.3 @@ -137,33 +128,33 @@ importers: version: 24.0.0(typescript@5.5.4) tsup: specifier: 8.2.4 - version: 8.2.4(jiti@1.21.6)(postcss@8.4.39)(tsx@4.16.2)(typescript@5.5.4) + version: 8.2.4(jiti@1.21.6)(postcss@8.4.41)(tsx@4.16.2)(typescript@5.5.4) typescript: specifier: 5.5.4 version: 5.5.4 vite: - specifier: 5.3.5 - version: 5.3.5(@types/node@20.14.14) + specifier: 5.4.0 + version: 5.4.0(@types/node@20.14.14) vitest: specifier: 2.0.5 version: 2.0.5(@types/node@20.14.14)(@vitest/ui@2.0.5) packages: - '@ai-sdk/anthropic@0.0.35': - resolution: {integrity: sha512-bCyonZyMQZdYTfSazLFRzQ2teuVc8icejbG6JhIZJvoQpcA5zzmruLD335vAf1hhGAFrHjrLNB+u4vTL52pfbQ==} + '@ai-sdk/anthropic@0.0.39': + resolution: {integrity: sha512-Ouku41O9ebyRi0EUW7pB8+lk4sI74SfJKydzK7FjynhNmCSvi42+U4WPlEjP64NluXUzpkYLvBa6BAd36VY4/g==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 - '@ai-sdk/openai@0.0.40': - resolution: {integrity: sha512-9Iq1UaBHA5ZzNv6j3govuKGXrbrjuWvZIgWNJv4xzXlDMHu9P9hnqlBr/Aiay54WwCuTVNhTzAUTfFgnTs2kbQ==} + '@ai-sdk/openai@0.0.44': + resolution: {integrity: sha512-fIUypicSnuWIOdoRG+dgzAnZDWQrhzqyUvQaLC8hQB6UDAUXkn8sIPdsqKEV35vs3mNXDz/KZ4EEeLxSVGrzMg==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 - '@ai-sdk/provider-utils@1.0.5': - resolution: {integrity: sha512-XfOawxk95X3S43arn2iQIFyWGMi0DTxsf9ETc6t7bh91RPWOOPYN1tsmS5MTKD33OGJeaDQ/gnVRzXUCRBrckQ==} + '@ai-sdk/provider-utils@1.0.7': + resolution: {integrity: sha512-xIDpinTnuInH16wBgKAVdN6pmrpjlJF+5f+f/8ahYMQlYzpe4Vj1qw8OL+rUGdCePcRSrFmjk8G/wdX5+J8dIw==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -171,12 +162,25 @@ packages: zod: optional: true - '@ai-sdk/provider@0.0.14': - resolution: {integrity: sha512-gaQ5Y033nro9iX1YUjEDFDRhmMcEiCk56LJdIUbX5ozEiCNCfpiBpEqrjSp/Gp5RzBS2W0BVxfG7UGW6Ezcrzg==} + '@ai-sdk/provider-utils@1.0.9': + resolution: {integrity: sha512-yfdanjUiCJbtGoRGXrcrmXn0pTyDfRIeY6ozDG96D66f2wupZaZvAgKptUa3zDYXtUCQQvcNJ+tipBBfQD/UYA==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@ai-sdk/provider@0.0.15': + resolution: {integrity: sha512-phX/YdwKd8q8/uZ7MsUytcHuN5KvT+wgM+y78eu6E+VyFE3GRwelctBFnaaA96uRL6xnKNmb0e7e+2fDOYuBoA==} + engines: {node: '>=18'} + + '@ai-sdk/provider@0.0.17': + resolution: {integrity: sha512-f9j+P5yYRkqKFHxvWae5FI0j6nqROPCoPnMkpc2hc2vC7vKjqzrxBJucD8rpSaUjqiBnY/QuRJ0QeV717Uz5tg==} engines: {node: '>=18'} - '@ai-sdk/react@0.0.36': - resolution: {integrity: sha512-LAxFLtHKN1BajTNP8YzyVIwXn45LSunmvm2Svrfq5oPOyJ2gUEjtaONnbme4mwRXJ1kk6b63SLrgOIXbz6XF/g==} + '@ai-sdk/react@0.0.40': + resolution: {integrity: sha512-irljzw5m9q2kz3g4Y59fbeHI7o29DFmPTPIKQNK+XrHcvYH1sDuj4rlOnQUQl6vpahnrU7fLO6FzVIYdwZv+0w==} engines: {node: '>=18'} peerDependencies: react: ^18 || ^19 @@ -187,8 +191,8 @@ packages: zod: optional: true - '@ai-sdk/solid@0.0.27': - resolution: {integrity: sha512-uEvlT7MBkRRZxk7teDgtrGe7G3U9tspgSJUvupdOE2d0a4vLlHrMqHb07ao97/Xo1aVHh7oBF9XIgRzKnFtbIQ==} + '@ai-sdk/solid@0.0.31': + resolution: {integrity: sha512-VYsrTCuNqAe8DzgPCyCqJl6MNdItnjjGMioxi2Pimns/3qet1bOw2LA0yUe5tTAoSpYaCAjGZZB4x8WC762asA==} engines: {node: '>=18'} peerDependencies: solid-js: ^1.7.7 @@ -196,8 +200,8 @@ packages: solid-js: optional: true - '@ai-sdk/svelte@0.0.29': - resolution: {integrity: sha512-7vrh61wXPVfy19nS4CqyAC3UWjsOgj/b94PCccVTGFoqbmVSa0VptXPYoFfgPTP/W71v7TjXqeq1ygLc4noTZw==} + '@ai-sdk/svelte@0.0.33': + resolution: {integrity: sha512-/QksvqVEv9fcq39nJfu4QqqeXeFX19pqkGUlVXBNhMQ6QQXhYsUasfJodIWy1DBsKDDV6dROJM0fHku5v+hrdw==} engines: {node: '>=18'} peerDependencies: svelte: ^3.0.0 || ^4.0.0 @@ -205,8 +209,8 @@ packages: svelte: optional: true - '@ai-sdk/ui-utils@0.0.24': - resolution: {integrity: sha512-NBhhICWJ5vAkN4BJnP/MyT+fOB6rTlGlsKGQwvjlhNxdrY1/jXqe2ikNkCbCSEz20IDk82bmg2JJBM96g1O3Ig==} + '@ai-sdk/ui-utils@0.0.28': + resolution: {integrity: sha512-yc+0EgC/Gz36ltoHsiBmuEv7iPjV85ihuj8W1vSqYe3+CblGYHG9h8MC5RdIl51GtrOtnH9aqnDjkX5Qy6Q9KQ==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -214,8 +218,8 @@ packages: zod: optional: true - '@ai-sdk/vue@0.0.28': - resolution: {integrity: sha512-ZnDjkkUH/9xoXqJEmyrG9Z8z7DKBnp2uyCd9ZVI8QSqKOaP0jOwhv8THUXlIqDEF+ULLGMWm0XeY/L7i3CMYTA==} + '@ai-sdk/vue@0.0.32': + resolution: {integrity: sha512-wEiH6J6VbuGcliA44UkQJM/JuSP1IwuFiovzPQ/bS0Gb/nJKuDQ8K2ru1JvdU7pEQFqmpTm7z+2R0Sduz0eKpA==} engines: {node: '>=18'} peerDependencies: vue: ^3.3.4 @@ -604,24 +608,24 @@ packages: cpu: [x64] os: [win32] - '@inquirer/checkbox@2.4.6': - resolution: {integrity: sha512-PvTeflvpyZMknHBVh9g9GPaffO/zyHcLk2i2HQN7q79SN1e0Tq2orAVzLAaZR1E5YDAdOB94znJurxzY/0HbFg==} + '@inquirer/checkbox@2.4.7': + resolution: {integrity: sha512-5YwCySyV1UEgqzz34gNsC38eKxRBtlRDpJLlKcRtTjlYA/yDKuc1rfw+hjw+2WJxbAZtaDPsRl5Zk7J14SBoBw==} engines: {node: '>=18'} - '@inquirer/confirm@3.1.21': - resolution: {integrity: sha512-v4O/jX5b6nm7Kxf9Gn/pjIz8RzGp1e8paFTl2GuMGL2OIWcaR9fx1HhkB8CnHZrGo3J7scLwSsgTK1fG8olxZA==} + '@inquirer/confirm@3.1.22': + resolution: {integrity: sha512-gsAKIOWBm2Q87CDfs9fEo7wJT3fwWIJfnDGMn9Qy74gBnNFOACDNfhUzovubbJjWnKLGBln7/NcSmZwj5DuEXg==} engines: {node: '>=18'} - '@inquirer/core@9.0.9': - resolution: {integrity: sha512-mvQmOz1hf5dtvY+bpVK22YiwLxn5arEhykSt1IWT5GS7ojgqKLSE9P8WXI4fPimtC0ggmnf0bVbKtERlIZkV0g==} + '@inquirer/core@9.0.10': + resolution: {integrity: sha512-TdESOKSVwf6+YWDz8GhS6nKscwzkIyakEzCLJ5Vh6O3Co2ClhCJ0A4MG909MUWfaWdpJm7DE45ii51/2Kat9tA==} engines: {node: '>=18'} - '@inquirer/editor@2.1.21': - resolution: {integrity: sha512-p5JYfAmEA6nqqDVCX0Cuu6EACA6/qejVBVataMew29mld3mtBWXy1g29Co86UMDQIiHA4HpOkH0hQGHlOvbGSw==} + '@inquirer/editor@2.1.22': + resolution: {integrity: sha512-K1QwTu7GCK+nKOVRBp5HY9jt3DXOfPGPr6WRDrPImkcJRelG9UTx2cAtK1liXmibRrzJlTWOwqgWT3k2XnS62w==} engines: {node: '>=18'} - '@inquirer/expand@2.1.21': - resolution: {integrity: sha512-SxoD3mM2UwS/ovRixXic9Aav84K9+zDXD54stIGxbNZ7AryJHtudQteXw73kFTlsZCH9AhHC1TmMyakpRiAhGw==} + '@inquirer/expand@2.1.22': + resolution: {integrity: sha512-wTZOBkzH+ItPuZ3ZPa9lynBsdMp6kQ9zbjVPYEtSBG7UulGjg2kQiAnUjgyG4SlntpTce5bOmXAPvE4sguXjpA==} engines: {node: '>=18'} '@inquirer/figures@1.0.4': @@ -632,32 +636,32 @@ packages: resolution: {integrity: sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==} engines: {node: '>=18'} - '@inquirer/input@2.2.8': - resolution: {integrity: sha512-DoLuc+DIJVZiDIn01hUQrxpPHF7MuE1bGfhxVfPWQDVFIqCoFQEmiUqMLx7zv4/pFArykY9j+i3uLUIOWqk+xg==} + '@inquirer/input@2.2.9': + resolution: {integrity: sha512-7Z6N+uzkWM7+xsE+3rJdhdG/+mQgejOVqspoW+w0AbSZnL6nq5tGMEVASaYVWbkoSzecABWwmludO2evU3d31g==} engines: {node: '>=18'} - '@inquirer/number@1.0.9': - resolution: {integrity: sha512-F5JqBCPnJTlLlZavRL15jGAtCXZGQiT64IMe2iOtcVIHQYYWecs5FpyqfkIDqvuOCyd4XgWPVmjeW+FssGEwFw==} + '@inquirer/number@1.0.10': + resolution: {integrity: sha512-kWTxRF8zHjQOn2TJs+XttLioBih6bdc5CcosXIzZsrTY383PXI35DuhIllZKu7CdXFi2rz2BWPN9l0dPsvrQOA==} engines: {node: '>=18'} - '@inquirer/password@2.1.21': - resolution: {integrity: sha512-kaz2jtA4xp3Y4J+weEs/gTppEBRjY82pIAWz1ycU23f+Blrv8enK2d58H4sv2dvzHtsOAcRE+rF2OXkdseQuTQ==} + '@inquirer/password@2.1.22': + resolution: {integrity: sha512-5Fxt1L9vh3rAKqjYwqsjU4DZsEvY/2Gll+QkqR4yEpy6wvzLxdSgFhUcxfDAOtO4BEoTreWoznC0phagwLU5Kw==} engines: {node: '>=18'} - '@inquirer/prompts@5.3.7': - resolution: {integrity: sha512-rGXU6k1Vcf1Jn3tcMTKfxCNTkWhwS9moOCTGerWG1fLtjv94/ug+ZLuqp5tq5MBjSuxFIaFfNFSD8mQn24OnIw==} + '@inquirer/prompts@5.3.8': + resolution: {integrity: sha512-b2BudQY/Si4Y2a0PdZZL6BeJtl8llgeZa7U2j47aaJSCeAl1e4UI7y8a9bSkO3o/ZbZrgT5muy/34JbsjfIWxA==} engines: {node: '>=18'} - '@inquirer/rawlist@2.2.3': - resolution: {integrity: sha512-qEqDLgCJ5jIJVAo1BpJBmqJunX6HDlhbQFMsufMH2/v3T4IeNCXTGgDG0xu7qwaPfw92c1VMP64BSQJYYvKoPA==} + '@inquirer/rawlist@2.2.4': + resolution: {integrity: sha512-pb6w9pWrm7EfnYDgQObOurh2d2YH07+eDo3xQBsNAM2GRhliz6wFXGi1thKQ4bN6B0xDd6C3tBsjdr3obsCl3Q==} engines: {node: '>=18'} - '@inquirer/search@1.0.6': - resolution: {integrity: sha512-dZ2zOsIHPo0NgUVfvYuC6aMqAq3mcGn/XPrMXjlQhoNtsN8/pR5BmavqSmlgQo9ZY25VXF3qohWX/JzBYxHypA==} + '@inquirer/search@1.0.7': + resolution: {integrity: sha512-p1wpV+3gd1eST/o5N3yQpYEdFNCzSP0Klrl+5bfD3cTTz8BGG6nf4Z07aBW0xjlKIj1Rp0y3x/X4cZYi6TfcLw==} engines: {node: '>=18'} - '@inquirer/select@2.4.6': - resolution: {integrity: sha512-fjcZQGyIviUBQ0Msoyf92vCmmN7Xv99vzPoybhLGBCM4cCz+l/U3p2++F7/xDJwvH70YHcvWrk8aN7STmrHMsQ==} + '@inquirer/select@2.4.7': + resolution: {integrity: sha512-JH7XqPEkBpNWp3gPCqWqY8ECbyMoFcCZANlL6pV9hf59qK6dGmkOlx1ydyhY+KZ0c5X74+W6Mtp+nm2QX0/MAQ==} engines: {node: '>=18'} '@inquirer/type@1.5.2': @@ -944,9 +948,6 @@ packages: '@types/diff-match-patch@1.0.36': resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==} - '@types/diff@5.2.1': - resolution: {integrity: sha512-uxpcuwWJGhe2AR1g8hD9F5OYGCqjqWnBUQFD8gMZsDbv8oPHzxJF6iMO6n8Tk0AdzlxoaaoQhOYlIg/PukVU8g==} - '@types/estree@1.0.5': resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} @@ -1080,8 +1081,8 @@ packages: resolution: {integrity: sha512-gOsf2YwSlleG6IjRYG2A7k0HmBMEo6qVNk9Bp/EaLgAJT5ngH6PXbqa4ItvnEwCm/velL5jAnQgsHsWnjhGmvw==} engines: {node: '>=18'} - ai@3.3.0: - resolution: {integrity: sha512-ndW4G9jw8ImIsTWK2iderOWMVn4H3B6u+KHlZ7hJEvFBdBYTFQ62qTw10AmHsQefjwHRC/2evr9qf79EkSwo9Q==} + ai@3.3.4: + resolution: {integrity: sha512-yb9ONWAnTq77J+O/fLtd+yvcTgasdN79k+U0IhBDJ6ssEc+HZeFldjqkSgr1jtKSc8rLZOu0GiVitwyAtwofNQ==} engines: {node: '>=18'} peerDependencies: openai: ^4.42.0 @@ -1422,10 +1423,6 @@ packages: diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} - diff@5.2.0: - resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} - engines: {node: '>=0.3.1'} - dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -2168,10 +2165,6 @@ packages: node-addon-api@3.2.1: resolution: {integrity: sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==} - node-diff3@3.1.2: - resolution: {integrity: sha512-wUd9TWy059I8mZdH6G3LPNlAEfxDvXtn/RcyFrbqL3v34WlDxn+Mh4HDhOwWuaMk/ROVepe5tTpnGHbve6Db2g==} - engines: {node: '>14'} - node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2291,8 +2284,8 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} - ollama-ai-provider@0.11.0: - resolution: {integrity: sha512-zobsS8pZJ7IUfqQF+QiqWBG92ocm4fHpV+iXUCXoFtNMSN65Nr3v0TyW2sTooku3BVm9EmOBeMenKSOIv7h5Sw==} + ollama-ai-provider@0.12.0: + resolution: {integrity: sha512-m9/CSY4ggCM7qMJ2oUYhOxmG2zu4hzp5Dot3qLLn0Q6Tq/0AqF8ZUs82A+YWnu358GfvMqVthimMXt0poeNbJg==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -2472,6 +2465,10 @@ packages: resolution: {integrity: sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==} engines: {node: ^10 || ^12 || >=14} + postcss@8.4.41: + resolution: {integrity: sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==} + engines: {node: ^10 || ^12 || >=14} + pretty-ms@9.0.0: resolution: {integrity: sha512-E9e9HJ9R9NasGOgPaPE8VMeiPKAyWR5jcFpNnwIejslIhWqdqOrb2wShBsncMPUb+BcCd2OPYfh7p2W6oemTng==} engines: {node: '>=18'} @@ -2969,8 +2966,8 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true - vite@5.3.5: - resolution: {integrity: sha512-MdjglKR6AQXQb9JGiS7Rc2wC6uMjcm7Go/NHNO63EwiJXfuk9PgqiP/n5IDJCziMkfw9n4Ubp7lttNwz+8ZVKA==} + vite@5.4.0: + resolution: {integrity: sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true peerDependencies: @@ -2978,6 +2975,7 @@ packages: less: '*' lightningcss: ^1.21.0 sass: '*' + sass-embedded: '*' stylus: '*' sugarss: '*' terser: ^5.4.0 @@ -2990,6 +2988,8 @@ packages: optional: true sass: optional: true + sass-embedded: + optional: true stylus: optional: true sugarss: @@ -3063,8 +3063,8 @@ packages: resolution: {integrity: sha512-wQCXXVgfv/wUPOfb2x0ruxzwkcZfxcktz6JIMUaPLmcNhO4bZTwA/WtDWK74xV3F2dKu8YadrFv0qhwYjVEwhA==} engines: {node: '>= 12.0.0'} - winston@3.13.1: - resolution: {integrity: sha512-SvZit7VFNvXRzbqGHsv5KSmgbEYR5EiQfDAL9gxYkRqa934Hnk++zze0wANKtMHcy/gI4W/3xmSDwlhf865WGw==} + winston@3.14.1: + resolution: {integrity: sha512-CJi4Il/msz8HkdDfXOMu+r5Au/oyEjFiOZzbX2d23hRLY0narGjqfE5lFlrT5hfYJhPtM8b85/GNFsxIML/RVA==} engines: {node: '>= 12.0.0'} wordwrap@1.0.0: @@ -3124,69 +3124,82 @@ packages: snapshots: - '@ai-sdk/anthropic@0.0.35(zod@3.23.8)': + '@ai-sdk/anthropic@0.0.39(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 0.0.17 + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + zod: 3.23.8 + + '@ai-sdk/openai@0.0.44(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 0.0.14 - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider': 0.0.17 + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) zod: 3.23.8 - '@ai-sdk/openai@0.0.40(zod@3.23.8)': + '@ai-sdk/provider-utils@1.0.7(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 0.0.14 - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider': 0.0.15 + eventsource-parser: 1.1.2 + nanoid: 3.3.6 + secure-json-parse: 2.7.0 + optionalDependencies: zod: 3.23.8 - '@ai-sdk/provider-utils@1.0.5(zod@3.23.8)': + '@ai-sdk/provider-utils@1.0.9(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 0.0.14 + '@ai-sdk/provider': 0.0.17 eventsource-parser: 1.1.2 nanoid: 3.3.6 secure-json-parse: 2.7.0 optionalDependencies: zod: 3.23.8 - '@ai-sdk/provider@0.0.14': + '@ai-sdk/provider@0.0.15': dependencies: json-schema: 0.4.0 - '@ai-sdk/react@0.0.36(react@18.3.1)(zod@3.23.8)': + '@ai-sdk/provider@0.0.17': dependencies: - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) - '@ai-sdk/ui-utils': 0.0.24(zod@3.23.8) + json-schema: 0.4.0 + + '@ai-sdk/react@0.0.40(react@18.3.1)(zod@3.23.8)': + dependencies: + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.28(zod@3.23.8) swr: 2.2.5(react@18.3.1) optionalDependencies: react: 18.3.1 zod: 3.23.8 - '@ai-sdk/solid@0.0.27(zod@3.23.8)': + '@ai-sdk/solid@0.0.31(zod@3.23.8)': dependencies: - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) - '@ai-sdk/ui-utils': 0.0.24(zod@3.23.8) + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.28(zod@3.23.8) transitivePeerDependencies: - zod - '@ai-sdk/svelte@0.0.29(svelte@4.2.18)(zod@3.23.8)': + '@ai-sdk/svelte@0.0.33(svelte@4.2.18)(zod@3.23.8)': dependencies: - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) - '@ai-sdk/ui-utils': 0.0.24(zod@3.23.8) + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.28(zod@3.23.8) sswr: 2.1.0(svelte@4.2.18) optionalDependencies: svelte: 4.2.18 transitivePeerDependencies: - zod - '@ai-sdk/ui-utils@0.0.24(zod@3.23.8)': + '@ai-sdk/ui-utils@0.0.28(zod@3.23.8)': dependencies: - '@ai-sdk/provider': 0.0.14 - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider': 0.0.17 + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) secure-json-parse: 2.7.0 optionalDependencies: zod: 3.23.8 - '@ai-sdk/vue@0.0.28(vue@3.4.34(typescript@5.5.4))(zod@3.23.8)': + '@ai-sdk/vue@0.0.32(vue@3.4.34(typescript@5.5.4))(zod@3.23.8)': dependencies: - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) - '@ai-sdk/ui-utils': 0.0.24(zod@3.23.8) + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.28(zod@3.23.8) swrv: 1.0.4(vue@3.4.34(typescript@5.5.4)) optionalDependencies: vue: 3.4.34(typescript@5.5.4) @@ -3425,20 +3438,20 @@ snapshots: '@esbuild/win32-x64@0.23.0': optional: true - '@inquirer/checkbox@2.4.6': + '@inquirer/checkbox@2.4.7': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/figures': 1.0.5 '@inquirer/type': 1.5.2 ansi-escapes: 4.3.2 yoctocolors-cjs: 2.1.2 - '@inquirer/confirm@3.1.21': + '@inquirer/confirm@3.1.22': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 - '@inquirer/core@9.0.9': + '@inquirer/core@9.0.10': dependencies: '@inquirer/figures': 1.0.5 '@inquirer/type': 1.5.2 @@ -3454,15 +3467,15 @@ snapshots: wrap-ansi: 6.2.0 yoctocolors-cjs: 2.1.2 - '@inquirer/editor@2.1.21': + '@inquirer/editor@2.1.22': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 external-editor: 3.1.0 - '@inquirer/expand@2.1.21': + '@inquirer/expand@2.1.22': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 yoctocolors-cjs: 2.1.2 @@ -3470,51 +3483,51 @@ snapshots: '@inquirer/figures@1.0.5': {} - '@inquirer/input@2.2.8': + '@inquirer/input@2.2.9': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 - '@inquirer/number@1.0.9': + '@inquirer/number@1.0.10': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 - '@inquirer/password@2.1.21': + '@inquirer/password@2.1.22': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 ansi-escapes: 4.3.2 - '@inquirer/prompts@5.3.7': + '@inquirer/prompts@5.3.8': dependencies: - '@inquirer/checkbox': 2.4.6 - '@inquirer/confirm': 3.1.21 - '@inquirer/editor': 2.1.21 - '@inquirer/expand': 2.1.21 - '@inquirer/input': 2.2.8 - '@inquirer/number': 1.0.9 - '@inquirer/password': 2.1.21 - '@inquirer/rawlist': 2.2.3 - '@inquirer/search': 1.0.6 - '@inquirer/select': 2.4.6 + '@inquirer/checkbox': 2.4.7 + '@inquirer/confirm': 3.1.22 + '@inquirer/editor': 2.1.22 + '@inquirer/expand': 2.1.22 + '@inquirer/input': 2.2.9 + '@inquirer/number': 1.0.10 + '@inquirer/password': 2.1.22 + '@inquirer/rawlist': 2.2.4 + '@inquirer/search': 1.0.7 + '@inquirer/select': 2.4.7 - '@inquirer/rawlist@2.2.3': + '@inquirer/rawlist@2.2.4': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/type': 1.5.2 yoctocolors-cjs: 2.1.2 - '@inquirer/search@1.0.6': + '@inquirer/search@1.0.7': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/figures': 1.0.5 '@inquirer/type': 1.5.2 yoctocolors-cjs: 2.1.2 - '@inquirer/select@2.4.6': + '@inquirer/select@2.4.7': dependencies: - '@inquirer/core': 9.0.9 + '@inquirer/core': 9.0.10 '@inquirer/figures': 1.0.5 '@inquirer/type': 1.5.2 ansi-escapes: 4.3.2 @@ -3824,8 +3837,6 @@ snapshots: '@types/diff-match-patch@1.0.36': {} - '@types/diff@5.2.1': {} - '@types/estree@1.0.5': {} '@types/fs-extra@11.0.4': @@ -4025,15 +4036,15 @@ snapshots: clean-stack: 5.2.0 indent-string: 5.0.0 - ai@3.3.0(react@18.3.1)(sswr@2.1.0(svelte@4.2.18))(svelte@4.2.18)(vue@3.4.34(typescript@5.5.4))(zod@3.23.8): + ai@3.3.4(react@18.3.1)(sswr@2.1.0(svelte@4.2.18))(svelte@4.2.18)(vue@3.4.34(typescript@5.5.4))(zod@3.23.8): dependencies: - '@ai-sdk/provider': 0.0.14 - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) - '@ai-sdk/react': 0.0.36(react@18.3.1)(zod@3.23.8) - '@ai-sdk/solid': 0.0.27(zod@3.23.8) - '@ai-sdk/svelte': 0.0.29(svelte@4.2.18)(zod@3.23.8) - '@ai-sdk/ui-utils': 0.0.24(zod@3.23.8) - '@ai-sdk/vue': 0.0.28(vue@3.4.34(typescript@5.5.4))(zod@3.23.8) + '@ai-sdk/provider': 0.0.17 + '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) + '@ai-sdk/react': 0.0.40(react@18.3.1)(zod@3.23.8) + '@ai-sdk/solid': 0.0.31(zod@3.23.8) + '@ai-sdk/svelte': 0.0.33(svelte@4.2.18)(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.28(zod@3.23.8) + '@ai-sdk/vue': 0.0.32(vue@3.4.34(typescript@5.5.4))(zod@3.23.8) '@opentelemetry/api': 1.9.0 eventsource-parser: 1.1.2 json-schema: 0.4.0 @@ -4360,8 +4371,6 @@ snapshots: diff-match-patch@1.0.5: {} - diff@5.2.0: {} - dir-glob@3.0.1: dependencies: path-type: 4.0.0 @@ -5118,8 +5127,6 @@ snapshots: node-addon-api@3.2.1: optional: true - node-diff3@3.1.2: {} - node-domexception@1.0.0: {} node-emoji@2.1.3: @@ -5158,10 +5165,10 @@ snapshots: object-assign@4.1.1: {} - ollama-ai-provider@0.11.0(zod@3.23.8): + ollama-ai-provider@0.12.0(zod@3.23.8): dependencies: - '@ai-sdk/provider': 0.0.14 - '@ai-sdk/provider-utils': 1.0.5(zod@3.23.8) + '@ai-sdk/provider': 0.0.15 + '@ai-sdk/provider-utils': 1.0.7(zod@3.23.8) partial-json: 0.1.7 optionalDependencies: zod: 3.23.8 @@ -5306,12 +5313,12 @@ snapshots: find-up: 2.1.0 load-json-file: 4.0.0 - postcss-load-config@6.0.1(jiti@1.21.6)(postcss@8.4.39)(tsx@4.16.2): + postcss-load-config@6.0.1(jiti@1.21.6)(postcss@8.4.41)(tsx@4.16.2): dependencies: lilconfig: 3.1.2 optionalDependencies: jiti: 1.21.6 - postcss: 8.4.39 + postcss: 8.4.41 tsx: 4.16.2 postcss@8.4.39: @@ -5320,6 +5327,12 @@ snapshots: picocolors: 1.0.1 source-map-js: 1.2.0 + postcss@8.4.41: + dependencies: + nanoid: 3.3.7 + picocolors: 1.0.1 + source-map-js: 1.2.0 + pretty-ms@9.0.0: dependencies: parse-ms: 4.0.0 @@ -5760,7 +5773,7 @@ snapshots: tslib@2.6.3: {} - tsup@8.2.4(jiti@1.21.6)(postcss@8.4.39)(tsx@4.16.2)(typescript@5.5.4): + tsup@8.2.4(jiti@1.21.6)(postcss@8.4.41)(tsx@4.16.2)(typescript@5.5.4): dependencies: bundle-require: 5.0.0(esbuild@0.23.0) cac: 6.7.14 @@ -5772,14 +5785,14 @@ snapshots: globby: 11.1.0 joycon: 3.1.1 picocolors: 1.0.1 - postcss-load-config: 6.0.1(jiti@1.21.6)(postcss@8.4.39)(tsx@4.16.2) + postcss-load-config: 6.0.1(jiti@1.21.6)(postcss@8.4.41)(tsx@4.16.2) resolve-from: 5.0.0 rollup: 4.19.0 source-map: 0.8.0-beta.0 sucrase: 3.35.0 tree-kill: 1.2.2 optionalDependencies: - postcss: 8.4.39 + postcss: 8.4.41 typescript: 5.5.4 transitivePeerDependencies: - jiti @@ -5844,21 +5857,22 @@ snapshots: debug: 4.3.5 pathe: 1.1.2 tinyrainbow: 1.2.0 - vite: 5.3.5(@types/node@20.14.14) + vite: 5.4.0(@types/node@20.14.14) transitivePeerDependencies: - '@types/node' - less - lightningcss - sass + - sass-embedded - stylus - sugarss - supports-color - terser - vite@5.3.5(@types/node@20.14.14): + vite@5.4.0(@types/node@20.14.14): dependencies: esbuild: 0.21.5 - postcss: 8.4.39 + postcss: 8.4.41 rollup: 4.19.0 optionalDependencies: '@types/node': 20.14.14 @@ -5882,7 +5896,7 @@ snapshots: tinybench: 2.8.0 tinypool: 1.0.0 tinyrainbow: 1.2.0 - vite: 5.3.5(@types/node@20.14.14) + vite: 5.4.0(@types/node@20.14.14) vite-node: 2.0.5(@types/node@20.14.14) why-is-node-running: 2.3.0 optionalDependencies: @@ -5892,6 +5906,7 @@ snapshots: - less - lightningcss - sass + - sass-embedded - stylus - sugarss - supports-color @@ -5943,7 +5958,7 @@ snapshots: readable-stream: 3.6.2 triple-beam: 1.4.1 - winston@3.13.1: + winston@3.14.1: dependencies: '@colors/colors': 1.6.0 '@dabh/diagnostics': 2.0.3 diff --git a/src/ai/parsers/diff-parser.ts b/src/ai/parsers/diff-parser.ts deleted file mode 100644 index 82cf979..0000000 --- a/src/ai/parsers/diff-parser.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { parsePatch } from 'diff'; -import type { AIFileInfo } from '../../types'; - -export function parseDiffFiles(response: string): AIFileInfo[] { - const files: AIFileInfo[] = []; - - const fileRegex = - /[\s\S]*?(.*?)<\/file_path>[\s\S]*?(.*?)<\/file_status>[\s\S]*?([\s\S]*?)<\/file_content>(?:[\s\S]*?([\s\S]*?)<\/explanation>)?[\s\S]*?<\/file>/gs; - let match: RegExpExecArray | null; - - // biome-ignore lint/suspicious/noAssignInExpressions: - while ((match = fileRegex.exec(response)) !== null) { - const [, path, status, language, content, explanation] = match; - const fileInfo: AIFileInfo = { - path: path.trim(), - language: language.trim(), - status: status.trim() as 'new' | 'modified' | 'deleted', - explanation: explanation ? explanation.trim() : undefined, - }; - - if (status.trim() === 'modified') { - try { - const normalizedContent = content.trim().replace(/\r\n/g, '\n'); - const parsedDiff = parsePatch(normalizedContent); - - if (parsedDiff.length > 0 && parsedDiff[0].hunks.length > 0) { - const diff = parsedDiff[0]; - fileInfo.diff = { - oldFileName: diff.oldFileName || path, - newFileName: diff.newFileName || path, - hunks: diff.hunks.map((hunk) => ({ - oldStart: hunk.oldStart, - oldLines: hunk.oldLines, - newStart: hunk.newStart, - newLines: hunk.newLines, - lines: hunk.lines, - })), - }; - } else { - console.error(`No valid diff found for ${path}`); - } - } catch (error) { - console.error(`Error parsing diff for ${path}:`, error); - } - } else if (status.trim() === 'new') { - fileInfo.content = content.trim(); - } - files.push(fileInfo); - } - - // Handle deleted files - const deletedFileRegex = - /[\s\S]*?(.*?)<\/file_path>[\s\S]*?deleted<\/file_status>[\s\S]*?<\/file>/g; - let deletedMatch: RegExpExecArray | null; - - // biome-ignore lint/suspicious/noAssignInExpressions: - while ((deletedMatch = deletedFileRegex.exec(response)) !== null) { - const [, path] = deletedMatch; - files.push({ - path: path.trim(), - language: '', - status: 'deleted', - }); - } - - return files; -} From 865fc1d7a8d6d4f5e25b7e48e54ff1c30e832449 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 22:41:11 +0200 Subject: [PATCH 52/54] chore: remove last vestiges of old diff mode --- src/types/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/types/index.ts b/src/types/index.ts index 9f19534..2e242ea 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,3 @@ -import type { ParsedDiff } from 'diff'; - export interface GitHubIssue { number: number; title: string; @@ -97,7 +95,6 @@ export interface AIFileInfo { path: string; language: string; content?: string; - diff?: ParsedDiff; status: 'new' | 'modified' | 'deleted'; explanation?: string; } @@ -179,7 +176,6 @@ export interface AIFileInfo { path: string; language: string; content?: string; - diff?: ParsedDiff; changes?: AIFileChange[]; status: 'new' | 'modified' | 'deleted'; explanation?: string; From ea5a6713aae5c6d31a64fef31e7a0ea0c2105a6b Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 22:48:05 +0200 Subject: [PATCH 53/54] fix: test --- tests/unit/apply-changes.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/apply-changes.test.ts b/tests/unit/apply-changes.test.ts index 4f04221..7908619 100644 --- a/tests/unit/apply-changes.test.ts +++ b/tests/unit/apply-changes.test.ts @@ -1,3 +1,4 @@ +import os from 'node:os'; import path from 'node:path'; import fs from 'fs-extra'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -186,7 +187,7 @@ describe('applyChanges', () => { expect.stringContaining(path.join('deep', 'nested')), ); expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining(path.join('var', 'folders')), + expect.stringContaining(path.join(os.tmpdir())), 'console.log("Nested file");', ); }); From 904c3fa5ffa54cf1e29c7265f82500ae1162c118 Mon Sep 17 00:00:00 2001 From: Gordon Mickel Date: Thu, 8 Aug 2024 23:14:25 +0200 Subject: [PATCH 54/54] docs: update evaluation table --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 91de0c3..598db5f 100644 --- a/README.md +++ b/README.md @@ -203,12 +203,15 @@ While CodeWhisper supports a variety of providers and models, our current recomm This section is still under development. We are actively testing and evaluating models. -| Model | Provider | Recommendation | Editing Mode | Notes | -| ----------------- | --------- | -------------- | ------------ | ----------------------------------------------------------------------------------- | -| Claude-3.5-Sonnet | Anthropic | Highest | Diff | Generates exceptional quality plans and results | -| GPT-4o | OpenAI | Excellent | Diff | Produces high-quality plans and good results, long max output length (16384 tokens) | -| GPT-4o-mini | OpenAI | Strong | Whole | Good quality plans and results, long max output length (16384 tokens) | -| DeepSeek Coder | DeepSeek | Good | Diff | Good quality plans and results, long max output length (16384 tokens) | +| Model | Provider | Recommendation | Editing Mode | Plan Quality | Code Quality | Edit Precision | Notes | +| ----------------- | --------- | -------------- | ------------ | ------------ | ------------ | -------------- | ----------------------------------------------------------------------------------- | +| Claude-3.5-Sonnet | Anthropic | Highest | Diff | Excellent | Excellent | High | Generates exceptional quality plans and results | +| GPT-4o | OpenAI | Excellent | Diff | Very Good | Good | Medium | Produces high-quality plans and good results, long max output length (16384 tokens) | +| GPT-4o-mini | OpenAI | Strong | Diff | Good | Good | Medium | Good quality plans and results, long max output length (16384 tokens) | +| GPT-4o-mini | OpenAI | Strong | Whole\* | Good | Very Good | High | Improved code quality and precision in whole-file edit mode | +| DeepSeek Coder | DeepSeek | Good | Diff | Good | Good | Medium | Good quality plans and results, long max output length (16384 tokens) | + +\* Whole-file edit mode is generally more precise but may lead to issues with maximum output token length, potentially limiting the ability to process larger files or multiple files simultaneously. It can also result in incomplete outputs for very large files, with the model resorting to placeholders like "// other functions here" instead of providing full implementations. #### Experimental Support