diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d471cd..6885e67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,6 +76,13 @@ jobs: echo 'echo BBai has been started in your default web browser.' >> build/bbai_start.bat echo 'echo You can close this window.' >> build/bbai_start.bat + echo '@echo off' > build/bbai_stop.bat + echo 'echo Stopping BBai...' >> build/bbai_stop.bat + echo 'start "" bbai stop' >> build/bbai_stop.bat + echo 'echo.' >> build/bbai_stop.bat + echo 'echo BBai has been stopped.' >> build/bbai_stop.bat + echo 'echo You can close this window.' >> build/bbai_stop.bat + - name: Create MSI installer (Windows) if: matrix.target == 'x86_64-pc-windows-msvc' run: | @@ -88,6 +95,7 @@ jobs: cp build/bbai-api.exe installer_files/ cp build/bbai_init.bat installer_files/ cp build/bbai_start.bat installer_files/ + cp build/bbai_stop.bat installer_files/ # Create a WiX source file @" @@ -122,6 +130,9 @@ jobs: + + + diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf177b..c16d7ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## [0.0.17-beta] - 2024-09-28 + +### Changed + +- Refactored search_project tool to use native stream reader with buffer and native regex, rather than external grep + + ## [0.0.16-beta] - 2024-09-27 ### Changed diff --git a/api/deno.jsonc b/api/deno.jsonc index 1a2e5f2..c43afb4 100644 --- a/api/deno.jsonc +++ b/api/deno.jsonc @@ -1,6 +1,6 @@ { "name": "bbai-api", - "version": "0.0.16-beta", + "version": "0.0.17-beta", "exports": "./src/main.ts", "tasks": { "start": "deno run --allow-read --allow-write --allow-run --allow-net --allow-env src/main.ts", diff --git a/api/src/editor/projectEditor.ts b/api/src/editor/projectEditor.ts index a44b69c..4104069 100644 --- a/api/src/editor/projectEditor.ts +++ b/api/src/editor/projectEditor.ts @@ -1,6 +1,11 @@ import { join } from '@std/path'; -import { FILE_LISTING_TIERS, generateFileListing, isPathWithinProject } from '../utils/fileHandling.utils.ts'; +import { + existsWithinProject, + FILE_LISTING_TIERS, + generateFileListing, + isPathWithinProject, +} from '../utils/fileHandling.utils.ts'; import type LLMConversationInteraction from '../llms/interactions/conversationInteraction.ts'; import type { ProjectInfo as BaseProjectInfo } from '../llms/interactions/conversationInteraction.ts'; import type { FileMetadata } from 'shared/types.ts'; @@ -160,6 +165,7 @@ class ProjectEditor { } // prepareFilesForConversation is called by request_files tool and by add_file handler for user requests + // only existing files can be prepared and added, otherwise call rewrite_file tools with createIfMissing:true async prepareFilesForConversation( fileNames: string[], ): Promise< @@ -183,6 +189,9 @@ class ProjectEditor { if (!await isPathWithinProject(this.projectRoot, fileName)) { throw new Error(`Access denied: ${fileName} is outside the project directory`); } + if (!await existsWithinProject(this.projectRoot, fileName)) { + throw new Error(`Access denied: ${fileName} does not exist in the project directory`); + } const fullFilePath = join(this.projectRoot, fileName); diff --git a/api/src/llms/tools/formatters/searchProjectTool.browser.tsx b/api/src/llms/tools/formatters/searchProjectTool.browser.tsx index 0bd8a6d..a14fde8 100644 --- a/api/src/llms/tools/formatters/searchProjectTool.browser.tsx +++ b/api/src/llms/tools/formatters/searchProjectTool.browser.tsx @@ -5,45 +5,45 @@ import type { LLMToolInputSchema, LLMToolRunResultContent } from 'api/llms/llmTo import { getContentFromToolResult } from '../../../utils/llms.utils.ts'; export const formatToolUse = (toolInput: LLMToolInputSchema): JSX.Element => { - const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as { - content_pattern?: string; - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; + const { contentPattern, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } = toolInput as { + contentPattern?: string; + filePattern?: string; + dateAfter?: string; + dateBefore?: string; + sizeMin?: number; + sizeMax?: number; }; return (

Project Search Parameters:

- {content_pattern && ( + {contentPattern && (

- Content pattern: ${content_pattern} + Content pattern: ${contentPattern}

)} - {file_pattern && ( + {filePattern && (

- File pattern: ${file_pattern} + File pattern: ${filePattern}

)} - {date_after && ( + {dateAfter && (

- Modified after: ${date_after} + Modified after: ${dateAfter}

)} - {date_before && ( + {dateBefore && (

- Modified before: ${date_before} + Modified before: ${dateBefore}

)} - {size_min && ( + {sizeMin && (

- Minimum size: ${size_min.toString()} bytes + Minimum size: ${sizeMin.toString()} bytes

)} - {size_max && ( + {sizeMax && (

- Maximum size: ${size_max.toString()} bytes + Maximum size: ${sizeMax.toString()} bytes

)}
diff --git a/api/src/llms/tools/formatters/searchProjectTool.console.ts b/api/src/llms/tools/formatters/searchProjectTool.console.ts index db28aef..1ed9e69 100644 --- a/api/src/llms/tools/formatters/searchProjectTool.console.ts +++ b/api/src/llms/tools/formatters/searchProjectTool.console.ts @@ -5,22 +5,22 @@ import { stripIndents } from 'common-tags'; import { getContentFromToolResult } from '../../../utils/llms.utils.ts'; export const formatToolUse = (toolInput: LLMToolInputSchema): string => { - const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as { - content_pattern?: string; - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; + const { contentPattern, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } = toolInput as { + contentPattern?: string; + filePattern?: string; + dateAfter?: string; + dateBefore?: string; + sizeMin?: number; + sizeMax?: number; }; return stripIndents` ${colors.bold('Project Search Parameters:')} - ${content_pattern ? `${colors.cyan('Content pattern:')} ${content_pattern}` : ''} - ${file_pattern ? `${colors.cyan('File pattern:')} ${file_pattern}` : ''} - ${date_after ? `${colors.cyan('Modified after:')} ${date_after}` : ''} - ${date_before ? `${colors.cyan('Modified before:')} ${date_before}` : ''} - ${size_min ? `${colors.cyan('Minimum size:')} ${size_min.toString()} bytes` : ''} - ${size_max ? `${colors.cyan('Maximum size:')} ${size_max.toString()} bytes` : ''} + ${contentPattern ? `${colors.cyan('Content pattern:')} ${contentPattern}` : ''} + ${filePattern ? `${colors.cyan('File pattern:')} ${filePattern}` : ''} + ${dateAfter ? `${colors.cyan('Modified after:')} ${dateAfter}` : ''} + ${dateBefore ? `${colors.cyan('Modified before:')} ${dateBefore}` : ''} + ${sizeMin ? `${colors.cyan('Minimum size:')} ${sizeMin.toString()} bytes` : ''} + ${sizeMax ? `${colors.cyan('Maximum size:')} ${sizeMax.toString()} bytes` : ''} `.trim(); }; diff --git a/api/src/llms/tools/searchProjectTool.ts b/api/src/llms/tools/searchProjectTool.ts index 118f883..d6ca7ab 100644 --- a/api/src/llms/tools/searchProjectTool.ts +++ b/api/src/llms/tools/searchProjectTool.ts @@ -30,28 +30,34 @@ export default class LLMToolSearchProject extends LLMTool { return { type: 'object', properties: { - content_pattern: { + contentPattern: { type: 'string', description: String .raw`The search pattern for file contents (grep-compatible regular expression). Ensure to escape special regex characters with backslashes, e.g., "\.", "\?", "\*"`, }, - file_pattern: { + caseSensitive: { + type: 'boolean', + description: + 'Whether the `contentPattern` is a case sensitive regex. The default is true, to use case sensitive regex.', + default: true, + }, + filePattern: { type: 'string', description: 'File name pattern to limit the search to specific file types or names', }, - date_after: { + dateAfter: { type: 'string', description: 'Search for files modified after this date (YYYY-MM-DD format)', }, - date_before: { + dateBefore: { type: 'string', description: 'Search for files modified before this date (YYYY-MM-DD format)', }, - size_min: { + sizeMin: { type: 'number', description: 'Minimum file size in bytes', }, - size_max: { + sizeMax: { type: 'number', description: 'Maximum file size in bytes', }, @@ -73,49 +79,55 @@ export default class LLMToolSearchProject extends LLMTool { projectEditor: ProjectEditor, ): Promise { const { toolInput } = toolUse; - const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as { - content_pattern?: string; - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; - }; + const { contentPattern, caseSensitive = true, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } = + toolInput as { + contentPattern?: string; + caseSensitive?: boolean; + filePattern?: string; + dateAfter?: string; + dateBefore?: string; + sizeMin?: number; + sizeMax?: number; + }; + // caseSensitive controls the regex flag + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions#advanced_searching_with_flags try { let files: string[] = []; let errorMessage: string | null = null; let result; - if (content_pattern) { + if (contentPattern) { // searchContent or searchContentInFiles - result = await searchFilesContent(projectEditor.projectRoot, content_pattern, { - file_pattern, - date_after, - date_before, - size_min, - size_max, + result = await searchFilesContent(projectEditor.projectRoot, contentPattern, caseSensitive, { + filePattern, + dateAfter, + dateBefore, + sizeMin, + sizeMax, }); } else { // searchForFiles (metadata-only search) result = await searchFilesMetadata(projectEditor.projectRoot, { - file_pattern, - date_after, - date_before, - size_min, - size_max, + filePattern, + dateAfter, + dateBefore, + sizeMin, + sizeMax, }); } files = result.files; errorMessage = result.errorMessage; const searchCriteria = [ - content_pattern && `content pattern "${content_pattern}"`, - file_pattern && `file pattern "${file_pattern}"`, - date_after && `modified after ${date_after}`, - date_before && `modified before ${date_before}`, - size_min !== undefined && `minimum size ${size_min} bytes`, - size_max !== undefined && `maximum size ${size_max} bytes`, + contentPattern && `content pattern "${contentPattern}"`, + // only include case sensitivity details if content pattern was supplied + contentPattern && `${caseSensitive ? 'case-sensitive' : 'case-insensitive'}`, + filePattern && `file pattern "${filePattern}"`, + dateAfter && `modified after ${dateAfter}`, + dateBefore && `modified before ${dateBefore}`, + sizeMin !== undefined && `minimum size ${sizeMin} bytes`, + sizeMax !== undefined && `maximum size ${sizeMax} bytes`, ].filter(Boolean).join(', '); const toolResults = stripIndents` diff --git a/api/src/main.ts b/api/src/main.ts index 7741756..0c4f34b 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -19,9 +19,9 @@ const { environment, apiHostname, apiPort, apiUseTls } = fullConfig.api; // Parse command line arguments const args = parseArgs(Deno.args, { - string: ['log-file', 'port', 'hostname'], + string: ['log-file', 'port', 'hostname', 'use-tls'], boolean: ['help', 'version'], - alias: { h: 'help', V: 'version', v: 'version', p: 'port', H: 'hostname', t: 'useTls', l: 'log-file' }, + alias: { h: 'help', V: 'version', v: 'version', p: 'port', H: 'hostname', t: 'use-tls', l: 'log-file' }, }); if (args.help) { @@ -33,7 +33,7 @@ Options: -V, --version Show version information -H, --hostname Specify the hostname to run the API server (default: ${apiHostname}) -p, --port Specify the port to run the API server (default: ${apiPort}) - -t, --useTls Specify whether the API server should use TLS (default: ${apiUseTls}) + -t, --use-tls Specify whether the API server should use TLS (default: ${apiUseTls}) -l, --log-file Specify a log file to write output `); Deno.exit(0); @@ -50,8 +50,8 @@ if (apiLogFile) await apiFileLogger(apiLogFile); const customHostname = args.hostname ? args.hostname : apiHostname; const customPort: number = args.port ? parseInt(args.port, 10) : apiPort as number; -const customUseTls: boolean = typeof args.useTls !== 'undefined' - ? (args.useTls === 'true' ? true : false) +const customUseTls: boolean = typeof args['use-tls'] !== 'undefined' + ? (args['use-tls'] === 'true' ? true : false) : !!apiUseTls; //console.debug(`BBai API starting at ${customHostname}:${customPort}`); diff --git a/api/src/utils/fileHandling.utils.ts b/api/src/utils/fileHandling.utils.ts index 4b88f97..faa4ecc 100644 --- a/api/src/utils/fileHandling.utils.ts +++ b/api/src/utils/fileHandling.utils.ts @@ -1,4 +1,6 @@ import { join, normalize, relative, resolve } from '@std/path'; +import { TextLineStream } from '@std/streams/text-line-stream'; +import { LRUCache } from 'npm:lru-cache'; import { exists, walk } from '@std/fs'; import globToRegExp from 'npm:glob-to-regexp'; //import { globToRegExp } from '@std/path'; @@ -86,7 +88,10 @@ function isMatch(path: string, pattern: string): boolean { // Handle simple wildcard patterns if (pattern.includes('*') && !pattern.includes('**')) { - pattern = pattern.split('*').join('**'); + // we were just changing '*' to '**' - why was that needed (it wasn't working to match subdirectories) + // [TODO] add more tests to search_project test to check for more complex file patterns with deeply nested sub directories + // pattern = pattern.split('*').join('**'); + pattern = `**/${pattern}`; } // Handle bare filename (no path, no wildcards) @@ -141,6 +146,16 @@ export async function isPathWithinProject(projectRoot: string, filePath: string) } } +export async function existsWithinProject(projectRoot: string, filePath: string): Promise { + const normalizedProjectRoot = normalize(projectRoot); + const normalizedFilePath = normalize(filePath); + const absoluteFilePath = resolve(normalizedProjectRoot, normalizedFilePath); + + return await exists(absoluteFilePath); + // [TODO] Using isReadable is causing tests to fail - is it a real error or some other problem + //return await exists(absoluteFilePath, { isReadable: true }); +} + export async function readProjectFileContent(projectRoot: string, filePath: string): Promise { const fullFilePath = join(projectRoot, filePath); logger.info(`Reading contents of File ${fullFilePath}`); @@ -168,106 +183,381 @@ export async function updateFile(projectRoot: string, filePath: string, _content logger.info(`File ${filePath} updated in the project`); } +const searchCache = new LRUCache({ max: 100 }); + +interface SearchFileOptions { + filePattern?: string; + dateAfter?: string; + dateBefore?: string; + sizeMin?: number; + sizeMax?: number; +} + +const MAX_CONCURRENT = 20; // Adjust based on system capabilities + export async function searchFilesContent( projectRoot: string, contentPattern: string, - options?: { - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; - }, + caseSensitive: boolean, + searchFileOptions?: SearchFileOptions, ): Promise<{ files: string[]; errorMessage: string | null }> { + const cacheKey = `${projectRoot}:${contentPattern}:${caseSensitive}:${JSON.stringify(searchFileOptions)}`; + const cachedResult = searchCache.get(cacheKey); + if (cachedResult) { + logger.info(`Returning cached result for search: ${cacheKey}`); + return { files: cachedResult, errorMessage: null }; + } + const matchingFiles: string[] = []; + logger.info(`Starting file content search in ${projectRoot} with pattern: ${contentPattern}`); + + let regex: RegExp; try { - const excludeOptions = await getExcludeOptions(projectRoot); - // Escape special characters for grep - //const escapedPattern = contentPattern.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"); - // Escape backslash characters for grep - const escapedPattern = contentPattern.replace(/(\\)/g, '\$1'); - //logger.error(`Using escaped pattern in ${projectRoot}: `, JSON.stringify(escapedPattern)); - const grepCommand = ['-r', '-l', '-E', escapedPattern]; - - if (options?.file_pattern) { - grepCommand.push('--include', `./${options.file_pattern}`); - } + // We're only supporting 'g' and 'i' flags at present - there are a few more we can support if needed + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions#advanced_searching_with_flags + //const regexFlags = `${!caseSensitive ? 'i' : ''}${replaceAll ? 'g' : ''}`; + const regexFlags = `${!caseSensitive ? 'i' : ''}`; + regex = new RegExp(contentPattern, regexFlags); + } catch (error) { + logger.error(`Invalid regular expression: ${contentPattern}`); + return { files: [], errorMessage: `Invalid regular expression: ${error.message}` }; + } - // Add exclude options - for (const option of excludeOptions) { - grepCommand.push('--exclude-dir', option); - grepCommand.push('--exclude', option); - } - grepCommand.push('.'); - logger.error(`Search command in dir ${projectRoot}: grep ${grepCommand.join(' ')}`); - - const command = new Deno.Command('grep', { - args: grepCommand, - cwd: projectRoot, - stdout: 'piped', - stderr: 'piped', - }); - - const { code, stdout, stderr } = await command.output(); - const rawOutput = new TextDecoder().decode(stdout).trim(); - const rawError = new TextDecoder().decode(stderr).trim(); - - if (code === 0 || code === 1) { // grep returns 1 if no matches found, which is not an error for us - let files = rawOutput.split('\n').filter(Boolean); - - // Apply additional metadata filters - if (options) { - files = await filterFilesByMetadata(projectRoot, files, options); + const excludeOptions = await getExcludeOptions(projectRoot); + + try { + const filesToProcess = []; + for await (const entry of walk(projectRoot, { includeDirs: false })) { + const relativePath = relative(projectRoot, entry.path); + if (shouldExclude(relativePath, excludeOptions)) { + logger.debug(`Skipping excluded file: ${relativePath}`); + continue; + } + if (searchFileOptions?.filePattern && !isMatch(relativePath, searchFileOptions.filePattern)) { + logger.debug(`Skipping file not matching pattern: ${relativePath}`); + continue; } - return { files, errorMessage: null }; - } else { - return { files: [], errorMessage: rawError }; + filesToProcess.push({ path: entry.path, relativePath }); } + + const results = await Promise.all( + chunk(filesToProcess, MAX_CONCURRENT).map(async (batch) => + Promise.all( + batch.map(({ path, relativePath }) => processFile(path, regex, searchFileOptions, relativePath)), + ) + ), + ); + + matchingFiles.push(...results.flat().filter((result): result is string => result !== null)); + + logger.info(`File content search completed. Found ${matchingFiles.length} matching files.`); + searchCache.set(cacheKey, matchingFiles); + return { files: matchingFiles, errorMessage: null }; } catch (error) { logger.error(`Error in searchFilesContent: ${error.message}`); return { files: [], errorMessage: error.message }; } } -async function filterFilesByMetadata( - projectRoot: string, - files: string[], - options: { - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; - }, -): Promise { - const filteredFiles: string[] = []; +function chunk(array: T[], size: number): T[][] { + return Array.from({ length: Math.ceil(array.length / size) }, (_, i) => array.slice(i * size, i * size + size)); +} + +/* +async function processFileManualBuffer( + filePath: string, + regex: RegExp, + searchFileOptions: SearchFileOptions | undefined, + relativePath: string, +): Promise { + logger.debug(`Starting to process file: ${relativePath}`); + let file: Deno.FsFile | null = null; + try { + const stat = await Deno.stat(filePath); + + if (!passesMetadataFilters(stat, searchFileOptions)) { + logger.debug(`File ${relativePath} did not pass metadata filters`); + return null; + } + + file = await Deno.open(filePath); + logger.debug(`File opened successfully: ${relativePath}`); + + const decoder = new TextDecoder(); + const buffer = new Uint8Array(1024); // Adjust buffer size as needed + let leftover = ''; + + while (true) { + const bytesRead = await file.read(buffer); + if (bytesRead === null) break; // End of file + + const chunk = decoder.decode(buffer.subarray(0, bytesRead), { stream: true }); + const lines = (leftover + chunk).split('\n'); + leftover = lines.pop() || ''; + + for (const line of lines) { + if (regex.test(line)) { + logger.debug(`Match found in file: ${relativePath}`); + return relativePath; + } + } + } + + // Check the last line + if (leftover && regex.test(leftover)) { + logger.debug(`Match found in file: ${relativePath}`); + return relativePath; + } + + logger.debug(`No match found in file: ${relativePath}`); + return null; + } catch (error) { + logger.warn(`Error processing file ${filePath}: ${error.message}`); + return null; + } finally { + logger.debug(`Entering finally block for file: ${relativePath}`); + if (file) { + try { + file.close(); + logger.debug(`File closed successfully: ${relativePath}`); + } catch (closeError) { + logger.warn(`Error closing file ${filePath}: ${closeError.message}`); + } + } + logger.debug(`Exiting finally block for file: ${relativePath}`); + } +} - for (const file of files) { - const fullPath = join(projectRoot, file); - const stat = await Deno.stat(fullPath); +async function processFileStreamLines( + filePath: string, + regex: RegExp, + searchFileOptions: SearchFileOptions | undefined, + relativePath: string, +): Promise { + logger.debug(`Starting to process file: ${relativePath}`); + let file: Deno.FsFile | null = null; + let reader: ReadableStreamDefaultReader | null = null; + try { + const stat = await Deno.stat(filePath); + + if (!passesMetadataFilters(stat, searchFileOptions)) { + logger.debug(`File ${relativePath} did not pass metadata filters`); + return null; + } + + file = await Deno.open(filePath); + logger.debug(`File opened successfully: ${relativePath}`); + const lineStream = file.readable + .pipeThrough(new TextDecoderStream()) + .pipeThrough(new TextLineStream()); + + reader = lineStream.getReader(); + while (true) { + const { done, value: line } = await reader.read(); + if (done) { + logger.debug(`Finished reading file: ${relativePath}`); + break; + } + if (regex.test(line)) { + logger.debug(`Match found in file: ${relativePath}`); + return relativePath; + } + } + return null; + } catch (error) { + logger.warn(`Error processing file ${filePath}: ${error.message}`); + return null; + } finally { + logger.debug(`Entering finally block for file: ${relativePath}`); + if (reader) { + try { + await reader.cancel(); + logger.debug(`Reader cancelled for file: ${relativePath}`); + } catch (cancelError) { + logger.warn(`Error cancelling reader for ${filePath}: ${cancelError.message}`); + } + reader.releaseLock(); + logger.debug(`Reader lock released for file: ${relativePath}`); + } + if (file) { + try { + file.close(); + logger.debug(`File closed successfully: ${relativePath}`); + } catch (closeError) { + if (closeError instanceof Deno.errors.BadResource) { + logger.debug(`File was already closed: ${relativePath}`); + } else { + logger.warn(`Error closing file ${filePath}: ${closeError.message}`); + } + } + } + logger.debug(`Exiting finally block for file: ${relativePath}`); + } +} + +async function processFileStreamBuffer( + filePath: string, + regex: RegExp, + searchFileOptions: SearchFileOptions | undefined, + relativePath: string, +): Promise { + logger.debug(`Starting to process file: ${relativePath}`); + let file: Deno.FsFile | null = null; + let reader: ReadableStreamDefaultReader | null = null; + try { + const stat = await Deno.stat(filePath); + if (!passesMetadataFilters(stat, searchFileOptions)) { + return null; + } + + file = await Deno.open(filePath); + const textStream = file.readable + .pipeThrough(new TextDecoderStream()); + + reader = textStream.getReader(); + let buffer = ''; + const maxBufferSize = 1024 * 1024; // 1MB, adjust as needed + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += value; + + // Check for matches + if (regex.test(buffer)) { + return relativePath; + } + + // Trim buffer if it gets too large + if (buffer.length > maxBufferSize) { + buffer = buffer.slice(-maxBufferSize); + } + } + + // Final check on remaining buffer + if (regex.test(buffer)) { + return relativePath; + } + + return null; + } catch (error) { + logger.warn(`Error processing file ${filePath}: ${error.message}`); + return null; + } finally { + if (reader) { + try { + await reader.cancel(); + reader.releaseLock(); + } catch (cancelError) { + logger.warn(`Error cancelling reader for ${filePath}: ${cancelError.message}`); + } + } + if (file) { + try { + file.close(); + } catch (closeError) { + if (closeError instanceof Deno.errors.BadResource) { + logger.debug(`File was already closed: ${relativePath}`); + } else { + logger.warn(`Error closing file ${filePath}: ${closeError.message}`); + } + } + } + } +} + */ + +async function processFile( + filePath: string, + regex: RegExp, + searchFileOptions: SearchFileOptions | undefined, + relativePath: string, +): Promise { + logger.debug(`Starting to process file: ${relativePath}`); + let file: Deno.FsFile | null = null; + let reader: ReadableStreamDefaultReader | null = null; + try { + const stat = await Deno.stat(filePath); + if (!passesMetadataFilters(stat, searchFileOptions)) { + return null; + } + + file = await Deno.open(filePath); + const textStream = file.readable + .pipeThrough(new TextDecoderStream()); + + reader = textStream.getReader(); + let buffer = ''; + const maxBufferSize = 1024 * 1024; // 1MB, adjust as needed + const overlapSize = 1024; // Size of overlap between buffers, adjust based on expected pattern size - // Check date range - if (options.date_after && stat.mtime && stat.mtime < new Date(options.date_after)) continue; - if (options.date_before && stat.mtime && stat.mtime > new Date(options.date_before)) continue; + while (true) { + const { done, value } = await reader.read(); + if (done) break; - // Check file size - if (options.size_min !== undefined && stat.size < options.size_min) continue; - if (options.size_max !== undefined && stat.size > options.size_max) continue; + buffer += value; - filteredFiles.push(file); + // Check for matches + if (regex.test(buffer)) { + return relativePath; + } + + // Trim buffer if it gets too large, keeping overlap + if (buffer.length > maxBufferSize) { + buffer = buffer.slice(-maxBufferSize - overlapSize); + } + } + + // Final check on remaining buffer + if (regex.test(buffer)) { + return relativePath; + } + + return null; + } catch (error) { + logger.warn(`Error processing file ${filePath}: ${error.message}`); + return null; + } finally { + if (reader) { + try { + await reader.cancel(); + reader.releaseLock(); + } catch (cancelError) { + logger.warn(`Error cancelling reader for ${filePath}: ${cancelError.message}`); + } + } + if (file) { + try { + file.close(); + } catch (closeError) { + if (closeError instanceof Deno.errors.BadResource) { + logger.debug(`File was already closed: ${relativePath}`); + } else { + logger.warn(`Error closing file ${filePath}: ${closeError.message}`); + } + } + } } +} - return filteredFiles; +function passesMetadataFilters(stat: Deno.FileInfo, searchFileOptions: SearchFileOptions | undefined): boolean { + if (!searchFileOptions) return true; + if (searchFileOptions.dateAfter && stat.mtime && stat.mtime < new Date(searchFileOptions.dateAfter)) return false; + if (searchFileOptions.dateBefore && stat.mtime && stat.mtime > new Date(searchFileOptions.dateBefore)) return false; + if (searchFileOptions.sizeMin !== undefined && stat.size < searchFileOptions.sizeMin) return false; + if (searchFileOptions.sizeMax !== undefined && stat.size > searchFileOptions.sizeMax) return false; + return true; } export async function searchFilesMetadata( projectRoot: string, - options: { - file_pattern?: string; - date_after?: string; - date_before?: string; - size_min?: number; - size_max?: number; + searchFileOptions: { + filePattern?: string; + dateAfter?: string; + dateBefore?: string; + sizeMin?: number; + sizeMax?: number; }, ): Promise<{ files: string[]; errorMessage: string | null }> { try { @@ -281,37 +571,37 @@ export async function searchFilesMetadata( const stat = await Deno.stat(entry.path); // Check file pattern - if (options.file_pattern && !isMatch(relativePath, options.file_pattern)) continue; + if (searchFileOptions.filePattern && !isMatch(relativePath, searchFileOptions.filePattern)) continue; // Check date range if (!stat.mtime) { console.log(`File ${relativePath} has no modification time, excluding from results`); continue; } - if (options.date_after) { - const afterDate = new Date(options.date_after); + if (searchFileOptions.dateAfter) { + const afterDate = new Date(searchFileOptions.dateAfter); //if (stat.mtime < afterDate || stat.mtime > now) { if (stat.mtime < afterDate) { console.log( - `File ${relativePath} modified at ${stat.mtime.toISOString()} is outside the valid range (after ${options.date_after})`, + `File ${relativePath} modified at ${stat.mtime.toISOString()} is outside the valid range (after ${searchFileOptions.dateAfter})`, ); continue; } } - if (options.date_before) { - const beforeDate = new Date(options.date_before); + if (searchFileOptions.dateBefore) { + const beforeDate = new Date(searchFileOptions.dateBefore); //if (stat.mtime >= beforeDate || stat.mtime > now) { if (stat.mtime >= beforeDate) { console.log( - `File ${relativePath} modified at ${stat.mtime.toISOString()} is outside the valid range (before ${options.date_before})`, + `File ${relativePath} modified at ${stat.mtime.toISOString()} is outside the valid range (before ${searchFileOptions.dateBefore})`, ); continue; } } // Check file size - if (options.size_min !== undefined && stat.size < options.size_min) continue; - if (options.size_max !== undefined && stat.size > options.size_max) continue; + if (searchFileOptions.sizeMin !== undefined && stat.size < searchFileOptions.sizeMin) continue; + if (searchFileOptions.sizeMax !== undefined && stat.size > searchFileOptions.sizeMax) continue; console.log(`File ${relativePath} matches all criteria`); matchingFiles.push(relativePath); diff --git a/api/tests/lib/testSetup.ts b/api/tests/lib/testSetup.ts index f4fd530..a336748 100644 --- a/api/tests/lib/testSetup.ts +++ b/api/tests/lib/testSetup.ts @@ -17,9 +17,9 @@ export async function setupTestProject(): Promise { return testProjectRoot; } -export function cleanupTestProject(testProjectRoot: string) { +export async function cleanupTestProject(testProjectRoot: string) { try { - Deno.removeSync(testProjectRoot, { recursive: true }); + await Deno.remove(testProjectRoot, { recursive: true }); } catch (error) { console.error(`Failed to clean up test directory: ${error}`); } @@ -52,6 +52,6 @@ export async function withTestProject( try { return await testFn(testProjectRoot); } finally { - cleanupTestProject(testProjectRoot); + await cleanupTestProject(testProjectRoot); } } diff --git a/api/tests/t/fileHandling.test.ts b/api/tests/t/fileHandling.test.ts index 6be5602..dcb1f84 100644 --- a/api/tests/t/fileHandling.test.ts +++ b/api/tests/t/fileHandling.test.ts @@ -34,8 +34,8 @@ Deno.test({ await createTestFiles(testProjectRoot); const result = await searchFilesMetadata(testProjectRoot, { - date_after: '2024-01-01', - date_before: '2026-01-01', + dateAfter: '2024-01-01', + dateBefore: '2026-01-01', }); //console.log('Date-based search results:', result); assertEquals(result.files.length, 3); @@ -57,8 +57,8 @@ Deno.test({ await createTestFiles(testProjectRoot); const result = await searchFilesMetadata(testProjectRoot, { - size_min: 5000, - size_max: 15000, + sizeMin: 5000, + sizeMax: 15000, }); //console.log('Size-based search results:', result); assertEquals(result.files.length, 1); @@ -78,11 +78,11 @@ Deno.test({ await createTestFiles(testProjectRoot); const result = await searchFilesMetadata(testProjectRoot, { - file_pattern: '*.txt', - date_after: '2022-01-01', - date_before: '2024-01-01', - size_min: 1, - size_max: 1000, + filePattern: '*.txt', + dateAfter: '2022-01-01', + dateBefore: '2024-01-01', + sizeMin: 1, + sizeMax: 1000, }); console.log('Combined criteria search results:', result); assertEquals(result.files.length, 2); @@ -183,7 +183,7 @@ Deno.test({ Deno.writeTextFileSync(join(testProjectRoot, 'file2.md'), 'Hello, markdown!'); const result = await searchFilesContent(testProjectRoot, 'Hello', { - file_pattern: '*.md', + filePattern: '*.md', }); assertEquals(result.files, ['./file2.md']); assertEquals(result.errorMessage, null); diff --git a/api/tests/t/llms/tools/forgetFilesTool.test.ts b/api/tests/t/llms/tools/forgetFilesTool.test.ts index 14201f1..203b050 100644 --- a/api/tests/t/llms/tools/forgetFilesTool.test.ts +++ b/api/tests/t/llms/tools/forgetFilesTool.test.ts @@ -19,10 +19,12 @@ Deno.test({ await Deno.writeTextFile(join(testProjectRoot, 'file2.txt'), 'Content of file2'); const initialConversation = await projectEditor.initConversation('test-conversation-id'); initialConversation.addFileForMessage('file1.txt', { + type: 'text', size: 'Content of file1'.length, lastModified: new Date(), }, messageId); initialConversation.addFileForMessage('file2.txt', { + type: 'text', size: 'Content of file2'.length, lastModified: new Date(), }, messageId); @@ -32,14 +34,23 @@ Deno.test({ toolUseId: 'test-id', toolName: 'forget_files', toolInput: { - fileNames: ['file1.txt', 'file2.txt'], + files: [{ filePath: 'file1.txt', revision: messageId }, { + filePath: 'file2.txt', + revision: messageId, + }], }, }; const result = await tool.runTool(initialConversation, toolUse, projectEditor); + // console.log('Forget existing files from conversation - bbaiResponse:', result.bbaiResponse); + // console.log('Forget existing files from conversation - toolResponse:', result.toolResponse); + // console.log('Forget existing files from conversation - toolResults:', result.toolResults); assertStringIncludes(result.bbaiResponse, 'BBai has removed these files from the conversation'); - assertStringIncludes(result.toolResponse, 'Removed files from the conversation:\n- file1.txt\n- file2.txt'); + assertStringIncludes( + result.toolResponse, + 'Removed files from the conversation:\n- file1.txt (Revision: 1111-2222)\n- file2.txt (Revision: 1111-2222)', + ); // Check toolResults assert(Array.isArray(result.toolResults), 'toolResults should be an array'); @@ -56,8 +67,8 @@ Deno.test({ // Check if files are removed from the conversation const conversation = await projectEditor.initConversation('test-conversation-id'); - const file1 = conversation.getFile('file1.txt'); - const file2 = conversation.getFile('file2.txt'); + const file1 = conversation.getFileMetadata('file1.txt', '1'); + const file2 = conversation.getFileMetadata('file2.txt', '2'); assertEquals(file1, undefined, 'file1.txt should not exist in the conversation'); assertEquals(file2, undefined, 'file2.txt should not exist in the conversation'); @@ -79,20 +90,27 @@ Deno.test({ const tool = new LLMToolForgetFiles(); + const messageId = '1111-2222'; const toolUse: LLMAnswerToolUse = { toolValidation: { validated: true, results: '' }, toolUseId: 'test-id', toolName: 'forget_files', toolInput: { - fileNames: ['non_existent.txt'], + files: [{ filePath: 'non_existent.txt', revision: messageId }], }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Attempt to forget non-existent file - bbaiResponse:', result.bbaiResponse); + // console.log('Attempt to forget non-existent file - toolResponse:', result.toolResponse); + // console.log('Attempt to forget non-existent file - toolResults:', result.toolResults); assertStringIncludes(result.bbaiResponse, 'BBai failed to remove these files from the conversation'); - assertStringIncludes(result.toolResponse, 'non_existent.txt: File is not in the conversation history'); + assertStringIncludes( + result.toolResponse, + 'non_existent.txt (1111-2222): File is not in the conversation history', + ); // Check toolResults assert(Array.isArray(result.toolResults), 'toolResults should be an array'); @@ -129,6 +147,7 @@ Deno.test({ await Deno.writeTextFile(join(testProjectRoot, 'existing_file.txt'), 'Content of existing file'); const conversation = await projectEditor.initConversation('test-conversation-id'); conversation.addFileForMessage('existing_file.txt', { + type: 'text', size: 'Content of existing file'.length, lastModified: new Date(), }, messageId); @@ -138,25 +157,31 @@ Deno.test({ toolUseId: 'test-id', toolName: 'forget_files', toolInput: { - fileNames: ['existing_file.txt', 'non_existent_file.txt'], + files: [{ filePath: 'existing_file.txt', revision: messageId }, { + filePath: 'non_existent_file.txt', + revision: messageId, + }], }, }; const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Forget mix of existing and non-existent files - bbaiResponse:', result.bbaiResponse); + // console.log('Forget mix of existing and non-existent files - toolResponse:', result.toolResponse); + // console.log('Forget mix of existing and non-existent files - toolResults:', result.toolResults); assertStringIncludes( result.bbaiResponse, - 'BBai has removed these files from the conversation: existing_file.txt', + 'BBai has removed these files from the conversation: existing_file.txt (Revision: 1111-2222)', ); assertStringIncludes( result.bbaiResponse, - 'BBai failed to remove these files from the conversation:\n- non_existent_file.txt: File is not in the conversation history', + 'BBai failed to remove these files from the conversation:\n- non_existent_file.txt (1111-2222): File is not in the conversation history', ); assertStringIncludes(result.toolResponse, 'Removed files from the conversation:\n- existing_file.txt'); assertStringIncludes( result.toolResponse, - 'Failed to remove files from the conversation:\n- non_existent_file.txt: File is not in the conversation history', + 'Failed to remove files from the conversation:\n- non_existent_file.txt (1111-2222): File is not in the conversation history', ); // Check toolResults @@ -166,7 +191,7 @@ Deno.test({ const firstResult = result.toolResults[0]; assert(firstResult.type === 'text', 'First result should be of type text'); - assertStringIncludes(firstResult.text, 'File removed: existing_file.txt'); + assertStringIncludes(firstResult.text, 'File removed: existing_file.txt (Revision: 1111-2222)'); const secondResult = result.toolResults[1]; assert(secondResult.type === 'text', 'Second result should be of type text'); @@ -176,7 +201,7 @@ Deno.test({ ); // Check if existing file is forgotten from the conversation - const existingFile = conversation.getFile('existing_file.txt'); + const existingFile = conversation.getFileMetadata('existing_file.txt', '1'); assertEquals(existingFile, undefined, 'existing_file.txt should not exist in the conversation'); // Check that listFiles doesn't include either file diff --git a/api/tests/t/llms/tools/requestFilesTool.test.ts b/api/tests/t/llms/tools/requestFilesTool.test.ts index 713a94e..59e91a1 100644 --- a/api/tests/t/llms/tools/requestFilesTool.test.ts +++ b/api/tests/t/llms/tools/requestFilesTool.test.ts @@ -28,6 +28,9 @@ Deno.test({ const initialConversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(initialConversation, toolUse, projectEditor); + console.log('Request existing files - bbaiResponse:', result.bbaiResponse); + console.log('Request existing files - toolResponse:', result.toolResponse); + console.log('Request existing files - toolResults:', result.toolResults); assertStringIncludes(result.bbaiResponse, 'BBai has added these files to the conversation'); assertStringIncludes(result.toolResponse, 'Added files to the conversation:\n- file1.txt\n- file2.txt'); @@ -100,6 +103,9 @@ Deno.test({ const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Request non-existent file - bbaiResponse:', result.bbaiResponse); + // console.log('Request non-existent file - toolResponse:', result.toolResponse); + // console.log('Request non-existent file - toolResults:', result.toolResults); assertStringIncludes(result.bbaiResponse, 'BBai failed to add these files to the conversation'); assertStringIncludes(result.toolResponse, 'No files added'); @@ -111,11 +117,14 @@ Deno.test({ const firstResult = result.toolResults[0]; assert(firstResult.type === 'text', 'First result should be of type text'); - assertStringIncludes(firstResult.text, 'Error adding file non_existent.txt: No such file or directory'); + assertStringIncludes( + firstResult.text, + 'Error adding file non_existent.txt: Access denied: non_existent.txt does not exist in the project directory', + ); assertStringIncludes(firstResult.text, 'non_existent.txt'); // Check that the non-existent file is not in the conversation - const nonExistentFile = conversation.getFile('non_existent.txt'); + const nonExistentFile = conversation.getFileMetadata('non_existent.txt', '1'); assertEquals(nonExistentFile, undefined, 'non_existent.txt should not be in the conversation'); // Check that listFiles doesn't include the non-existent file @@ -149,6 +158,9 @@ Deno.test({ }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Request file outside project root - bbaiResponse:', result.bbaiResponse); + // console.log('Request file outside project root - toolResponse:', result.toolResponse); + // console.log('Request file outside project root - toolResults:', result.toolResults); assertStringIncludes(result.bbaiResponse, 'BBai failed to add these files to the conversation'); assertStringIncludes(result.toolResponse, '../outside_project.txt: Access denied'); @@ -160,10 +172,13 @@ Deno.test({ const firstResult = result.toolResults[0]; assert(firstResult.type === 'text', 'First result should be of type text'); - assertStringIncludes(firstResult.text, '../outside_project.txt: Access denied'); + assertStringIncludes( + firstResult.text, + '../outside_project.txt: Access denied: ../outside_project.txt is outside the project directory', + ); // Check that the outside file is not in the conversation - const outsideFile = conversation.getFile('../outside_project.txt'); + const outsideFile = conversation.getFileMetadata('../outside_project.txt', '1'); assertEquals(outsideFile, undefined, '../outside_project.txt should not be in the conversation'); // Check that listFiles doesn't include the outside file diff --git a/api/tests/t/llms/tools/searchProjectTool.test.ts b/api/tests/t/llms/tools/searchProjectTool.test.ts index 05c8860..a906015 100644 --- a/api/tests/t/llms/tools/searchProjectTool.test.ts +++ b/api/tests/t/llms/tools/searchProjectTool.test.ts @@ -14,6 +14,15 @@ async function createTestFiles(testProjectRoot: string) { // Create an empty file for edge case testing Deno.writeTextFileSync(join(testProjectRoot, 'empty_file.txt'), ''); + // Create a large file with a pattern that spans potential buffer boundaries + const largeFileContent = 'A'.repeat(1024 * 1024) + // 1MB of 'A's + 'Start of pattern\n' + + 'B'.repeat(1024) + // 1KB of 'B's + '\nEnd of pattern' + + 'C'.repeat(1024 * 1024); // Another 1MB of 'C's + + Deno.writeTextFileSync(join(testProjectRoot, 'large_file_with_pattern.txt'), largeFileContent); + // Create files with special content for regex testing Deno.writeTextFileSync(join(testProjectRoot, 'regex_test1.txt'), 'This is a test. Another test.'); Deno.writeTextFileSync(join(testProjectRoot, 'regex_test2.txt'), 'Testing 123, testing 456.'); @@ -30,6 +39,8 @@ async function createTestFiles(testProjectRoot: string) { await setFileModificationTime(join(testProjectRoot, 'subdir', 'file3.txt'), pastDate); await setFileModificationTime(join(testProjectRoot, 'large_file.txt'), currentDate); await setFileModificationTime(join(testProjectRoot, 'empty_file.txt'), currentDate); + // Set modification time for the very large file + await setFileModificationTime(join(testProjectRoot, 'large_file_with_pattern.txt'), currentDate); } // Helper function to set file modification time @@ -51,7 +62,7 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: 'Hello', + contentPattern: 'Hello', }, }; @@ -79,6 +90,61 @@ Deno.test({ assert(foundFiles.some((f) => f.endsWith(file)), `File ${file} not found in the result`); }); assert(foundFiles.length === expectedFiles.length, 'Number of found files does not match expected'); + + // Add a delay before cleanup + await new Promise((resolve) => setTimeout(resolve, 1000)); + }); + }, + sanitizeResources: false, + sanitizeOps: false, +}); + +Deno.test({ + name: 'SearchProjectTool - Search pattern spanning multiple buffers', + fn: async () => { + await withTestProject(async (testProjectRoot) => { + const projectEditor = await getProjectEditor(testProjectRoot); + await createTestFiles(testProjectRoot); + + const tool = new LLMToolSearchProject(); + + const toolUse: LLMAnswerToolUse = { + toolValidation: { validated: true, results: '' }, + toolUseId: 'test-id', + toolName: 'search_project', + toolInput: { + contentPattern: 'Start of pattern\\n[B]+\\nEnd of pattern', + }, + }; + + const conversation = await projectEditor.initConversation('test-conversation-id'); + const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Search pattern spanning multiple buffers - bbaiResponse:', result.bbaiResponse); + // console.log('Search pattern spanning multiple buffers - toolResponse:', result.toolResponse); + // console.log('Search pattern spanning multiple buffers - toolResults:', result.toolResults); + + assertStringIncludes( + result.bbaiResponse, + 'BBai found 1 files matching the search criteria', + ); + assertStringIncludes( + result.toolResponse, + 'Found 1 files matching the search criteria', + ); + const toolResults = result.toolResults as string; + assertStringIncludes(toolResults, '1 files match the search criteria'); + + assertStringIncludes(toolResults, ''); + assertStringIncludes(toolResults, ''); + + const expectedFiles = ['large_file_with_pattern.txt']; + const fileContent = toolResults.split('')[1].split('')[0].trim(); + const foundFiles = fileContent.split('\n'); + + expectedFiles.forEach((file) => { + assert(foundFiles.some((f) => f.endsWith(file)), `File ${file} not found in the result`); + }); + assert(foundFiles.length === expectedFiles.length, 'Number of found files does not match expected'); }); }, sanitizeResources: false, @@ -99,8 +165,8 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - date_after: '2024-01-01', - date_before: '2026-01-01', + dateAfter: '2024-01-01', + dateBefore: '2026-01-01', }, }; @@ -111,17 +177,17 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, - 'BBai found 8 files matching the search criteria: modified after 2024-01-01, modified before 2026-01-01', + 'BBai found 9 files matching the search criteria: modified after 2024-01-01, modified before 2026-01-01', ); assertStringIncludes( result.toolResponse, - 'Found 8 files matching the search criteria: modified after 2024-01-01, modified before 2026-01-01', + 'Found 9 files matching the search criteria: modified after 2024-01-01, modified before 2026-01-01', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '8 files match the search criteria: modified after 2024-01-01, modified before 2026-01-01', + '9 files match the search criteria: modified after 2024-01-01, modified before 2026-01-01', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -130,6 +196,7 @@ Deno.test({ 'file2.js', 'large_file.txt', 'empty_file.txt', + 'large_file_with_pattern.txt', 'regex_test1.txt', 'regex_test2.txt', 'regex_test3.txt', @@ -164,28 +231,29 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - file_pattern: '*.txt', - size_min: 1, + filePattern: '*.txt', + sizeMin: 1, }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); - //console.log('File-only search response:', result.bbaiResponse); - //console.log('File-only search files:', result.toolResults); + console.log('File-only search (metadata) - bbaiResponse:', result.bbaiResponse); + console.log('File-only search (metadata) - toolResponse:', result.toolResponse); + console.log('File-only search (metadata) - toolResults:', result.toolResults); assertStringIncludes( result.bbaiResponse, - 'BBai found 7 files matching the search criteria: file pattern "*.txt", minimum size 1 bytes', + 'BBai found 9 files matching the search criteria: file pattern "*.txt", minimum size 1 bytes', ); assertStringIncludes( result.toolResponse, - 'Found 7 files matching the search criteria: file pattern "*.txt", minimum size 1 bytes', + 'Found 9 files matching the search criteria: file pattern "*.txt", minimum size 1 bytes', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '7 files match the search criteria: file pattern "*.txt", minimum size 1 bytes', + '9 files match the search criteria: file pattern "*.txt", minimum size 1 bytes', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -193,11 +261,13 @@ Deno.test({ const expectedFiles = [ 'file1.txt', 'large_file.txt', + 'large_file_with_pattern.txt', 'regex_test1.txt', 'regex_test2.txt', 'regex_test3.txt', 'regex_test4.txt', 'regex_test5.txt', + 'subdir/file3.txt', ]; const fileContent = toolResults.split('')[1].split('')[0].trim(); const foundFiles = fileContent.split('\n'); @@ -226,12 +296,12 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: 'Hello', - file_pattern: '*.txt', - size_min: 1, - size_max: 1000, - date_after: '2022-01-01', - date_before: '2024-01-01', + contentPattern: 'Hello', + filePattern: '*.txt', + sizeMin: 1, + sizeMax: 1000, + dateAfter: '2022-01-01', + dateBefore: '2024-01-01', }, }; @@ -242,17 +312,17 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, - 'BBai found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', + 'BBai found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', ); assertStringIncludes( result.toolResponse, - 'Found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', + 'Found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '2 files match the search criteria: content pattern "Hello", file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', + '2 files match the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", modified after 2022-01-01, modified before 2024-01-01, minimum size 1 bytes, maximum size 1000 bytes', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -286,8 +356,8 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - file_pattern: '*.txt', - size_max: 0, + filePattern: '*.txt', + sizeMax: 0, }, }; @@ -338,8 +408,8 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: 'Hello', - file_pattern: '*.txt', + contentPattern: 'Hello', + filePattern: '*.txt', }, }; @@ -348,16 +418,16 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, - 'BBai found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt"', + 'BBai found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt"', ); assertStringIncludes( result.toolResponse, - 'Found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt"', + 'Found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt"', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '2 files match the search criteria: content pattern "Hello", file pattern "*.txt"', + '2 files match the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt"', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -390,8 +460,8 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - file_pattern: '*.txt', - size_min: 5000, + filePattern: '*.txt', + sizeMin: 5000, }, }; @@ -400,21 +470,21 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, - 'BBai found 1 files matching the search criteria: file pattern "*.txt", minimum size 5000 bytes', + 'BBai found 2 files matching the search criteria: file pattern "*.txt", minimum size 5000 bytes', ); assertStringIncludes( result.toolResponse, - 'Found 1 files matching the search criteria: file pattern "*.txt", minimum size 5000 bytes', + 'Found 2 files matching the search criteria: file pattern "*.txt", minimum size 5000 bytes', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '1 files match the search criteria: file pattern "*.txt", minimum size 5000 bytes', + '2 files match the search criteria: file pattern "*.txt", minimum size 5000 bytes', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); - const expectedFiles = ['large_file.txt']; + const expectedFiles = ['large_file.txt', 'large_file_with_pattern.txt']; const fileContent = toolResults.split('')[1].split('')[0].trim(); const foundFiles = fileContent.split('\n'); @@ -442,7 +512,7 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: 'NonexistentPattern', + contentPattern: 'NonexistentPattern', }, }; @@ -481,11 +551,14 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: '[', // Invalid regex pattern + contentPattern: '[', // Invalid regex pattern }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Error handling for invalid search pattern - bbaiResponse:', result.bbaiResponse); + // console.log('Error handling for invalid search pattern - toolResponse:', result.toolResponse); + // console.log('Error handling for invalid search pattern - toolResults:', result.toolResults); assertStringIncludes( result.bbaiResponse, @@ -495,6 +568,10 @@ Deno.test({ result.toolResponse, 'Found 0 files matching the search criteria: content pattern "["', ); + assertStringIncludes( + result.toolResults as string, + 'Invalid regular expression: /[/: Unterminated character class', + ); assertStringIncludes( result.toolResults as string, '0 files match the search criteria: content pattern "["', @@ -519,9 +596,9 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: 'Hello', - file_pattern: '*.txt', - size_max: 1000, + contentPattern: 'Hello', + filePattern: '*.txt', + sizeMax: 1000, }, }; @@ -530,16 +607,16 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, - 'BBai found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt", maximum size 1000 bytes', + 'BBai found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", maximum size 1000 bytes', ); assertStringIncludes( result.toolResponse, - 'Found 2 files matching the search criteria: content pattern "Hello", file pattern "*.txt", maximum size 1000 bytes', + 'Found 2 files matching the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", maximum size 1000 bytes', ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, - '2 files match the search criteria: content pattern "Hello", file pattern "*.txt", maximum size 1000 bytes', + '2 files match the search criteria: content pattern "Hello", case-sensitive, file pattern "*.txt", maximum size 1000 bytes', ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -572,7 +649,7 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - file_pattern: 'file2.js', + filePattern: 'file2.js', }, }; @@ -628,30 +705,30 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: String.raw`currentConversation\?\.title`, - file_pattern: 'bui/src/islands/Chat.tsx', + contentPattern: String.raw`currentConversation\?\.title`, + filePattern: 'bui/src/islands/Chat.tsx', }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); - console.info('Tool result:', result); + // console.info('Tool result:', result); assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "currentConversation\?\.title", file pattern "bui/src/islands/Chat.tsx"`, + .raw`BBai found 1 files matching the search criteria: content pattern "currentConversation\?\.title", case-sensitive, file pattern "bui/src/islands/Chat.tsx"`, ); assertStringIncludes( result.toolResponse, String - .raw`Found 1 files matching the search criteria: content pattern "currentConversation\?\.title", file pattern "bui/src/islands/Chat.tsx"`, + .raw`Found 1 files matching the search criteria: content pattern "currentConversation\?\.title", case-sensitive, file pattern "bui/src/islands/Chat.tsx"`, ); const toolResults = result.toolResults as string; assertStringIncludes( toolResults, String - .raw`1 files match the search criteria: content pattern "currentConversation\?\.title", file pattern "bui/src/islands/Chat.tsx"`, + .raw`1 files match the search criteria: content pattern "currentConversation\?\.title", case-sensitive, file pattern "bui/src/islands/Chat.tsx"`, ); assertStringIncludes(toolResults, ''); assertStringIncludes(toolResults, ''); @@ -687,8 +764,8 @@ Deno.test({ toolUseId: 'test-id', toolName: 'search_project', toolInput: { - content_pattern: String.raw`\btest\b`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`\btest\b`, + filePattern: 'regex_test*.txt', }, }; @@ -698,7 +775,7 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 3 files matching the search criteria: content pattern "\btest\b", file pattern "regex_test*.txt"`, + .raw`BBai found 3 files matching the search criteria: content pattern "\btest\b", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; assertStringIncludes(toolResults, 'regex_test1.txt'); @@ -723,8 +800,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`, + filePattern: 'regex_test*.txt', }, }; @@ -734,7 +811,7 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; assertStringIncludes(toolResults, 'regex_test3.txt'); @@ -758,8 +835,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`https?://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`https?://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`, + filePattern: 'regex_test*.txt', }, }; @@ -769,7 +846,7 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "https?://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "https?://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; assertStringIncludes(toolResults, 'regex_test4.txt'); @@ -793,8 +870,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`(\d{3}[-.]?\d{3}[-.]?\d{4}|\(\d{3}\)\s*\d{3}[-.]?\d{4})`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`(\d{3}[-.]?\d{3}[-.]?\d{4}|\(\d{3}\)\s*\d{3}[-.]?\d{4})`, + filePattern: 'regex_test*.txt', }, }; @@ -804,7 +881,7 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "(\d{3}[-.]?\d{3}[-.]?\d{4}|\(\d{3}\)\s*\d{3}[-.]?\d{4})", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "(\d{3}[-.]?\d{3}[-.]?\d{4}|\(\d{3}\)\s*\d{3}[-.]?\d{4})", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; assertStringIncludes(toolResults, 'regex_test5.txt'); @@ -814,12 +891,8 @@ Deno.test({ sanitizeOps: false, }); -/* -// [TODO] these two tests are failing - I suspect incompatible `grep` pattern -// Claude: The issue is likely with the escaping of the pattern. In grep's extended regex (-E), parentheses have special meaning and need to be escaped. -// I tried escaping the parens but test still fails Deno.test({ - name: 'SearchProjectTool - Search with lookahead regex', + name: 'SearchProjectTool - Search with complex regex pattern', fn: async () => { await withTestProject(async (testProjectRoot) => { const projectEditor = await getProjectEditor(testProjectRoot); @@ -832,8 +905,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`Test(?=ing)`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`, + filePattern: 'regex_test*.txt', }, }; @@ -843,10 +916,10 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "Test(?=ing)", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; - assertStringIncludes(toolResults, 'regex_test2.txt'); + assertStringIncludes(toolResults, 'regex_test3.txt'); }); }, sanitizeResources: false, @@ -854,7 +927,7 @@ Deno.test({ }); Deno.test({ - name: 'SearchProjectTool - Search with negative lookahead regex', + name: 'SearchProjectTool - Search with regex using quantifiers', fn: async () => { await withTestProject(async (testProjectRoot) => { const projectEditor = await getProjectEditor(testProjectRoot); @@ -867,18 +940,21 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`test(?!ing)`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`test.*test`, + filePattern: 'regex_test*.txt', }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Search with regex using quantifiers - bbaiResponse:', result.bbaiResponse); + // console.log('Search with regex using quantifiers - toolResponse:', result.toolResponse); + // console.log('Search with regex using quantifiers - toolResults:', result.toolResults); assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "test(?!ing)", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "test.*test", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; assertStringIncludes(toolResults, 'regex_test1.txt'); @@ -887,10 +963,9 @@ Deno.test({ sanitizeResources: false, sanitizeOps: false, }); - */ Deno.test({ - name: 'SearchProjectTool - Search with complex regex pattern', + name: 'SearchProjectTool - Search with regex using character classes', fn: async () => { await withTestProject(async (testProjectRoot) => { const projectEditor = await getProjectEditor(testProjectRoot); @@ -903,8 +978,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`[Tt]esting [0-9]+`, + filePattern: 'regex_test*.txt', }, }; @@ -914,10 +989,10 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "[Tt]esting [0-9]+", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; - assertStringIncludes(toolResults, 'regex_test3.txt'); + assertStringIncludes(toolResults, 'regex_test2.txt'); }); }, sanitizeResources: false, @@ -925,7 +1000,7 @@ Deno.test({ }); Deno.test({ - name: 'SearchProjectTool - Search with regex using quantifiers', + name: 'SearchProjectTool - Search with lookahead regex', fn: async () => { await withTestProject(async (testProjectRoot) => { const projectEditor = await getProjectEditor(testProjectRoot); @@ -938,8 +1013,8 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`test.*test`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`Test(?=ing)`, + filePattern: 'regex_test*.txt', }, }; @@ -949,10 +1024,10 @@ Deno.test({ assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "test.*test", file pattern "regex_test*.txt"`, + .raw`BBai found 1 files matching the search criteria: content pattern "Test(?=ing)", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; - assertStringIncludes(toolResults, 'regex_test1.txt'); + assertStringIncludes(toolResults, 'regex_test2.txt'); }); }, sanitizeResources: false, @@ -960,7 +1035,7 @@ Deno.test({ }); Deno.test({ - name: 'SearchProjectTool - Search with regex using character classes', + name: 'SearchProjectTool - Search with negative lookahead regex', fn: async () => { await withTestProject(async (testProjectRoot) => { const projectEditor = await getProjectEditor(testProjectRoot); @@ -973,21 +1048,134 @@ Deno.test({ toolName: 'search_project', toolUseId: 'test-id', toolInput: { - content_pattern: String.raw`[Tt]esting [0-9]+`, - file_pattern: 'regex_test*.txt', + contentPattern: String.raw`test(?!ing)`, + filePattern: 'regex_test*.txt', }, }; const conversation = await projectEditor.initConversation('test-conversation-id'); const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Search with negative lookahead regex - bbaiResponse:', result.bbaiResponse); + // console.log('Search with negative lookahead regex - toolResponse:', result.toolResponse); + // console.log('Search with negative lookahead regex - toolResults:', result.toolResults); assertStringIncludes( result.bbaiResponse, String - .raw`BBai found 1 files matching the search criteria: content pattern "[Tt]esting [0-9]+", file pattern "regex_test*.txt"`, + .raw`BBai found 3 files matching the search criteria: content pattern "test(?!ing)", case-sensitive, file pattern "regex_test*.txt"`, ); const toolResults = result.toolResults as string; - assertStringIncludes(toolResults, 'regex_test2.txt'); + + assertStringIncludes(toolResults, ''); + assertStringIncludes(toolResults, ''); + + const expectedFiles = ['regex_test1.txt', 'regex_test3.txt', 'regex_test4.txt']; + const fileContent = toolResults.split('')[1].split('')[0].trim(); + const foundFiles = fileContent.split('\n'); + + expectedFiles.forEach((file) => { + assert(foundFiles.some((f) => f.endsWith(file)), `File ${file} not found in the result`); + }); + assert(foundFiles.length === expectedFiles.length, 'Number of found files does not match expected'); + }); + }, + sanitizeResources: false, + sanitizeOps: false, +}); + +Deno.test({ + name: 'SearchProjectTool - Case-sensitive search', + fn: async () => { + await withTestProject(async (testProjectRoot) => { + const projectEditor = await getProjectEditor(testProjectRoot); + await createTestFiles(testProjectRoot); + + const tool = new LLMToolSearchProject(); + + const toolUse: LLMAnswerToolUse = { + toolValidation: { validated: true, results: '' }, + toolName: 'search_project', + toolUseId: 'test-id', + toolInput: { + contentPattern: 'Test', + caseSensitive: true, + filePattern: 'regex_test*.txt', + }, + }; + + const conversation = await projectEditor.initConversation('test-conversation-id'); + const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Case-sensitive search - bbaiResponse:', result.bbaiResponse); + // console.log('Case-sensitive search - toolResponse:', result.toolResponse); + // console.log('Case-sensitive search - toolResults:', result.toolResults); + + assertStringIncludes( + result.bbaiResponse, + 'BBai found 2 files matching the search criteria: content pattern "Test", case-sensitive, file pattern "regex_test*.txt"', + ); + const toolResults = result.toolResults as string; + assertStringIncludes(toolResults, ''); + assertStringIncludes(toolResults, ''); + + const expectedFiles = ['regex_test2.txt', 'regex_test3.txt']; + const fileContent = toolResults.split('')[1].split('')[0].trim(); + const foundFiles = fileContent.split('\n'); + + expectedFiles.forEach((file) => { + assert(foundFiles.some((f) => f.endsWith(file)), `File ${file} not found in the result`); + }); + assert(foundFiles.length === expectedFiles.length, 'Number of found files does not match expected'); + + assert(!toolResults.includes('regex_test1.txt'), `This file contains 'test' but not 'Test'`); + assert(!toolResults.includes('regex_test4.txt'), `This file contains 'test' but not 'Test'`); + }); + }, + sanitizeResources: false, + sanitizeOps: false, +}); + +Deno.test({ + name: 'SearchProjectTool - Case-insensitive search', + fn: async () => { + await withTestProject(async (testProjectRoot) => { + const projectEditor = await getProjectEditor(testProjectRoot); + await createTestFiles(testProjectRoot); + + const tool = new LLMToolSearchProject(); + + const toolUse: LLMAnswerToolUse = { + toolValidation: { validated: true, results: '' }, + toolName: 'search_project', + toolUseId: 'test-id', + toolInput: { + contentPattern: 'Test', + caseSensitive: false, + filePattern: 'regex_test*.txt', + }, + }; + + const conversation = await projectEditor.initConversation('test-conversation-id'); + const result = await tool.runTool(conversation, toolUse, projectEditor); + // console.log('Case-insensitive search - bbaiResponse:', result.bbaiResponse); + // console.log('Case-insensitive search - toolResponse:', result.toolResponse); + // console.log('Case-insensitive search - toolResults:', result.toolResults); + + assertStringIncludes( + result.bbaiResponse, + 'BBai found 4 files matching the search criteria: content pattern "Test", case-insensitive, file pattern "regex_test*.txt"', + ); + const toolResults = result.toolResults as string; + assertStringIncludes(toolResults, ''); + assertStringIncludes(toolResults, ''); + + const expectedFiles = ['regex_test1.txt', 'regex_test2.txt', 'regex_test3.txt', 'regex_test4.txt']; + const fileContent = toolResults.split('')[1].split('')[0].trim(); + const foundFiles = fileContent.split('\n'); + + expectedFiles.forEach((file) => { + assert(foundFiles.some((f) => f.endsWith(file)), `File ${file} not found in the result`); + }); + assert(foundFiles.length === expectedFiles.length, 'Number of found files does not match expected'); }); }, sanitizeResources: false, diff --git a/bui/deno.jsonc b/bui/deno.jsonc index 595b230..5a3e0ae 100644 --- a/bui/deno.jsonc +++ b/bui/deno.jsonc @@ -46,5 +46,5 @@ "src/fixtures/**/*.ts" ] }, - "version": "0.0.16-beta" + "version": "0.0.17-beta" } diff --git a/cli/deno.jsonc b/cli/deno.jsonc index fcce118..02781a1 100644 --- a/cli/deno.jsonc +++ b/cli/deno.jsonc @@ -1,6 +1,6 @@ { "name": "bbai-cli", - "version": "0.0.16-beta", + "version": "0.0.17-beta", "exports": "./src/main.ts", "tasks": { "start": "deno run --allow-env --allow-read --allow-write --allow-run --allow-net src/main.ts", diff --git a/cli/src/commands/apiRestart.ts b/cli/src/commands/apiRestart.ts index bef4c5b..eb093a7 100644 --- a/cli/src/commands/apiRestart.ts +++ b/cli/src/commands/apiRestart.ts @@ -10,7 +10,7 @@ export const apiRestart = new Command() .option('--log-file ', 'Specify a log file to write API output', { default: undefined }) .option('--hostname ', 'Specify the hostname for API to listen on', { default: undefined }) .option('--port ', 'Specify the port for API to listen on', { default: undefined }) - .option('--useTls ', 'Specify whether API should listen with TLS', { default: undefined }) + .option('--use-tls ', 'Specify whether API should listen with TLS', { default: undefined }) .action(async ({ logLevel: apiLogLevel, logFile: apiLogFile, hostname, port, useTls }) => { const startDir = Deno.cwd(); const fullConfig = await ConfigManager.fullConfig(startDir); diff --git a/cli/src/commands/apiStart.ts b/cli/src/commands/apiStart.ts index 2323e7d..1b9cd9f 100644 --- a/cli/src/commands/apiStart.ts +++ b/cli/src/commands/apiStart.ts @@ -11,7 +11,7 @@ export const apiStart = new Command() .option('--log-file ', 'Specify a log file to write output', { default: undefined }) .option('--hostname ', 'Specify the hostname for API to listen on', { default: undefined }) .option('--port ', 'Specify the port for API to listen on', { default: undefined }) - .option('--useTls ', 'Specify whether API should listen with TLS', { default: undefined }) + .option('--use-tls ', 'Specify whether API should listen with TLS', { default: undefined }) .option('--follow', 'Do not detach and follow the API logs', { default: false }) .action(async ({ logLevel: apiLogLevel, logFile: apiLogFile, hostname, port, useTls, follow }) => { const startDir = Deno.cwd(); @@ -60,7 +60,7 @@ export const apiStart = new Command() if (!fullConfig.noBrowser) { try { const command = Deno.build.os === 'windows' - ? new Deno.Command('cmd', { args: ['/c', 'start', chatUrl] }) + ? new Deno.Command('cmd', { args: ['/c', 'start', `"${chatUrl}"`] }) : Deno.build.os === 'darwin' ? new Deno.Command('open', { args: [chatUrl] }) : new Deno.Command('xdg-open', { args: [chatUrl] }); diff --git a/cli/src/utils/apiControl.utils.ts b/cli/src/utils/apiControl.utils.ts index 81644bc..24cc82b 100644 --- a/cli/src/utils/apiControl.utils.ts +++ b/cli/src/utils/apiControl.utils.ts @@ -42,7 +42,7 @@ export async function startApiServer( } const apiHostnameArgs = apiHostname ? ['--hostname', apiHostname] : []; const apiPortArgs = apiPort ? ['--port', apiPort] : []; - const apiUseTlsArgs = typeof apiUseTls !== 'undefined' ? ['--useTls', apiUseTls ? 'true' : 'false'] : []; + const apiUseTlsArgs = typeof apiUseTls !== 'undefined' ? ['--use-tls', apiUseTls ? 'true' : 'false'] : []; logger.debug(`Starting BBai API server on ${apiHostname}:${apiPort}, logging to ${apiLogFilePath}`); diff --git a/deno.jsonc b/deno.jsonc index 21ecb34..ae6a313 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -1,6 +1,6 @@ { "name": "bbai", - "version": "0.0.16-beta", + "version": "0.0.17-beta", "exports": "./cli/src/main.ts", "tasks": { "tool:check-types-project": "deno task -c ./cli/deno.jsonc check-types && deno task -c ./bui/deno.jsonc check-types && deno task -c ./api/deno.jsonc check-types", diff --git a/import_map.json b/import_map.json index 66bf408..aee89b7 100644 --- a/import_map.json +++ b/import_map.json @@ -44,6 +44,7 @@ "@std/cli": "jsr:@std/cli@^1.0.3", "@std/fmt": "jsr:@std/fmt@^1.0.0", "@std/fs": "jsr:@std/fs@^1.0.1", + "@std/streams": "jsr:@std/streams@^1.0.4", "@std/path": "jsr:@std/path@^1.0.2", "@std/io": "jsr:@std/io@^0.224.4", "@std/log": "jsr:@std/log@^0.224.5", diff --git a/scripts/build.sh b/scripts/build.sh index b0cf817..f0ba13c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,6 +6,6 @@ DEST_DIR="${1:-/usr/local/bin}" deno task build # Copy build files to the specified destination directory -cp build/* "$DEST_DIR" +cp build/bbai build/bbai-api "$DEST_DIR" echo "Installation complete. Files copied to $DEST_DIR" \ No newline at end of file diff --git a/version.ts b/version.ts index a773ccc..8261ca6 100644 --- a/version.ts +++ b/version.ts @@ -1 +1 @@ -export const VERSION = "0.0.16-beta"; \ No newline at end of file +export const VERSION = "0.0.17-beta"; \ No newline at end of file