Merge pull request #31 from BBai-Tips/staging

Refactored search_project tool to use native stream reader

.github/workflows/release.yml

@@ -76,6 +76,13 @@ jobs:
           echo 'echo BBai has been started in your default web browser.' >> build/bbai_start.bat
           echo 'echo You can close this window.' >> build/bbai_start.bat
 
+          echo '@echo off' > build/bbai_stop.bat
+          echo 'echo Stopping BBai...' >> build/bbai_stop.bat
+          echo 'start "" bbai stop' >> build/bbai_stop.bat
+          echo 'echo.' >> build/bbai_stop.bat
+          echo 'echo BBai has been stopped.' >> build/bbai_stop.bat
+          echo 'echo You can close this window.' >> build/bbai_stop.bat
+
       - name: Create MSI installer (Windows)
         if: matrix.target == 'x86_64-pc-windows-msvc'
         run: |
@@ -88,6 +95,7 @@ jobs:
           cp build/bbai-api.exe installer_files/
           cp build/bbai_init.bat installer_files/
           cp build/bbai_start.bat installer_files/
+          cp build/bbai_stop.bat installer_files/
           
           # Create a WiX source file
           @"
@@ -122,6 +130,9 @@ jobs:
                 <Component Id="bbai_start.bat" Guid="*">
                   <File Id="bbai_start.bat" Name="bbai_start.bat" Source="installer_files\bbai_start.bat" KeyPath="yes" />
                 </Component>
+                <Component Id="bbai_stop.bat" Guid="*">
+                  <File Id="bbai_stop.bat" Name="bbai_stop.bat" Source="installer_files\bbai_stop.bat" KeyPath="yes" />
+                </Component>
               </ComponentGroup>
             </Fragment>
           </Wix>

CHANGELOG.md

@@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
+## [0.0.17-beta] - 2024-09-28
+
+### Changed
+
+- Refactored search_project tool to use native stream reader with buffer and native regex, rather than external grep
+
+
 ## [0.0.16-beta] - 2024-09-27
 
 ### Changed

api/deno.jsonc

@@ -1,6 +1,6 @@
 {
 	"name": "bbai-api",
-	"version": "0.0.16-beta",
+	"version": "0.0.17-beta",
 	"exports": "./src/main.ts",
 	"tasks": {
 		"start": "deno run --allow-read --allow-write --allow-run --allow-net --allow-env src/main.ts",

api/src/editor/projectEditor.ts

@@ -1,6 +1,11 @@
 import { join } from '@std/path';
 
-import { FILE_LISTING_TIERS, generateFileListing, isPathWithinProject } from '../utils/fileHandling.utils.ts';
+import {
+	existsWithinProject,
+	FILE_LISTING_TIERS,
+	generateFileListing,
+	isPathWithinProject,
+} from '../utils/fileHandling.utils.ts';
 import type LLMConversationInteraction from '../llms/interactions/conversationInteraction.ts';
 import type { ProjectInfo as BaseProjectInfo } from '../llms/interactions/conversationInteraction.ts';
 import type { FileMetadata } from 'shared/types.ts';
@@ -160,6 +165,7 @@ class ProjectEditor {
 	}
 
 	// prepareFilesForConversation is called by request_files tool and by add_file handler for user requests
+	// only existing files can be prepared and added, otherwise call rewrite_file tools with createIfMissing:true
 	async prepareFilesForConversation(
 		fileNames: string[],
 	): Promise<
@@ -183,6 +189,9 @@ class ProjectEditor {
 				if (!await isPathWithinProject(this.projectRoot, fileName)) {
 					throw new Error(`Access denied: ${fileName} is outside the project directory`);
 				}
+				if (!await existsWithinProject(this.projectRoot, fileName)) {
+					throw new Error(`Access denied: ${fileName} does not exist in the project directory`);
+				}
 
 				const fullFilePath = join(this.projectRoot, fileName);
 

api/src/llms/tools/formatters/searchProjectTool.browser.tsx

@@ -5,45 +5,45 @@ import type { LLMToolInputSchema, LLMToolRunResultContent } from 'api/llms/llmTo
 import { getContentFromToolResult } from '../../../utils/llms.utils.ts';
 
 export const formatToolUse = (toolInput: LLMToolInputSchema): JSX.Element => {
-	const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as {
-		content_pattern?: string;
-		file_pattern?: string;
-		date_after?: string;
-		date_before?: string;
-		size_min?: number;
-		size_max?: number;
+	const { contentPattern, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } = toolInput as {
+		contentPattern?: string;
+		filePattern?: string;
+		dateAfter?: string;
+		dateBefore?: string;
+		sizeMin?: number;
+		sizeMax?: number;
 	};
 	return (
 		<div className='tool-use'>
 			<h3>Project Search Parameters:</h3>
-			{content_pattern && (
+			{contentPattern && (
 				<p>
-					<strong>Content pattern:</strong> <span style='color: #DAA520;'>${content_pattern}</span>
+					<strong>Content pattern:</strong> <span style='color: #DAA520;'>${contentPattern}</span>
 				</p>
 			)}
-			{file_pattern && (
+			{filePattern && (
 				<p>
-					<strong>File pattern:</strong> <span style='color: #4169E1;'>${file_pattern}</span>
+					<strong>File pattern:</strong> <span style='color: #4169E1;'>${filePattern}</span>
 				</p>
 			)}
-			{date_after && (
+			{dateAfter && (
 				<p>
-					<strong>Modified after:</strong> <span style='color: #008000;'>${date_after}</span>
+					<strong>Modified after:</strong> <span style='color: #008000;'>${dateAfter}</span>
 				</p>
 			)}
-			{date_before && (
+			{dateBefore && (
 				<p>
-					<strong>Modified before:</strong> <span style='color: #008000;'>${date_before}</span>
+					<strong>Modified before:</strong> <span style='color: #008000;'>${dateBefore}</span>
 				</p>
 			)}
-			{size_min && (
+			{sizeMin && (
 				<p>
-					<strong>Minimum size:</strong> <span style='color: #FF00FF;'>${size_min.toString()} bytes</span>
+					<strong>Minimum size:</strong> <span style='color: #FF00FF;'>${sizeMin.toString()} bytes</span>
 				</p>
 			)}
-			{size_max && (
+			{sizeMax && (
 				<p>
-					<strong>Maximum size:</strong> <span style='color: #FF00FF;'>${size_max.toString()} bytes</span>
+					<strong>Maximum size:</strong> <span style='color: #FF00FF;'>${sizeMax.toString()} bytes</span>
 				</p>
 			)}
 		</div>

api/src/llms/tools/formatters/searchProjectTool.console.ts

@@ -5,22 +5,22 @@ import { stripIndents } from 'common-tags';
 import { getContentFromToolResult } from '../../../utils/llms.utils.ts';
 
 export const formatToolUse = (toolInput: LLMToolInputSchema): string => {
-	const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as {
-		content_pattern?: string;
-		file_pattern?: string;
-		date_after?: string;
-		date_before?: string;
-		size_min?: number;
-		size_max?: number;
+	const { contentPattern, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } = toolInput as {
+		contentPattern?: string;
+		filePattern?: string;
+		dateAfter?: string;
+		dateBefore?: string;
+		sizeMin?: number;
+		sizeMax?: number;
 	};
 	return stripIndents`
     ${colors.bold('Project Search Parameters:')}
-    ${content_pattern ? `${colors.cyan('Content pattern:')} ${content_pattern}` : ''}
-    ${file_pattern ? `${colors.cyan('File pattern:')} ${file_pattern}` : ''}
-    ${date_after ? `${colors.cyan('Modified after:')} ${date_after}` : ''}
-    ${date_before ? `${colors.cyan('Modified before:')} ${date_before}` : ''}
-    ${size_min ? `${colors.cyan('Minimum size:')} ${size_min.toString()} bytes` : ''}
-    ${size_max ? `${colors.cyan('Maximum size:')} ${size_max.toString()} bytes` : ''}
+    ${contentPattern ? `${colors.cyan('Content pattern:')} ${contentPattern}` : ''}
+    ${filePattern ? `${colors.cyan('File pattern:')} ${filePattern}` : ''}
+    ${dateAfter ? `${colors.cyan('Modified after:')} ${dateAfter}` : ''}
+    ${dateBefore ? `${colors.cyan('Modified before:')} ${dateBefore}` : ''}
+    ${sizeMin ? `${colors.cyan('Minimum size:')} ${sizeMin.toString()} bytes` : ''}
+    ${sizeMax ? `${colors.cyan('Maximum size:')} ${sizeMax.toString()} bytes` : ''}
   `.trim();
 };
 

api/src/llms/tools/searchProjectTool.ts

@@ -30,28 +30,34 @@ export default class LLMToolSearchProject extends LLMTool {
 		return {
 			type: 'object',
 			properties: {
-				content_pattern: {
+				contentPattern: {
 					type: 'string',
 					description: String
 						.raw`The search pattern for file contents (grep-compatible regular expression). Ensure to escape special regex characters with backslashes, e.g., "\.", "\?", "\*"`,
 				},
-				file_pattern: {
+				caseSensitive: {
+					type: 'boolean',
+					description:
+						'Whether the `contentPattern` is a case sensitive regex. The default is true, to use case sensitive regex.',
+					default: true,
+				},
+				filePattern: {
 					type: 'string',
 					description: 'File name pattern to limit the search to specific file types or names',
 				},
-				date_after: {
+				dateAfter: {
 					type: 'string',
 					description: 'Search for files modified after this date (YYYY-MM-DD format)',
 				},
-				date_before: {
+				dateBefore: {
 					type: 'string',
 					description: 'Search for files modified before this date (YYYY-MM-DD format)',
 				},
-				size_min: {
+				sizeMin: {
 					type: 'number',
 					description: 'Minimum file size in bytes',
 				},
-				size_max: {
+				sizeMax: {
 					type: 'number',
 					description: 'Maximum file size in bytes',
 				},
@@ -73,49 +79,55 @@ export default class LLMToolSearchProject extends LLMTool {
 		projectEditor: ProjectEditor,
 	): Promise<LLMToolRunResult> {
 		const { toolInput } = toolUse;
-		const { content_pattern, file_pattern, date_after, date_before, size_min, size_max } = toolInput as {
-			content_pattern?: string;
-			file_pattern?: string;
-			date_after?: string;
-			date_before?: string;
-			size_min?: number;
-			size_max?: number;
-		};
+		const { contentPattern, caseSensitive = true, filePattern, dateAfter, dateBefore, sizeMin, sizeMax } =
+			toolInput as {
+				contentPattern?: string;
+				caseSensitive?: boolean;
+				filePattern?: string;
+				dateAfter?: string;
+				dateBefore?: string;
+				sizeMin?: number;
+				sizeMax?: number;
+			};
+		// caseSensitive controls the regex flag
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions#advanced_searching_with_flags
 
 		try {
 			let files: string[] = [];
 			let errorMessage: string | null = null;
 
 			let result;
-			if (content_pattern) {
+			if (contentPattern) {
 				// searchContent or searchContentInFiles
-				result = await searchFilesContent(projectEditor.projectRoot, content_pattern, {
-					file_pattern,
-					date_after,
-					date_before,
-					size_min,
-					size_max,
+				result = await searchFilesContent(projectEditor.projectRoot, contentPattern, caseSensitive, {
+					filePattern,
+					dateAfter,
+					dateBefore,
+					sizeMin,
+					sizeMax,
 				});
 			} else {
 				// searchForFiles (metadata-only search)
 				result = await searchFilesMetadata(projectEditor.projectRoot, {
-					file_pattern,
-					date_after,
-					date_before,
-					size_min,
-					size_max,
+					filePattern,
+					dateAfter,
+					dateBefore,
+					sizeMin,
+					sizeMax,
 				});
 			}
 			files = result.files;
 			errorMessage = result.errorMessage;
 
 			const searchCriteria = [
-				content_pattern && `content pattern "${content_pattern}"`,
-				file_pattern && `file pattern "${file_pattern}"`,
-				date_after && `modified after ${date_after}`,
-				date_before && `modified before ${date_before}`,
-				size_min !== undefined && `minimum size ${size_min} bytes`,
-				size_max !== undefined && `maximum size ${size_max} bytes`,
+				contentPattern && `content pattern "${contentPattern}"`,
+				// only include case sensitivity details if content pattern was supplied
+				contentPattern && `${caseSensitive ? 'case-sensitive' : 'case-insensitive'}`,
+				filePattern && `file pattern "${filePattern}"`,
+				dateAfter && `modified after ${dateAfter}`,
+				dateBefore && `modified before ${dateBefore}`,
+				sizeMin !== undefined && `minimum size ${sizeMin} bytes`,
+				sizeMax !== undefined && `maximum size ${sizeMax} bytes`,
 			].filter(Boolean).join(', ');
 
 			const toolResults = stripIndents`

api/src/main.ts

@@ -19,9 +19,9 @@ const { environment, apiHostname, apiPort, apiUseTls } = fullConfig.api;
 
 // Parse command line arguments
 const args = parseArgs(Deno.args, {
-	string: ['log-file', 'port', 'hostname'],
+	string: ['log-file', 'port', 'hostname', 'use-tls'],
 	boolean: ['help', 'version'],
-	alias: { h: 'help', V: 'version', v: 'version', p: 'port', H: 'hostname', t: 'useTls', l: 'log-file' },
+	alias: { h: 'help', V: 'version', v: 'version', p: 'port', H: 'hostname', t: 'use-tls', l: 'log-file' },
 });
 
 if (args.help) {
@@ -33,7 +33,7 @@ Options:
   -V, --version             Show version information
   -H, --hostname <string>   Specify the hostname to run the API server (default: ${apiHostname})
   -p, --port <number>       Specify the port to run the API server (default: ${apiPort})
-  -t, --useTls <boolean>    Specify whether the API server should use TLS (default: ${apiUseTls})
+  -t, --use-tls <boolean>    Specify whether the API server should use TLS (default: ${apiUseTls})
   -l, --log-file <file>     Specify a log file to write output
   `);
 	Deno.exit(0);
@@ -50,8 +50,8 @@ if (apiLogFile) await apiFileLogger(apiLogFile);
 
 const customHostname = args.hostname ? args.hostname : apiHostname;
 const customPort: number = args.port ? parseInt(args.port, 10) : apiPort as number;
-const customUseTls: boolean = typeof args.useTls !== 'undefined'
-	? (args.useTls === 'true' ? true : false)
+const customUseTls: boolean = typeof args['use-tls'] !== 'undefined'
+	? (args['use-tls'] === 'true' ? true : false)
 	: !!apiUseTls;
 //console.debug(`BBai API starting at ${customHostname}:${customPort}`);