Merge changes for v0.3 release (#25)

## What's Changed

This is a huge release 🚀

Main features:
- Multimodal image input column: Now you can insert an image alongside text as input to LLMs
- OSS now supports multiple projects: You are now able to create projects to manage your tables and files
- Added ability to turn any column into multi-turn chat via the `multi_turn` parameter in `LLMGenConfig`
- Added default prompts when creating Generative Tables: Setup time from table creation to running worlflows is now even shorter.
- Added support for templates: We will progresively add more templates that showcase various use cases
- Support for search query when listing projects, tables, rows
- Table and project import and export
- Various improvements to backend and frontend

There are several breaking changes and deprecations as well, some highlights are listed here, see CHANGELOG for a complete list:
- Added `version` and `meta` to table metadata. Please run the provided migration script to upgrade your existing DB files.
- Delete endpoints will return 404 if resource is not found
- `/v1/gen_tables/{table_type}/{table_id}/thread` has one new required query parameter: `column_id`
- Table list endpoint now defaults to not counting table rows
- Output columns must be string type
- Deprecations:
  - Endpoint `/v1/gen_tables/{table_type}/duplicate/{table_id_src}/{table_id_dst}`

## New Contributors

* @zec0816
* @Hoipang

## Contributors

* @deafnv
* @haoshan98
* @kamil-hassan201
* @noobHappylife
* @jiahuei

## Full Changelog

v0.2...v0.3

.env → .env.example

@@ -3,19 +3,28 @@ OPENAI_API_KEY=
 ANTHROPIC_API_KEY=
 COHERE_API_KEY=
 TOGETHER_API_KEY=
+HYPERBOLIC_API_KEY=
+CEREBRAS_API_KEY=
+SAMBANOVA_API_KEY=
 
 # Service URLs
 DOCIO_URL=http://docio:6979/api/docio
 UNSTRUCTUREDIO_URL=http://unstructuredio:6989
+JAMAI_API_BASE=http://owl:6969/api
+
+# Frontend config
+JAMAI_URL=http://owl:6969
+PUBLIC_JAMAI_URL=
+PUBLIC_IS_SPA=false
+CHECK_ORIGIN=false
 
 # Configuration
 OWL_PORT=6969
-OWL_WORKERS=1
-OWL_DB_DIR=db
-OWL_LOG_DIR=logs
+OWL_WORKERS=3
 DOCIO_WORKERS=1
 DOCIO_DEVICE=cpu
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 RERANKER_MODEL=cross-encoder/ms-marco-TinyBERT-L-2
-OWL_CONCURRENT_ROWS_BATCH_SIZE=3
-OWL_CONCURRENT_COLS_BATCH_SIZE=5
+OWL_CONCURRENT_ROWS_BATCH_SIZE=5
+OWL_CONCURRENT_COLS_BATCH_SIZE=5
+OWL_MAX_WRITE_BATCH_SIZE=1000

.flake8

@@ -4,10 +4,12 @@ select = C,E,F,W,B,B950
 extend-ignore = E203,E402,E501,F541,W503,F401
 extend-exclude =
     .vscode/,
+    __ref__/
     archive/,
     build/,
-    configs/,
+    clients/typescript/,
     dependencies/,
+    services/app/,
     venv/,
     **/dist
     **/build

.gitattributes

@@ -54,3 +54,6 @@
 # These files should not be processed by Linguist for language detection on github.com
 *.p linguist-detectable=false
 *.gz linguist-detectable=false
+
+# Track with Git LFS
+*.parquet filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci-win.yml

@@ -10,13 +10,95 @@ on:
     tags:
       - "v*"
 
+# Cancel in-progress CI jobs if there is a new push
+# https://stackoverflow.com/a/72408109
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
+  pyinstaller_electron_app:
+    name: PyInstaller JamAIBase Electron App Compilation
+    runs-on: windows-11-desktop
+    timeout-minutes: 60
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js 20.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+
+      - name: Install Git
+        run: |
+          $installer_url = "https://github.com/git-for-windows/git/releases/download/v2.45.2.windows.1/Git-2.45.2-64-bit.exe"
+          Invoke-WebRequest -Uri $installer_url -OutFile "GitInstaller.exe"
+          Start-Process -FilePath "GitInstaller.exe" -Args "/VERYSILENT /NORESTART /NOCANCEL /SP- /CLOSEAPPLICATIONS /RESTARTAPPLICATIONS /COMPONENTS='icons,ext\reg\shellhere,assoc,assoc_sh'" -Wait
+          Remove-Item "GitInstaller.exe"
+
+          # Add Git to PATH
+          $gitPath = "C:\Program Files\Git\cmd"
+          $env:PATH = "$gitPath;$env:PATH"
+          [Environment]::SetEnvironmentVariable("PATH", $env:PATH, [EnvironmentVariableTarget]::Machine)
+
+          # Output the new PATH to a step output
+          echo "PATH=$env:PATH" >> $env:GITHUB_ENV
+
+          # Verify Git installation
+          git --version
+        shell: powershell
+
+      - name: Verify Git in PATH
+        run: |
+          Write-Host "Current PATH: $env:PATH"
+          $gitPath = (Get-Command git -ErrorAction SilentlyContinue).Path
+          if ($gitPath) {
+            Write-Host "Git found at: $gitPath"
+          } else {
+            Write-Host "Git not found in PATH"
+            exit 1
+          }
+        shell: powershell
+
+      - name: Inspect git version
+        run: |
+          git --version
+
+      - name: Remove cloud-only modules and start compiling JamAIBase Electron App
+        run: |
+          mv .env.example .env
+          $ErrorActionPreference = "Stop"
+          .\scripts\compile_jamaibase_app.ps1
+        shell: powershell
+
+      - name: Validate jamaibase.exe is healthy
+        run: |
+          cd services\app\build-electron\make\zip\win32\x64\
+
+          Expand-Archive -Path 'jamaibase-app-win32-x64-0.2.0.zip' -DestinationPath 'jamaibase-app-win32-x64-0.2.0'
+
+          $process = Start-Process -NoNewWindow -FilePath ".\jamaibase-app-win32-x64-0.2.0\jamaibase-app.exe" -PassThru
+
+          $processId = $process.Id
+          Write-Output "Process ID: $processId"
+
+          # Wait for 5 seconds
+          Start-Sleep -Seconds 10
+
+          # Check if the process is still running
+          if (Get-Process -Id $processId -ErrorAction SilentlyContinue) {
+              Write-Output "The process is still running."
+          } else {
+              Write-Output "The process has exited."
+          }
+        shell: powershell
+
   pyinstaller_api:
     name: PyInstaller API Service Compilation
     runs-on: windows-11-desktop
-    strategy:
-      matrix:
-        python-version: ["3.10"]
+    timeout-minutes: 60
 
     steps:
       - name: Checkout code
@@ -25,7 +107,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.12"
 
       - name: Inspect Python version
         run: python --version
@@ -67,6 +149,7 @@ jobs:
 
       - name: Remove cloud-only modules and start compiling API service
         run: |
+          mv .env.example .env
           $ErrorActionPreference = "Stop"
           .\scripts\compile_api_exe.ps1
         shell: powershell
@@ -75,7 +158,7 @@ jobs:
         run: |
           $env:OWL_WORKERS=1
           $process = Start-Process -NoNewWindow -FilePath ".\services\api\dist\api\api.exe" -PassThru
-          Start-Sleep -Seconds 10
+          Start-Sleep -Seconds 60
           Write-Output "API process ID: $($process.Id)"
           Get-Process
           Test-NetConnection -ComputerName localhost -Port 6969
@@ -96,9 +179,7 @@ jobs:
   pyinstaller_docio:
     name: PyInstaller DocIO Service Compilation
     runs-on: windows-11-desktop
-    strategy:
-      matrix:
-        python-version: ["3.10"]
+    timeout-minutes: 60
 
     steps:
       - name: Checkout code
@@ -107,7 +188,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.10"
 
       - name: Display Python version
         run: python --version
@@ -150,6 +231,7 @@ jobs:
 
       - name: Remove cloud-only modules and start compiling DocIO service
         run: |
+          mv .env.example .env
           $ErrorActionPreference = "Stop"
           .\scripts\compile_docio_exe.ps1
         shell: powershell