diff --git a/.github/workflows/build-installer.yml b/.github/workflows/build-installer.yml index 64af941a..f62d6584 100644 --- a/.github/workflows/build-installer.yml +++ b/.github/workflows/build-installer.yml @@ -1,8 +1,19 @@ name: build installer +run-name: installer-build-${{ inputs.platform }}-${{ github.sha }} + on: workflow_dispatch: - + inputs: + platform: + type: choice + description: installer(s) to build + required: true + options: + - arc + - ultra + - ultra2 + - all jobs: build: runs-on: windows-latest @@ -11,8 +22,59 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: pwd + - name: setup miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + python-version: "3.11" + activate-environment: 'cp311_libuv' + + - name: Install libuv dependency + run: | + conda install -y libuv + conda env list | findstr libuv + echo "LIBUV_DLLS_PATH=$($(conda env list | findstr libuv) -split ' ' | Select-Object -Last 1)\Library\bin" + echo "LIBUV_DLLS_PATH=$($(conda env list | findstr libuv) -split ' ' | Select-Object -Last 1)\Library\bin" >> $env:GITHUB_ENV + + - name: copy libuv dlls to workspace + run: | + New-Item -ItemType Directory -Path "python_package_res\conda\Library\bin" -Force + copy "${{ env.LIBUV_DLLS_PATH }}\*.dll" "python_package_res\conda\Library\bin" + + - name: Install Node.js and npm + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: setup npm project + working-directory: "WebUI" + run: npm install + + - name: build installer + working-directory: "WebUI" run: | - pwd - ls + npm install + npm run fetch-build-resources + npm run pack-python + npm run prebuild + npm run build:${{ inputs.platform }} + + - name: set release path + working-directory: "release" + run: | + echo "RELEASE_DIR=$((pwd).Path)" + echo "RELEASE_DIR=$((pwd).Path)" >> $env:GITHUB_ENV + + - name: upload single release + uses: actions/upload-artifact@v4 + with: + # Name of the artifact to upload. + # Optional. Default is 'artifact' + name: "${{ inputs.platform }}_installers" + # A file, directory or wildcard pattern that describes what to upload + # Required. + path: ${{ env.RELEASE_DIR }}\*.exe + if-no-files-found: error + retention-days: 1 + overwrite: true diff --git a/.gitignore b/.gitignore index b64a0969..ddad1f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .vscode/ env/ package_res/ +python_package_res/ release/ WebUI/external/service/ diff --git a/WebUI/build/README.md b/WebUI/build/README.md index 575e6337..ed097837 100644 --- a/WebUI/build/README.md +++ b/WebUI/build/README.md @@ -1,26 +1,18 @@ ## Prepare a base python 3.11 environment -1. Download embedded python for windows from https://github.com/adang1345/PythonWindows -2. Download get-pip.py from https://bootstrap.pypa.io/get-pip.py -3. Install miniforge: https://github.com/conda-forge/miniforge -4. Create a reference conda environment with libuv installed -5. Download 7zr executable from https://www.7-zip.org/a/7zr.exe, put it under `WebUI\package_res` folder. - -``` -conda create -n cp311_libuv python=3.11 libuv -y - -# copy the path to this conda env -conda env list | findstr cp311_libuv -``` - -5. Run prepack script with 3 additional arguments, this will generate `env.7z` under `WebUI\package_res` folder. - -``` -cd WebUI -npm run prepack -``` - -`package_res/env.7z` could be reused for all platforms. +1. fetch resources via web by calling ```npm run fetch-build-resources``` +2. provide windows libuv dlls: + - Install miniforge: https://github.com/conda-forge/miniforge + - Create a reference conda environment with libuv installed + ``` + conda create -n cp311_libuv python=3.11 libuv -y + # copy the path to this conda env + conda env list | findstr cp311_libuv + ``` + - symlink or copy the conda env path into /python_package_resources +3. run ```npm run pack-python``` + +The resulting `WebUI/npm_package_res/env.7z` could be reused for all platforms. ## Package diff --git a/WebUI/build/installer.nsh.template b/WebUI/build/installer.nsh.template index 13bb2181..8e6730e1 100644 --- a/WebUI/build/installer.nsh.template +++ b/WebUI/build/installer.nsh.template @@ -23,7 +23,9 @@ Abort ; Abort the installation if Cancel is clicked continue: - nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements-${PLATFORM}.txt"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "$INSTDIR\resources\env\get-pip.py" "--no-warn-script-location"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements-${PLATFORM}.txt" "--no-warn-script-location"' + nsExec::ExecToLog '"$INSTDIR\resources\env\python.exe" "-m" "pip" "install" "-r" "$INSTDIR\resources\service\requirements.txt" "--no-warn-script-location"' end: DetailPrint "Installation completed." !macroend diff --git a/WebUI/build/license.rtf b/WebUI/build/license.rtf index 0eb34f30..859d9fee 100644 --- a/WebUI/build/license.rtf +++ b/WebUI/build/license.rtf @@ -1,18 +1,18 @@ Important AI Playground Notices and Disclaimers -Intel technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Your costs and results may vary. Intel does not control or audit third-party data. You should consult other sources to evaluate accuracy. Intel is committed to respecting human rights and avoiding causing or contributing to adverse impacts on human rights. See Intel’s Global Human Rights Principles (https://www.intel.com/content/www/us/en/policy/policy-human-rights.html). Intel’s products and software are intended only to be used in applications that do not cause or contribute to adverse impacts on human rights. The software may include third party components with separate legal notices or governed by other agreements, as may be described in the Third-Party Notices file accompanying the software. +Intel technologies may require enabled hardware, software or service activation. No product or component can be absolutely secure. Your costs and results may vary. Intel does not control or audit third-party data. You should consult other sources to evaluate accuracy. Intel is committed to respecting human rights and avoiding causing or contributing to adverse impacts on human rights. See Intel’s Global Human Rights Principles (https://www.intel.com/content/www/us/en/policy/policy-human-rights.html). Intel’s products and software are intended only to be used in applications that do not cause or contribute to adverse impacts on human rights. The software may include third party components with separate legal notices or governed by other agreements, as may be described in the Third-Party Notices file accompanying the software. -Data Privacy. Prompts and images being used in the application will not be collected or stored by Intel. The user of AI Playground is responsible for storing and processing any personal information using the app. For general information regarding the handling of personal data collected by Intel, refer to Intel’s Global Privacy Notice (https://www.intel.com/content/www/us/en/privacy/intelprivacy-notice.html). +Data Privacy. Prompts and images being used in the application will not be collected or stored by Intel. The user of AI Playground is responsible for storing and processing any personal information using the app. For general information regarding the handling of personal data collected by Intel, refer to Intel’s Global Privacy Notice (https://www.intel.com/content/www/us/en/privacy/intelprivacy-notice.html). Generative AI Large-Language Model (LLM)/Chatbot Disclaimers AI Playground utilizes GenAI technology and interactions with a chatbot. Best practices in such cases recommend that users at least: -Review outputs before distributing or taking action. -Take caution around incorrect attribution/explanation. -Be skeptical of tone. -Be cognizant of automation bias. -Assume responsibility for action taken. +Review outputs before distributing or taking action. +Take caution around incorrect attribution/explanation. +Be skeptical of tone. +Be cognizant of automation bias. +Assume responsibility for action taken. You are solely responsible for your use of output from your operation of the AI Playground. @@ -32,7 +32,7 @@ Sharing content that is an alteration of copyrighted or licensed material in vio Limitations. The model does not achieve perfect photorealism The model cannot render legible text -The model does not perform well on more difficult tasks which involve compositionality, such as rendering an image corresponding to “A red cube on top of a blue sphere” +The model does not perform well on more difficult tasks which involve compositionality, such as rendering an image corresponding to “A red cube on top of a blue sphere” Faces and people in general may not be generated properly. The model was trained mainly with English captions and will not work as well in other languages. The autoencoding part of the model is lossy @@ -43,25 +43,25 @@ Bias: While the capabilities of image generation models are impressive, they can Third-Party Models -In the course of using AI Playground, users may choose to download models created and distributed by third parties after reviewing background information about the models and agreeing to the license governing those models. +In the course of using AI Playground, users may choose to download models created and distributed by third parties after reviewing background information about the models and agreeing to the license governing those models. Notice: Intel does not create the content and does not warrant its accuracy or quality. By accessing the third-party content, or using materials trained on or with such content, you are indicating your acceptance of the terms associated with that content and warranting that your use complies with the applicable license. - + Intel expressly disclaims the accuracy, adequacy, or completeness of any such third-party content, and is not liable for any errors, omissions, or defects in the content, or for any reliance on the content. You agree Intel is not liable for any liability or damages relating to your use of third-party content. -Intel’s identification of these resources does not expand or otherwise alter Intel’s applicable published warranties or warranty disclaimers for Intel products or solutions, and you agree that no additional obligations, indemnifications, or liabilities arise from Intel identifying such resources. Intel reserves the right, without notice, to make corrections, enhancements, improvements, and other changes to its materials. +Intel’s identification of these resources does not expand or otherwise alter Intel’s applicable published warranties or warranty disclaimers for Intel products or solutions, and you agree that no additional obligations, indemnifications, or liabilities arise from Intel identifying such resources. Intel reserves the right, without notice, to make corrections, enhancements, improvements, and other changes to its materials. The table below contains links to the licenses for certain third-party models and detailed information about the capabilities, limitations, and best practices for those models. Model License Background Information/Model Card -Dreamshaper 8 Model https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/Lykon/dreamshaper-8 -Dreamshaper 8 Inpainting Model https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/Lykon/dreamshaper-8-inpainting +Dreamshaper 8 Model https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/Lykon/dreamshaper-8 +Dreamshaper 8 Inpainting Model https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/Lykon/dreamshaper-8-inpainting JuggernautXL v9 Model - https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/RunDiffusion/Juggernaut-XL-v9 -Phi3-mini-4k-instruct https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/LICENSE https://huggingface.co/microsoft/Phi-3-mini-4k-instruct -bge-large-en-v1.5 https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md https://huggingface.co/BAAI/bge-large-en-v1.5 -Latent Consistency Model (LCM) LoRA: SD1.5 https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENSE.md https://huggingface.co/latent-consistency/lcm-lora-sdv1-5 -Latent Consistency Model (LCM) LoRA:SDXL https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENSE.md https://huggingface.co/latent-consistency/lcm-lora-sdxl + https://huggingface.co/spaces/CompVis/stable-diffusion-license https://huggingface.co/RunDiffusion/Juggernaut-XL-v9 +Phi3-mini-4k-instruct https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/LICENSE https://huggingface.co/microsoft/Phi-3-mini-4k-instruct +bge-large-en-v1.5 https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md https://huggingface.co/BAAI/bge-large-en-v1.5 +Latent Consistency Model (LCM) LoRA: SD1.5 https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENSE.md https://huggingface.co/latent-consistency/lcm-lora-sdv1-5 +Latent Consistency Model (LCM) LoRA:SDXL https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENSE.md https://huggingface.co/latent-consistency/lcm-lora-sdxl -© Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. +© Intel Corporation.  Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries.  Other names and brands may be claimed as the property of others. diff --git a/WebUI/build/pack-offline.js b/WebUI/build/pack-offline.js deleted file mode 100644 index 7252a302..00000000 --- a/WebUI/build/pack-offline.js +++ /dev/null @@ -1,83 +0,0 @@ -// Usage: node pack-offline.js - -const fs = require('fs'); -const path = require('path'); -const AdmZip = require('adm-zip'); - -if (process.argv.length < 4) { - console.error('Usage: node pack-offline.js '); - process.exit(1); -} - -const packageResDir = path.resolve(process.argv[2]); -const platform = process.argv[3]; - -if (!fs.existsSync(packageResDir)) { - console.error('Directory not found:', packageResDir); - process.exit(1); -} - -const sevenZipExe = path.join(packageResDir, '7zr.exe'); -if (!fs.existsSync(sevenZipExe)) { - console.error('7zr.exe not found:', sevenZipExe); - process.exit(1); -} - -const baseEnvArchive = path.join(packageResDir, 'env.7z'); -if (!fs.existsSync(baseEnvArchive)) { - console.error('env.7z not found:', baseEnvArchive); - process.exit(1); -} -const workDir = path.join(__dirname, `env-${platform}`); -if (fs.existsSync(workDir)) { - console.warn("Removing existing offline env directory:", workDir); - fs.rmSync(workDir, { recursive: true }); -} - -const spawnSync = require('child_process').spawnSync; -const unzip = spawnSync(sevenZipExe, ['x', baseEnvArchive, `-o${workDir}`]); -console.log(unzip.stdout.toString()); -console.error(unzip.stderr.toString()); -if (unzip.status !== 0) { - console.error('Failed to extract env.7z'); - process.exit(1); -} - -const offlineEnvDir = path.join(workDir, 'env'); -const pythonExe = path.join(offlineEnvDir, 'python.exe'); -if (!fs.existsSync(pythonExe)) { - console.error('python.exe not found:', pythonExe); - process.exit(1); -} - -const requirementsTxt = path.join(__dirname, '..', '..', 'service', `requirements-${platform}.txt`); -if (!fs.existsSync(requirementsTxt)) { - console.error('requirements.txt not found:', requirementsTxt); - process.exit(1); -} - -const spawn = require('child_process').spawn; -const pipInstall = spawn(pythonExe, ['-m', 'pip', 'install', '-r', requirementsTxt], { cwd: offlineEnvDir }); -pipInstall.stdout.on('data', (data) => { - console.log(data.toString()); -}); -pipInstall.stderr.on('data', (data) => { - console.error(data.toString()); -}); -pipInstall.on('close', (code) => { - if (code !== 0) { - console.error('Failed to install requirements'); - process.exit(1); - } - - const offlineEnvArchive = path.join(packageResDir, `env-offline-${platform}.7z`); - const zip = spawnSync(sevenZipExe, ['a', offlineEnvArchive, offlineEnvDir]); - console.log(zip.stdout.toString()); - console.error(zip.stderr.toString()); - if (zip.status !== 0) { - console.error('Failed to compress offline env directory'); - process.exit(1); - } - - console.log('Offline env has been created successfully:', offlineEnvArchive); -}); diff --git a/WebUI/build/pack-python.js b/WebUI/build/pack-python.js deleted file mode 100644 index 1191819a..00000000 --- a/WebUI/build/pack-python.js +++ /dev/null @@ -1,100 +0,0 @@ -// Usage: node pack-python.js - -const fs = require('fs'); -const path = require('path'); -const AdmZip = require('adm-zip'); - -if (process.argv.length < 6) { - console.error('Usage: node pack-python.js '); - process.exit(1); -} - -const packageResDir = path.resolve(process.argv[2]); -const pythonEmbedZip = path.resolve(process.argv[3]); -const getPipFile = path.resolve(process.argv[4]); -const referenceCondaEnv = path.resolve(process.argv[5]); - -if (!fs.existsSync(pythonEmbedZip)) { - console.error('File not found:', pythonEmbedZip); - process.exit(1); -} - -// unzip python embed -const pythonEmbed = new AdmZip(pythonEmbedZip); -const pythonEmbedDir = path.join(__dirname, 'env'); -if (fs.existsSync(pythonEmbedDir)) { - console.warn("Removing existing python env directory:", pythonEmbedDir); - fs.rmSync(pythonEmbedDir, { recursive: true }); -} -pythonEmbed.extractAllTo(pythonEmbedDir, true); -console.log('Extracted python embed to:', pythonEmbedDir); - -// copy get-pip.py -const getPipDest = path.join(pythonEmbedDir, 'get-pip.py'); -fs.copyFileSync(getPipFile, getPipDest); -console.log('Copied get-pip.py to:', getPipDest); - -// execute env/python.exe get-pip.py -const spawnSync = require('child_process').spawnSync; -const getpip = spawnSync(path.join(pythonEmbedDir, 'python.exe'), [getPipDest]); -console.log(getpip.stdout.toString()); -console.error(getpip.stderr.toString()); -if (getpip.status !== 0) { - console.error('Failed to run get-pip.py'); - process.exit(1); -} - -// check whether libuv has been installed in the reference conda env -const condaBinDir = path.join(referenceCondaEnv, 'Library', 'bin'); -const uvDll = path.join(condaBinDir, 'uv.dll'); -if (!fs.existsSync(uvDll)) { - console.error('libuv.dll not found in reference conda env:', uvDll); - process.exit(1); -} - -// copy conda dlls from Library/bin to env -const condaDlls = fs.readdirSync(condaBinDir); -for (const condaDll of condaDlls) { - const src = path.join(condaBinDir, condaDll); - const dest = path.join(pythonEmbedDir, condaDll); - fs.copyFileSync(src, dest); - console.log('Copied conda dll to:', dest); -} - -// write custom content to env/python311._pth -const pthFile = path.join(pythonEmbedDir, 'python311._pth'); -const pthContent = ` -python311.zip -. -../service - -# Uncomment to run site.main() automatically -import site -`; -fs.writeFileSync(pthFile, pthContent); -console.log('Wrote custom content to:', pthFile); - -// 7z compress the env directory -// create folder if not exists -if (!fs.existsSync(packageResDir)) { - fs.mkdirSync(packageResDir, { recursive: true }); -} -const sevenZipExe = path.join(packageResDir, '7zr.exe'); -if (!fs.existsSync(sevenZipExe)) { - console.error('7zr.exe not found:', sevenZipExe); - process.exit(1); -} -const outputFile = path.join(packageResDir, 'env.7z'); -if (fs.existsSync(outputFile)) { - console.warn("Removing existing 7z file:", outputFile); - fs.rmSync(outputFile); -} -const zip = spawnSync(sevenZipExe, ['a', outputFile, pythonEmbedDir]); -console.log(zip.stdout.toString()); -console.error(zip.stderr.toString()); -if (zip.status !== 0) { - console.error('Failed to compress env directory'); - process.exit(1); -} - -console.log('Compressed env directory to:', outputFile); diff --git a/WebUI/build/scripts/fetch-python-package-resources.js b/WebUI/build/scripts/fetch-python-package-resources.js new file mode 100644 index 00000000..7b6d6c0b --- /dev/null +++ b/WebUI/build/scripts/fetch-python-package-resources.js @@ -0,0 +1,69 @@ +// Usage: node fetch-python-package-resources.js +const https = require('https'); +const fs = require('fs'); +const path = require('path'); + + +if (process.argv.length < 3) { + + console.error('Usage: fetch-python-package-resources.js '); + process.exit(1); +} + +const targetDir = path.resolve(process.argv[2]); + +const embeddablePythonUrl = 'https://raw.githubusercontent.com/adang1345/PythonWindows/master/3.11.10/python-3.11.10-embed-amd64.zip'; +const getPipScriptUrl = 'https://bootstrap.pypa.io/get-pip.py' +const sevenZrExeUrl = 'https://www.7-zip.org/a/7zr.exe' + +function fetchFileIfNotPresent(url, targetDir) { + const expectedFilePath = path.join(targetDir, getBaseFileName(url)) + if (fs.existsSync(expectedFilePath)) { + console.log(`omitting fetching of ${url} as ${expectedFilePath} already exists`) + } else { + fetchFile(url, targetDir) + } +} + +function fetchFile(url, targetDir) { + https.get(url, (response) => { + const filePath = path.join(targetDir, getBaseFileName(url)) + const file = fs.createWriteStream(filePath); + response.pipe(file); + + file.on('finish', () => { + file.close(); + console.log(`Downloaded ${filePath} successfully!`); + }); + }).on('error', (err) => { + console.error(`Error downloading ${embeddablePythonUrl}: ${err}`); + }); +} + + +function getBaseFileName(url) { + const urlPathSegments = url.split('/'); + const baseFileName = urlPathSegments[urlPathSegments.length - 1] + return baseFileName; +} + +function prepareTargetPath() { + if (!fs.existsSync(targetDir)) { + fs.mkdirSync(targetDir, { recursive: true }); + } +} + +function provideLibuvDlls() { + console.error("provideLibuvDlls is currently only mocked") + console.error(`please simlink to conda virtual env into ${targetDir} manually`) +} + +function main() { + prepareTargetPath() + fetchFileIfNotPresent(embeddablePythonUrl, targetDir) + fetchFileIfNotPresent(getPipScriptUrl, targetDir) + fetchFileIfNotPresent(sevenZrExeUrl, targetDir) + provideLibuvDlls() +} + +main() diff --git a/WebUI/build/scripts/pack-offline.js b/WebUI/build/scripts/pack-offline.js new file mode 100644 index 00000000..14e29f1c --- /dev/null +++ b/WebUI/build/scripts/pack-offline.js @@ -0,0 +1,90 @@ +// Usage: node pack-offline.js +// +const fs = require('fs'); +const path = require('path'); +const AdmZip = require('adm-zip'); +const childProcess = require('child_process'); + +if (process.argv.length < 4) { + console.error('Usage: node pack-offline.js '); + process.exit(1); +} + +const packageResDir = existingFileOrExit(path.resolve(process.argv[2])); +const platform = process.argv[3]; + +const sevenZipExe = existingFileOrExit(path.join(packageResDir, '7zr.exe')); +const zippedPyenv = existingFileOrExit(path.join(packageResDir, 'env.7z')); + +function existingFileOrExit(filePath) { + if (!fs.existsSync(filePath)) { + console.error('Resource not found:', filePath); + process.exit(1); + } + return filePath +} + + +function unzippedPackagedPyenv(pyenvArchive) { + existingFileOrExit(pyenvArchive) + const pyenvTargetDir = workDir + + if (fs.existsSync(pyenvTargetDir)) { + console.warn("Removing existing offline env directory:", pyenvTargetDir); + fs.rmSync(workDir, { recursive: true }); + } + + const unzip = childProcess.spawnSync(sevenZipExe, ['x', pyenvArchive, `-o${workDir}`]); + console.log(unzip.stdout.toString()); + console.error(unzip.stderr.toString()); + if (unzip.status !== 0) { + console.error('Failed to extract env.7z'); + process.exit(1); + } + + const offlineEnvDir = existingFileOrExit(path.join(workDir, 'env')); + return offlineEnvDir +} + +function runPipInstall(requirementsFilePath, workingDir) { + const pipInstall = spawn(pythonExe, ['-m', 'pip', 'install', '-r', requirementsFilePath], { cwd: workingDir }); + pipInstall.stdout.on('data', (data) => { + console.log(data.toString()); + }); + pipInstall.stderr.on('data', (data) => { + console.error(data.toString()); + }); + pipInstall.on('close', (code) => { + if (code !== 0) { + console.error('Failed to install requirements'); + process.exit(1); + } + }); +} + +function zipPyenv(pyEnvDir) { + const offlineEnvArchiveTargetPath = path.join(packageResDir, `env-offline-${platform}.7z`); + const zip = childProcess.spawnSync((sevenZipExe, ['a', offlineEnvArchiveTargetPath, existingFileOrExit(pyEnvDir)]); + console.log(zip.stdout.toString()); + console.error(zip.stderr.toString()); + if (zip.status !== 0) { + console.error('Failed to compress offline env directory'); + process.exit(1); + } + + console.log('Offline env has been created successfully:', offlineEnvArchiveTargetPath); +} + +function main() { + const workDir = path.join(__dirname, `env-${platform}`); + + const offlineEnvDir = unzippedPackagedPyenv(zippedPyenv) + const pythonExe = existingFileOrExit(path.join(offlineEnvDir, 'python.exe')); + + const platformSpecificRequirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'service', `requirements-${platform}.txt`)); + const requirementsTxt = existingFileOrExit(path.join(__dirname, '..', '..', 'service', `requirements-${platform}.txt`)); + runPipInstall(platformSpecificRequirementsTxt, offlineEnvDir) + runPipInstall(requirementsTxt, offlineEnvDir) + + zipPyenv(offlineEnvDir) +} \ No newline at end of file diff --git a/WebUI/build/scripts/pack-python.js b/WebUI/build/scripts/pack-python.js new file mode 100644 index 00000000..c05174ae --- /dev/null +++ b/WebUI/build/scripts/pack-python.js @@ -0,0 +1,149 @@ +// Usage: node pack-python.js + +const fs = require('fs'); +const path = require('path'); +const AdmZip = require('adm-zip'); +const childProcess = require('child_process'); + +if (process.argv.length < 3) { + console.error('Usage: node pack-python.js '); + process.exit(1); +} + +const pythonPackageResourcesDir = path.resolve(process.argv[2]); +const targetResDir = path.resolve(process.argv[3]); +const webUIBuildDir = path.join(__dirname, '..', '..'); +const pythonEmbedDir = path.join(webUIBuildDir, '..', 'env'); + +const packageResourceFiles = fs.readdirSync(pythonPackageResourcesDir); +const pythonEmbedZipFile = path.join(pythonPackageResourcesDir, packageResourceFiles.find((fileName) => { return fileName.startsWith('python') && fileName.endsWith('.zip') })); +const condaDir = path.join(pythonPackageResourcesDir, packageResourceFiles.find((fileName) => { return fileName.includes('conda') })); +const condaBinDir = path.join(condaDir, 'Library', 'bin'); +const getPipFile = path.join(pythonPackageResourcesDir, 'get-pip.py'); +const sevenZipExe = path.join(pythonPackageResourcesDir, '7zr.exe') +const sevenZipBinary = () => { + if (childProcess.execSync('which 7z').toString().includes('7z')) { + return "7z"; + } else if (fs.existsSync(sevenZipExe)) { + return sevenZipExe; + } else { + console.error('No 7z executable found'); + process.exit(1); + } +} + +function verifyFilesExist() { + console.log('verifying all required files exist.') + if (!fs.existsSync(pythonEmbedZipFile)) { + console.error('File not found:', pythonEmbedZipFile); + process.exit(1); + } + + if (!fs.existsSync(getPipFile)) { + console.error('File not found:', getPipFile); + process.exit(1); + } + + if (!fs.existsSync(sevenZipExe)) { + console.error('File not found:', sevenZipExe); + process.exit(1); + } + + // check whether libuv has been installed in the conda env + const uvDll = path.join(condaBinDir, 'uv.dll'); + if (!fs.existsSync(uvDll)) { + console.error('libuv.dll not found in reference conda env:', uvDll); + process.exit(1); + } + console.log('all required files exist.') + + if (!fs.existsSync(targetResDir)) { + console.log(`Creating missing target dir: ${targetResDir}`) + fs.mkdirSync(targetResDir, { recursive: true }); + } +} + +function preparePythonEnvDir(pyEnvTargetPath) { + if (fs.existsSync(pyEnvTargetPath)) { + console.warn("Removing existing python env directory:", pyEnvTargetPath); + fs.rmSync(pyEnvTargetPath, {recursive: true}); + } +} + +function createPythonEnvFromEmbedabblePythonZip() { + const pyEnvTargetPath = pythonEmbedDir + preparePythonEnvDir(pyEnvTargetPath); + console.log('Creating python env.') + const pythonEmbed = new AdmZip(pythonEmbedZipFile); + pythonEmbed.extractAllTo(pyEnvTargetPath, true); + console.log('Extracted embeddable python to:', pyEnvTargetPath); + + // configure path of python env: + console.log('Patching path of python environment'); + const pthFile = path.join(pyEnvTargetPath, 'python311._pth'); + const pthContent = ` +python311.zip +. +../service + +# Uncomment to run site.main() automatically +import site +`; + fs.writeFileSync(pthFile, pthContent); + console.log('patched python paths'); + + console.log('Copying get-pip.py'); + const getPipDest = path.join(pyEnvTargetPath, 'get-pip.py'); + fs.copyFileSync(getPipFile, getPipDest); + console.log('Copied get-pip.py to:', getPipDest); + return pyEnvTargetPath; +} + +function patchCondaDllsIntoPythonEnv(pyEnvDirPath) { + console.log('Copying conda dlls to python env'); + + for (const condaDll of fs.readdirSync(condaBinDir)) { + const src = path.join(condaBinDir, condaDll); + const dest = path.join(pyEnvDirPath, condaDll); + fs.copyFileSync(src, dest); + } + console.log('Copied conda dlls into:', pyEnvDirPath); +} + +function compressPythonEnvDirectory(pyEnvDirPath) { + const outputFile = path.join(pythonEmbedDir, 'env.7z'); + console.log(`7zipping env directory into ${outputFile}`) + if (fs.existsSync(outputFile)) { + console.warn("Removing existing 7z file:", outputFile); + fs.rmSync(outputFile); + } + + const zip = childProcess.spawnSync(sevenZipBinary(), ['a', outputFile, pyEnvDirPath]); + console.log(zip.stdout.toString()); + console.error(zip.stderr.toString()); + if (zip.status !== 0) { + console.error('Failed to compress env directory'); + process.exit(1); + } + + console.log('Compressed env directory to:', outputFile); + return outputFile +} + +function copyToTargetDir(filePaths) { + for (const sourceFilePath of filePaths) { + const destinationPath = path.join(targetResDir, path.basename(sourceFilePath)); + fs.copyFileSync(sourceFilePath, destinationPath); + console.log(`copied ${path.basename(sourceFilePath)} into ${targetResDir}`) + } +} + +function main() { + verifyFilesExist(); + const pyEnvPath = createPythonEnvFromEmbedabblePythonZip(); + patchCondaDllsIntoPythonEnv(pyEnvPath); + const sevenZippedPyenv = compressPythonEnvDirectory(pyEnvPath); + copyToTargetDir([sevenZippedPyenv, sevenZipExe]); +} + +main(); diff --git a/WebUI/build/prebuild.js b/WebUI/build/scripts/prebuild.js similarity index 100% rename from WebUI/build/prebuild.js rename to WebUI/build/scripts/prebuild.js diff --git a/WebUI/build/render-template.js b/WebUI/build/scripts/render-template.js similarity index 86% rename from WebUI/build/render-template.js rename to WebUI/build/scripts/render-template.js index 5eebdecf..bd328932 100644 --- a/WebUI/build/render-template.js +++ b/WebUI/build/scripts/render-template.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); - +const buildDir = path.join(__dirname, '..') // Function to render the template with environment variables function renderTemplate(templatePath, outputPath, variables) { // Read the template content @@ -29,8 +29,8 @@ function renderTemplate(templatePath, outputPath, variables) { } // Example usage -const templatePath = path.join(__dirname, 'installer.nsh.template'); -const outputPath = path.join(__dirname, 'installer.nsh'); +const templatePath = path.join(buildDir, 'installer.nsh.template'); +const outputPath = path.join(buildDir, 'installer.nsh'); const variables = { PLATFORM: process.env.PLATFORM || 'arc' }; diff --git a/WebUI/package-lock.json b/WebUI/package-lock.json index f617524b..0e4e5a80 100644 --- a/WebUI/package-lock.json +++ b/WebUI/package-lock.json @@ -1,12 +1,12 @@ { "name": "ai-playground", - "version": "1.22.0-beta", + "version": "1.22.1-beta", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ai-playground", - "version": "1.22.0-beta", + "version": "1.22.1-beta", "dependencies": { "@radix-icons/vue": "^1.0.0", "@vueuse/core": "^11.1.0", diff --git a/WebUI/package.json b/WebUI/package.json index 30cb2044..9272e21e 100644 --- a/WebUI/package.json +++ b/WebUI/package.json @@ -4,14 +4,15 @@ "version": "1.22.1-beta", "scripts": { "dev": "cross-env VITE_PLATFORM_TITLE=\"for Local® Dev™ Mode\" vite", - "pack-python": "node build\\pack-python.js .\\package_res", - "pack-offline": "node .\\build\\pack-offline.js .\\package_res", - "prebuild": "node build\\prebuild.js .\\package_res ..\\service .\\external", - "build": "npm run prebuild && npm run build:arc && npm run build:ultra && npm run build:ultra2", - "build:arc": "cross-env-shell PLATFORM=\"arc\" VITE_PLATFORM_TITLE=\"for Intel® Arc™\" \"vue-tsc && vite build && node .\\build\\render-template.js && electron-builder --config build\\build-config.json\"", - "build:ultra": "cross-env-shell PLATFORM=\"ultra\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra\" \"vue-tsc && vite build && node .\\build\\render-template.js && electron-builder --config build\\build-config.json\"", - "build:ultra2": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"vue-tsc && vite build && node .\\build\\render-template.js && electron-builder --config build\\build-config.json\"", - "build:ultra2-offline": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"npm run pack-offline ultra2 && npm run prebuild && vue-tsc && vite build && electron-builder --config build\\build-config-offline.json\"", + "fetch-build-resources": "cross-env node ./build/scripts/fetch-python-package-resources.js ../python_package_res", + "pack-offline": "cross-env node build/scripts/pack-offline.js ./package_res", + "pack-python": "cross-env node build/scripts/pack-python.js ../python_package_res ./npm_package_res", + "prebuild": "cross-env node build/scripts/prebuild.js ./npm_package_res ../service ./external", + "build:all": "cross-env npm run prebuild && npm run build:arc && npm run build:ultra && npm run build:ultra2", + "build:arc": "cross-env-shell PLATFORM=\"arc\" VITE_PLATFORM_TITLE=\"for Intel® Arc™\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", + "build:ultra": "cross-env-shell PLATFORM=\"ultra\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", + "build:ultra2": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"vue-tsc && vite build && node ./build/scripts/render-template.js && electron-builder --config build/build-config.json --win --x64\"", + "build:ultra2-offline": "cross-env-shell PLATFORM=\"ultra2\" VITE_PLATFORM_TITLE=\"for Intel® Core™ Ultra Series 2\" \"npm run pack-offline ultra2 && npm run prebuild && vue-tsc && vite build && electron-builder --config build/build-config-offline.json --win --x64\"", "preview": "vite preview" }, "dependencies": { diff --git a/service/requirements-arc.txt b/service/requirements-arc.txt index 1d06d494..dc51b802 100644 --- a/service/requirements-arc.txt +++ b/service/requirements-arc.txt @@ -1,5 +1,3 @@ --r requirements.txt - # IPEX --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ diff --git a/service/requirements-ultra.txt b/service/requirements-ultra.txt index f90fc2a7..dd8019ce 100644 --- a/service/requirements-ultra.txt +++ b/service/requirements-ultra.txt @@ -1,5 +1,3 @@ --r requirements.txt - # IPEX --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/mtl/us/ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/mtl/cn/ diff --git a/service/requirements-ultra2.txt b/service/requirements-ultra2.txt index a6e55599..ecbd22d9 100644 --- a/service/requirements-ultra2.txt +++ b/service/requirements-ultra2.txt @@ -1,5 +1,3 @@ --r requirements.txt - # IPEX --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/lnl/us/ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/lnl/cn/