From 62dd15e201f40a713d8ae6a5c4481b17393f19ab Mon Sep 17 00:00:00 2001 From: Mishig Date: Wed, 15 May 2024 16:43:27 +0200 Subject: [PATCH] Revert "Generic Multimodal Support" (#1146) Revert "Generic Multimodal Support (#1021)" This reverts commit 57f89346cc6d953246d0f1705e342e2e66fce386. --- README.md | 27 +- package-lock.json | 399 +++++++----------- package.json | 7 +- src/lib/components/chat/ChatMessage.svelte | 8 +- src/lib/components/chat/ChatWindow.svelte | 8 +- .../endpoints/anthropic/endpointAnthropic.ts | 30 +- .../anthropic/endpointAnthropicVertex.ts | 28 +- src/lib/server/endpoints/anthropic/utils.ts | 44 -- src/lib/server/endpoints/endpoints.ts | 4 +- src/lib/server/endpoints/images.ts | 211 --------- .../server/endpoints/openai/endpointOai.ts | 80 +--- .../server/endpoints/preprocessMessages.ts | 56 --- src/lib/server/endpoints/tgi/endpointTgi.ts | 50 +-- src/lib/server/files/downloadFile.ts | 7 +- src/lib/server/files/uploadFile.ts | 16 +- src/lib/server/generateFromDefaultEndpoint.ts | 4 +- src/lib/server/models.ts | 2 +- src/lib/server/preprocessMessages.ts | 61 +++ src/lib/server/summarize.ts | 4 +- .../server/websearch/search/generateQuery.ts | 3 +- src/lib/types/Message.ts | 12 +- src/lib/utils/messageUpdates.ts | 3 +- src/routes/conversation/[id]/+page.svelte | 31 +- src/routes/conversation/[id]/+server.ts | 74 ++-- .../[id]/output/[sha256]/+server.ts | 4 +- 25 files changed, 342 insertions(+), 831 deletions(-) delete mode 100644 src/lib/server/endpoints/anthropic/utils.ts delete mode 100644 src/lib/server/endpoints/images.ts delete mode 100644 src/lib/server/endpoints/preprocessMessages.ts create mode 100644 src/lib/server/preprocessMessages.ts diff --git a/README.md b/README.md index ec1bdae4e1e..db742297db3 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ The following is the default `chatPromptTemplate`, although newlines and indenti #### Multi modal model -We currently support [IDEFICS](https://huggingface.co/blog/idefics) (hosted on TGI), OpenAI and Claude 3 as multimodal models. You can enable it by setting `multimodal: true` in your `MODELS` configuration. For IDEFICS, you must have a [PRO HF Api token](https://huggingface.co/settings/tokens). For OpenAI, see the [OpenAI section](#OpenAI). For Anthropic, see the [Anthropic section](#Anthropic). +We currently only support IDEFICS as a multimodal model, hosted on TGI. You can enable it by using the following config (if you have a PRO HF Api token): ```env { @@ -465,34 +465,14 @@ MODELS=`[ #### Anthropic -We also support Anthropic models (including multimodal ones via `multmodal: true`) through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example. +We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example. ``` MODELS=`[ - { - "name": "claude-3-haiku-20240307", - "displayName": "Claude 3 Haiku", - "description": "Fastest and most compact model for near-instant responsiveness", - "multimodal": true, - "parameters": { - "max_new_tokens": 4096, - }, - "endpoints": [ - { - "type": "anthropic", - // optionals - "apiKey": "sk-ant-...", - "baseURL": "https://api.anthropic.com", - "defaultHeaders": {}, - "defaultQuery": {} - } - ] - }, { "name": "claude-3-sonnet-20240229", "displayName": "Claude 3 Sonnet", "description": "Ideal balance of intelligence and speed", - "multimodal": true, "parameters": { "max_new_tokens": 4096, }, @@ -511,7 +491,6 @@ MODELS=`[ "name": "claude-3-opus-20240229", "displayName": "Claude 3 Opus", "description": "Most powerful model for highly complex tasks", - "multimodal": true, "parameters": { "max_new_tokens": 4096 }, @@ -537,7 +516,6 @@ MODELS=`[ "name": "claude-3-sonnet@20240229", "displayName": "Claude 3 Sonnet", "description": "Ideal balance of intelligence and speed", - "multimodal": true, "parameters": { "max_new_tokens": 4096, }, @@ -556,7 +534,6 @@ MODELS=`[ "name": "claude-3-haiku@20240307", "displayName": "Claude 3 Haiku", "description": "Fastest, most compact model for near-instant responsiveness", - "multimodal": true, "parameters": { "max_new_tokens": 4096 }, diff --git a/package-lock.json b/package-lock.json index b82e2f623ad..2c3e8aad09f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,14 +13,13 @@ "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", - "@resvg/resvg-js": "^2.6.2", + "@resvg/resvg-js": "^2.6.0", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.19.2", - "file-type": "^19.0.0", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", @@ -42,7 +41,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.3", + "sharp": "^0.33.2", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -89,7 +88,7 @@ "@google-cloud/vertexai": "^1.1.0", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", - "openai": "^4.44.0" + "openai": "^4.14.2" } }, "node_modules/@alloc/quick-lru": { @@ -238,9 +237,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", - "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==", + "version": "0.45.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz", + "integrity": "sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==", "optional": true, "dependencies": { "tslib": "^2.4.0" @@ -812,9 +811,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz", - "integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz", + "integrity": "sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==", "cpu": [ "arm64" ], @@ -833,13 +832,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.2" + "@img/sharp-libvips-darwin-arm64": "1.0.1" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz", - "integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz", + "integrity": "sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==", "cpu": [ "x64" ], @@ -858,13 +857,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.2" + "@img/sharp-libvips-darwin-x64": "1.0.1" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz", - "integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz", + "integrity": "sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==", "cpu": [ "arm64" ], @@ -883,9 +882,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", - "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz", + "integrity": "sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==", "cpu": [ "x64" ], @@ -904,9 +903,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", - "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz", + "integrity": "sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==", "cpu": [ "arm" ], @@ -925,9 +924,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", - "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz", + "integrity": "sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==", "cpu": [ "arm64" ], @@ -946,9 +945,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", - "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz", + "integrity": "sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==", "cpu": [ "s390x" ], @@ -967,9 +966,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", - "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz", + "integrity": "sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==", "cpu": [ "x64" ], @@ -988,9 +987,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", - "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz", + "integrity": "sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==", "cpu": [ "arm64" ], @@ -1009,9 +1008,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", - "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz", + "integrity": "sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==", "cpu": [ "x64" ], @@ -1030,9 +1029,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz", - "integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz", + "integrity": "sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==", "cpu": [ "arm" ], @@ -1051,13 +1050,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.2" + "@img/sharp-libvips-linux-arm": "1.0.1" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz", - "integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz", + "integrity": "sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==", "cpu": [ "arm64" ], @@ -1076,13 +1075,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.2" + "@img/sharp-libvips-linux-arm64": "1.0.1" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz", - "integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz", + "integrity": "sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==", "cpu": [ "s390x" ], @@ -1101,13 +1100,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.2" + "@img/sharp-libvips-linux-s390x": "1.0.1" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz", - "integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz", + "integrity": "sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==", "cpu": [ "x64" ], @@ -1126,13 +1125,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.2" + "@img/sharp-libvips-linux-x64": "1.0.1" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz", - "integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz", + "integrity": "sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==", "cpu": [ "arm64" ], @@ -1151,13 +1150,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" + "@img/sharp-libvips-linuxmusl-arm64": "1.0.1" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz", - "integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz", + "integrity": "sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==", "cpu": [ "x64" ], @@ -1176,19 +1175,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.2" + "@img/sharp-libvips-linuxmusl-x64": "1.0.1" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz", - "integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz", + "integrity": "sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==", "cpu": [ "wasm32" ], "optional": true, "dependencies": { - "@emnapi/runtime": "^1.1.0" + "@emnapi/runtime": "^0.45.0" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0", @@ -1201,9 +1200,9 @@ } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz", - "integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz", + "integrity": "sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==", "cpu": [ "ia32" ], @@ -1222,9 +1221,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz", - "integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz", + "integrity": "sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==", "cpu": [ "x64" ], @@ -1445,31 +1444,31 @@ "integrity": "sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==" }, "node_modules/@resvg/resvg-js": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz", - "integrity": "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.0.tgz", + "integrity": "sha512-Tf3YpbBKcQn991KKcw/vg7vZf98v01seSv6CVxZBbRkL/xyjnoYB6KgrFL6zskT1A4dWC/vg77KyNOW+ePaNlA==", "engines": { "node": ">= 10" }, "optionalDependencies": { - "@resvg/resvg-js-android-arm-eabi": "2.6.2", - "@resvg/resvg-js-android-arm64": "2.6.2", - "@resvg/resvg-js-darwin-arm64": "2.6.2", - "@resvg/resvg-js-darwin-x64": "2.6.2", - "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2", - "@resvg/resvg-js-linux-arm64-gnu": "2.6.2", - "@resvg/resvg-js-linux-arm64-musl": "2.6.2", - "@resvg/resvg-js-linux-x64-gnu": "2.6.2", - "@resvg/resvg-js-linux-x64-musl": "2.6.2", - "@resvg/resvg-js-win32-arm64-msvc": "2.6.2", - "@resvg/resvg-js-win32-ia32-msvc": "2.6.2", - "@resvg/resvg-js-win32-x64-msvc": "2.6.2" + "@resvg/resvg-js-android-arm-eabi": "2.6.0", + "@resvg/resvg-js-android-arm64": "2.6.0", + "@resvg/resvg-js-darwin-arm64": "2.6.0", + "@resvg/resvg-js-darwin-x64": "2.6.0", + "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.0", + "@resvg/resvg-js-linux-arm64-gnu": "2.6.0", + "@resvg/resvg-js-linux-arm64-musl": "2.6.0", + "@resvg/resvg-js-linux-x64-gnu": "2.6.0", + "@resvg/resvg-js-linux-x64-musl": "2.6.0", + "@resvg/resvg-js-win32-arm64-msvc": "2.6.0", + "@resvg/resvg-js-win32-ia32-msvc": "2.6.0", + "@resvg/resvg-js-win32-x64-msvc": "2.6.0" } }, "node_modules/@resvg/resvg-js-android-arm-eabi": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.2.tgz", - "integrity": "sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.0.tgz", + "integrity": "sha512-lJnZ/2P5aMocrFMW7HWhVne5gH82I8xH6zsfH75MYr4+/JOaVcGCTEQ06XFohGMdYRP3v05SSPLPvTM/RHjxfA==", "cpu": [ "arm" ], @@ -1482,9 +1481,9 @@ } }, "node_modules/@resvg/resvg-js-android-arm64": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.2.tgz", - "integrity": "sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.0.tgz", + "integrity": "sha512-N527f529bjMwYWShZYfBD60dXA4Fux+D695QsHQ93BDYZSHUoOh1CUGUyICevnTxs7VgEl98XpArmUWBZQVMfQ==", "cpu": [ "arm64" ], @@ -1497,9 +1496,9 @@ } }, "node_modules/@resvg/resvg-js-darwin-arm64": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.2.tgz", - "integrity": "sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.0.tgz", + "integrity": "sha512-MabUKLVayEwlPo0mIqAmMt+qESN8LltCvv5+GLgVga1avpUrkxj/fkU1TKm8kQegutUjbP/B0QuMuUr0uhF8ew==", "cpu": [ "arm64" ], @@ -1512,9 +1511,9 @@ } }, "node_modules/@resvg/resvg-js-darwin-x64": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.2.tgz", - "integrity": "sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.0.tgz", + "integrity": "sha512-zrFetdnSw/suXjmyxSjfDV7i61hahv6DDG6kM7BYN2yJ3Es5+BZtqYZTcIWogPJedYKmzN1YTMWGd/3f0ubFiA==", "cpu": [ "x64" ], @@ -1527,9 +1526,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm-gnueabihf": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.2.tgz", - "integrity": "sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.0.tgz", + "integrity": "sha512-sH4gxXt7v7dGwjGyzLwn7SFGvwZG6DQqLaZ11MmzbCwd9Zosy1TnmrMJfn6TJ7RHezmQMgBPi18bl55FZ1AT4A==", "cpu": [ "arm" ], @@ -1542,9 +1541,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm64-gnu": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.2.tgz", - "integrity": "sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.0.tgz", + "integrity": "sha512-fCyMncqCJtrlANADIduYF4IfnWQ295UKib7DAxFXQhBsM9PLDTpizr0qemZcCNadcwSVHnAIzL4tliZhCM8P6A==", "cpu": [ "arm64" ], @@ -1557,9 +1556,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm64-musl": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.2.tgz", - "integrity": "sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.0.tgz", + "integrity": "sha512-ouLjTgBQHQyxLht4FdMPTvuY8xzJigM9EM2Tlu0llWkN1mKyTQrvYWi6TA6XnKdzDJHy7ZLpWpjZi7F5+Pg+Vg==", "cpu": [ "arm64" ], @@ -1572,9 +1571,9 @@ } }, "node_modules/@resvg/resvg-js-linux-x64-gnu": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.2.tgz", - "integrity": "sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.0.tgz", + "integrity": "sha512-n3zC8DWsvxC1AwxpKFclIPapDFibs5XdIRoV/mcIlxlh0vseW1F49b97F33BtJQRmlntsqqN6GMMqx8byB7B+Q==", "cpu": [ "x64" ], @@ -1587,9 +1586,9 @@ } }, "node_modules/@resvg/resvg-js-linux-x64-musl": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.2.tgz", - "integrity": "sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.0.tgz", + "integrity": "sha512-n4tasK1HOlAxdTEROgYA1aCfsEKk0UOFDNd/AQTTZlTmCbHKXPq+O8npaaKlwXquxlVK8vrkcWbksbiGqbCAcw==", "cpu": [ "x64" ], @@ -1602,9 +1601,9 @@ } }, "node_modules/@resvg/resvg-js-win32-arm64-msvc": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.2.tgz", - "integrity": "sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.0.tgz", + "integrity": "sha512-X2+EoBJFwDI5LDVb51Sk7ldnVLitMGr9WwU/i21i3fAeAXZb3hM16k67DeTy16OYkT2dk/RfU1tP1wG+rWbz2Q==", "cpu": [ "arm64" ], @@ -1617,9 +1616,9 @@ } }, "node_modules/@resvg/resvg-js-win32-ia32-msvc": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.2.tgz", - "integrity": "sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.0.tgz", + "integrity": "sha512-L7oevWjQoUgK5W1fCKn0euSVemhDXVhrjtwqpc7MwBKKimYeiOshO1Li1pa8bBt5PESahenhWgdB6lav9O0fEg==", "cpu": [ "ia32" ], @@ -1632,9 +1631,9 @@ } }, "node_modules/@resvg/resvg-js-win32-x64-msvc": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.2.tgz", - "integrity": "sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.0.tgz", + "integrity": "sha512-8lJlghb+Unki5AyKgsnFbRJwkEj9r1NpwyuBG8yEJiG1W9eEGl03R3I7bsVa3haof/3J1NlWf0rzSa1G++A2iw==", "cpu": [ "x64" ], @@ -2076,11 +2075,6 @@ "node": ">=4" } }, - "node_modules/@tokenizer/token": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", - "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" - }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -3837,9 +3831,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", "engines": { "node": ">=8" } @@ -4562,22 +4556,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/file-type": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.0.0.tgz", - "integrity": "sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==", - "dependencies": { - "readable-web-to-node-stream": "^3.0.2", - "strtok3": "^7.0.0", - "token-types": "^5.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/file-type?sponsor=1" - } - }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -6407,15 +6385,16 @@ } }, "node_modules/openai": { - "version": "4.47.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.47.1.tgz", - "integrity": "sha512-WWSxhC/69ZhYWxH/OBsLEirIjUcfpQ5+ihkXKp06hmeYXgBBIUCa9IptMzYx6NdkiOCsSGYCnTIsxaic3AjRCQ==", + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.14.2.tgz", + "integrity": "sha512-JGlm7mMC7J+cyQZnQMOH7daD9cBqqWqLtlBsejElEkgoehPrYfdyxSxIGICz5xk4YimbwI5FlLATSVojLtCKXQ==", "optional": true, "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", + "digest-fetch": "^1.3.0", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7", @@ -6641,18 +6620,6 @@ "node": "*" } }, - "node_modules/peek-readable": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", - "integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==", - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "node_modules/periscopic": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz", @@ -7508,21 +7475,6 @@ "node": ">= 6" } }, - "node_modules/readable-web-to-node-stream": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", - "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", - "dependencies": { - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -7784,9 +7736,12 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dependencies": { + "lru-cache": "^6.0.0" + }, "bin": { "semver": "bin/semver.js" }, @@ -7885,42 +7840,42 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/sharp": { - "version": "0.33.3", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz", - "integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==", + "version": "0.33.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz", + "integrity": "sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==", "hasInstallScript": true, "dependencies": { "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.0" + "detect-libc": "^2.0.2", + "semver": "^7.5.4" }, "engines": { - "libvips": ">=8.15.2", + "libvips": ">=8.15.1", "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.3", - "@img/sharp-darwin-x64": "0.33.3", - "@img/sharp-libvips-darwin-arm64": "1.0.2", - "@img/sharp-libvips-darwin-x64": "1.0.2", - "@img/sharp-libvips-linux-arm": "1.0.2", - "@img/sharp-libvips-linux-arm64": "1.0.2", - "@img/sharp-libvips-linux-s390x": "1.0.2", - "@img/sharp-libvips-linux-x64": "1.0.2", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.2", - "@img/sharp-libvips-linuxmusl-x64": "1.0.2", - "@img/sharp-linux-arm": "0.33.3", - "@img/sharp-linux-arm64": "0.33.3", - "@img/sharp-linux-s390x": "0.33.3", - "@img/sharp-linux-x64": "0.33.3", - "@img/sharp-linuxmusl-arm64": "0.33.3", - "@img/sharp-linuxmusl-x64": "0.33.3", - "@img/sharp-wasm32": "0.33.3", - "@img/sharp-win32-ia32": "0.33.3", - "@img/sharp-win32-x64": "0.33.3" + "@img/sharp-darwin-arm64": "0.33.2", + "@img/sharp-darwin-x64": "0.33.2", + "@img/sharp-libvips-darwin-arm64": "1.0.1", + "@img/sharp-libvips-darwin-x64": "1.0.1", + "@img/sharp-libvips-linux-arm": "1.0.1", + "@img/sharp-libvips-linux-arm64": "1.0.1", + "@img/sharp-libvips-linux-s390x": "1.0.1", + "@img/sharp-libvips-linux-x64": "1.0.1", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.1", + "@img/sharp-libvips-linuxmusl-x64": "1.0.1", + "@img/sharp-linux-arm": "0.33.2", + "@img/sharp-linux-arm64": "0.33.2", + "@img/sharp-linux-s390x": "0.33.2", + "@img/sharp-linux-x64": "0.33.2", + "@img/sharp-linuxmusl-arm64": "0.33.2", + "@img/sharp-linuxmusl-x64": "0.33.2", + "@img/sharp-wasm32": "0.33.2", + "@img/sharp-win32-ia32": "0.33.2", + "@img/sharp-win32-x64": "0.33.2" } }, "node_modules/shebang-command": { @@ -8232,22 +8187,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "node_modules/strtok3": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz", - "integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==", - "dependencies": { - "@tokenizer/token": "^0.3.0", - "peek-readable": "^5.0.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "node_modules/sucrase": { "version": "3.32.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz", @@ -8770,22 +8709,6 @@ "node": ">=0.6" } }, - "node_modules/token-types": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", - "integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==", - "dependencies": { - "@tokenizer/token": "^0.3.0", - "ieee754": "^1.2.1" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "node_modules/totalist": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz", diff --git a/package.json b/package.json index 5952e57c9b8..c150794d57b 100644 --- a/package.json +++ b/package.json @@ -57,15 +57,14 @@ "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", - "@resvg/resvg-js": "^2.6.2", "@playwright/browser-chromium": "^1.43.1", + "@resvg/resvg-js": "^2.6.0", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.19.2", - "file-type": "^19.0.0", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", @@ -87,7 +86,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.3", + "sharp": "^0.33.2", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -99,6 +98,6 @@ "@google-cloud/vertexai": "^1.1.0", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", - "openai": "^4.44.0" + "openai": "^4.14.2" } } diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index f76cd12a358..c1ddb5a23b6 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -308,17 +308,17 @@ {#if message.files && message.files.length > 0}
{#each message.files as file} - - {#if file.type === "hash"} + + {#if file.length === 64} input from user {:else} input from user diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index a00133d94e1..5805c98aeb3 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -92,9 +92,7 @@ (lastMessage.from === "user" || lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1); - $: sources = files?.map((file) => - file2base64(file).then((value) => ({ type: "base64", value, mime: file.type })) - ); + $: sources = files.map((file) => file2base64(file)); function onShare() { dispatch("share"); @@ -231,13 +229,13 @@
- {#if sources?.length} + {#if sources.length}
{#each sources as source, index} {#await source then src}
input content diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 889bceeff0c..4353c6b11a5 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -1,9 +1,7 @@ import { z } from "zod"; -import type { Endpoint } from "../endpoints"; import { env } from "$env/dynamic/private"; +import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; -import { createImageProcessorOptionsValidator } from "../images"; -import { endpointMessagesToAnthropicMessages } from "./utils"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -13,24 +11,12 @@ export const endpointAnthropicParametersSchema = z.object({ apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"), defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), - multimodal: z - .object({ - image: createImageProcessorOptionsValidator({ - supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"], - preferredMimeType: "image/webp", - // The 4 / 3 compensates for the 33% increase in size when converting to base64 - maxSizeInMB: (5 / 4) * 3, - maxWidth: 4096, - maxHeight: 4096, - }), - }) - .default({}), }); export async function endpointAnthropic( input: z.input ): Promise { - const { baseURL, apiKey, model, defaultHeaders, defaultQuery, multimodal } = + const { baseURL, apiKey, model, defaultHeaders, defaultQuery } = endpointAnthropicParametersSchema.parse(input); let Anthropic; try { @@ -52,6 +38,16 @@ export async function endpointAnthropic( system = messages[0].content; } + const messagesFormatted = messages + .filter((message) => message.from !== "system") + .map((message) => ({ + role: message.from, + content: message.content, + })) as unknown as { + role: "user" | "assistant"; + content: string; + }[]; + let tokenId = 0; const parameters = { ...model.parameters, ...generateSettings }; @@ -59,7 +55,7 @@ export async function endpointAnthropic( return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: await endpointMessagesToAnthropicMessages(messages, multimodal), + messages: messagesFormatted, max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, top_p: parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts index a90dd627b21..620b6382128 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts @@ -1,8 +1,6 @@ import { z } from "zod"; import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; -import { createImageProcessorOptionsValidator } from "../images"; -import { endpointMessagesToAnthropicMessages } from "./utils"; export const endpointAnthropicVertexParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -12,24 +10,12 @@ export const endpointAnthropicVertexParametersSchema = z.object({ projectId: z.string(), defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), - multimodal: z - .object({ - image: createImageProcessorOptionsValidator({ - supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"], - preferredMimeType: "image/webp", - // The 4 / 3 compensates for the 33% increase in size when converting to base64 - maxSizeInMB: (5 / 4) * 3, - maxWidth: 4096, - maxHeight: 4096, - }), - }) - .default({}), }); export async function endpointAnthropicVertex( input: z.input ): Promise { - const { region, projectId, model, defaultHeaders, defaultQuery, multimodal } = + const { region, projectId, model, defaultHeaders, defaultQuery } = endpointAnthropicVertexParametersSchema.parse(input); let AnthropicVertex; try { @@ -52,11 +38,21 @@ export async function endpointAnthropicVertex( system = messages[0].content; } + const messagesFormatted = messages + .filter((message) => message.from !== "system") + .map((message) => ({ + role: message.from, + content: message.content, + })) as unknown as { + role: "user" | "assistant"; + content: string; + }[]; + let tokenId = 0; return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: await endpointMessagesToAnthropicMessages(messages, multimodal), + messages: messagesFormatted, max_tokens: model.parameters?.max_new_tokens, temperature: model.parameters?.temperature, top_p: model.parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts deleted file mode 100644 index 93e15d15bf9..00000000000 --- a/src/lib/server/endpoints/anthropic/utils.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources"; -import { makeImageProcessor, type ImageProcessorOptions } from "../images"; -import type { EndpointMessage } from "../endpoints"; -import type { MessageFile } from "$lib/types/Message"; - -export async function fileToImageBlock( - file: MessageFile, - opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> -): Promise { - const processor = makeImageProcessor(opts); - const { image, mime } = await processor(file); - - return { - type: "image", - source: { - type: "base64", - media_type: mime, - data: image.toString("base64"), - }, - }; -} - -type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" }; - -export async function endpointMessagesToAnthropicMessages( - messages: EndpointMessage[], - multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> } -): Promise { - return await Promise.all( - messages - .filter((message): message is NonSystemMessage => message.from !== "system") - .map>(async (message) => { - return { - role: message.from, - content: [ - ...(await Promise.all( - (message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image)) - )), - { type: "text", text: message.content }, - ], - }; - }) - ); -} diff --git a/src/lib/server/endpoints/endpoints.ts b/src/lib/server/endpoints/endpoints.ts index 09a78c345fb..dddc1325bc5 100644 --- a/src/lib/server/endpoints/endpoints.ts +++ b/src/lib/server/endpoints/endpoints.ts @@ -1,5 +1,4 @@ import type { Conversation } from "$lib/types/Conversation"; -import type { Message } from "$lib/types/Message"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi"; import { z } from "zod"; @@ -26,10 +25,9 @@ import endpointLangserve, { endpointLangserveParametersSchema, } from "./langserve/endpointLangserve"; -export type EndpointMessage = Omit; // parameters passed when generating text export interface EndpointParameters { - messages: EndpointMessage[]; + messages: Omit[]; preprompt?: Conversation["preprompt"]; continueMessage?: boolean; // used to signal that the last message will be extended generateSettings?: Partial; diff --git a/src/lib/server/endpoints/images.ts b/src/lib/server/endpoints/images.ts deleted file mode 100644 index 7d408814cf2..00000000000 --- a/src/lib/server/endpoints/images.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { Sharp } from "sharp"; -import sharp from "sharp"; -import type { MessageFile } from "$lib/types/Message"; -import { z, type util } from "zod"; - -export interface ImageProcessorOptions { - supportedMimeTypes: TMimeType[]; - preferredMimeType: TMimeType; - maxSizeInMB: number; - maxWidth: number; - maxHeight: number; -} -export type ImageProcessor = (file: MessageFile) => Promise<{ - image: Buffer; - mime: TMimeType; -}>; - -export function createImageProcessorOptionsValidator( - defaults: ImageProcessorOptions -) { - return z - .object({ - supportedMimeTypes: z - .array( - z.enum([ - defaults.supportedMimeTypes[0], - ...defaults.supportedMimeTypes.slice(1), - ]) - ) - .default(defaults.supportedMimeTypes), - preferredMimeType: z - .enum([defaults.supportedMimeTypes[0], ...defaults.supportedMimeTypes.slice(1)]) - .default(defaults.preferredMimeType as util.noUndefined), - maxSizeInMB: z.number().positive().default(defaults.maxSizeInMB), - maxWidth: z.number().int().positive().default(defaults.maxWidth), - maxHeight: z.number().int().positive().default(defaults.maxHeight), - }) - .default(defaults); -} - -export function makeImageProcessor( - options: ImageProcessorOptions -): ImageProcessor { - return async (file) => { - const { supportedMimeTypes, preferredMimeType, maxSizeInMB, maxWidth, maxHeight } = options; - const { mime, value } = file; - - const buffer = Buffer.from(value, "base64"); - let sharpInst = sharp(buffer); - - const metadata = await sharpInst.metadata(); - if (!metadata) throw Error("Failed to read image metadata"); - const { width, height } = metadata; - if (width === undefined || height === undefined) throw Error("Failed to read image size"); - - const tooLargeInSize = width > maxWidth || height > maxHeight; - const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000; - - const outputMime = chooseMimeType(supportedMimeTypes, preferredMimeType, mime, { - preferSizeReduction: tooLargeInBytes, - }); - - // Resize if necessary - if (tooLargeInSize || tooLargeInBytes) { - const size = chooseImageSize({ - mime: outputMime, - width, - height, - maxWidth, - maxHeight, - maxSizeInMB, - }); - if (size.width !== width || size.height !== height) { - sharpInst = resizeImage(sharpInst, size.width, size.height); - } - } - - // Convert format if necessary - // We always want to convert the image when the file was too large in bytes - // so we can guarantee that ideal options are used, which are expected when - // choosing the image size - if (outputMime !== mime || tooLargeInBytes) { - sharpInst = convertImage(sharpInst, outputMime); - } - - const processedImage = await sharpInst.toBuffer(); - return { image: processedImage, mime: outputMime }; - }; -} - -const outputFormats = ["png", "jpeg", "webp", "avif", "tiff", "gif"] as const; -type OutputImgFormat = (typeof outputFormats)[number]; -const isOutputFormat = (format: string): format is (typeof outputFormats)[number] => - outputFormats.includes(format as OutputImgFormat); - -export function convertImage(sharpInst: Sharp, outputMime: string): Sharp { - const [type, format] = outputMime.split("/"); - if (type !== "image") throw Error(`Requested non-image mime type: ${outputMime}`); - if (!isOutputFormat(format)) { - throw Error(`Requested to convert to an unsupported format: ${format}`); - } - - return sharpInst[format](); -} - -// heic/heif requires proprietary license -// TODO: blocking heif may be incorrect considering it also supports av1, so we should instead -// detect the compression method used via sharp().metadata().compression -// TODO: consider what to do about animated formats: apng, gif, animated webp, ... -const blocklistedMimes = ["image/heic", "image/heif"]; - -/** Sorted from largest to smallest */ -const mimesBySizeDesc = [ - "image/png", - "image/tiff", - "image/gif", - "image/jpeg", - "image/webp", - "image/avif", -]; - -/** - * Defaults to preferred format or uses existing mime if supported - * When preferSizeReduction is true, it will choose the smallest format that is supported - **/ -function chooseMimeType( - supportedMimes: T, - preferredMime: string, - mime: string, - { preferSizeReduction }: { preferSizeReduction: boolean } -): T[number] { - if (!supportedMimes.includes(preferredMime)) { - const supportedMimesStr = supportedMimes.join(", "); - throw Error( - `Preferred format "${preferredMime}" not found in supported mimes: ${supportedMimesStr}` - ); - } - - const [type] = mime.split("/"); - if (type !== "image") throw Error(`Received non-image mime type: ${mime}`); - - if (supportedMimes.includes(mime) && !preferSizeReduction) return mime; - - if (blocklistedMimes.includes(mime)) throw Error(`Received blocklisted mime type: ${mime}`); - - const smallestMime = mimesBySizeDesc.findLast((m) => supportedMimes.includes(m)); - return smallestMime ?? preferredMime; -} - -interface ImageSizeOptions { - mime: string; - width: number; - height: number; - maxWidth: number; - maxHeight: number; - maxSizeInMB: number; -} - -/** Resizes the image to fit within the specified size in MB by guessing the output size */ -export function chooseImageSize({ - mime, - width, - height, - maxWidth, - maxHeight, - maxSizeInMB, -}: ImageSizeOptions): { width: number; height: number } { - const biggestDiscrepency = Math.max(1, width / maxWidth, height / maxHeight); - - let selectedWidth = Math.ceil(width / biggestDiscrepency); - let selectedHeight = Math.ceil(height / biggestDiscrepency); - - do { - const estimatedSize = estimateImageSizeInBytes(mime, selectedWidth, selectedHeight); - if (estimatedSize < maxSizeInMB * 1024 * 1024) { - return { width: selectedWidth, height: selectedHeight }; - } - selectedWidth = Math.floor(selectedWidth / 1.1); - selectedHeight = Math.floor(selectedHeight / 1.1); - } while (selectedWidth > 1 && selectedHeight > 1); - - throw Error(`Failed to resize image to fit within ${maxSizeInMB}MB`); -} - -const mimeToCompressionRatio: Record = { - "image/png": 1 / 2, - "image/jpeg": 1 / 10, - "image/webp": 1 / 4, - "image/avif": 1 / 5, - "image/tiff": 1, - "image/gif": 1 / 5, -}; - -/** - * Guesses the side of an image in MB based on its format and dimensions - * Should guess the worst case - **/ -function estimateImageSizeInBytes(mime: string, width: number, height: number): number { - const compressionRatio = mimeToCompressionRatio[mime]; - if (!compressionRatio) throw Error(`Unsupported image format: ${mime}`); - - const bitsPerPixel = 32; // Assuming 32-bit color depth for 8-bit R G B A - const bytesPerPixel = bitsPerPixel / 8; - const uncompressedSize = width * height * bytesPerPixel; - - return uncompressedSize * compressionRatio; -} - -export function resizeImage(sharpInst: Sharp, maxWidth: number, maxHeight: number): Sharp { - return sharpInst.resize({ width: maxWidth, height: maxHeight, fit: "inside" }); -} diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 7611ed246f0..84a221bdfc0 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -6,10 +6,6 @@ import type { ChatCompletionCreateParamsStreaming } from "openai/resources/chat/ import { buildPrompt } from "$lib/buildPrompt"; import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; -import type OpenAI from "openai"; -import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images"; -import type { MessageFile } from "$lib/types/Message"; -import type { EndpointMessage } from "../endpoints"; export const endpointOAIParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -23,41 +19,13 @@ export const endpointOAIParametersSchema = z.object({ defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), extraBody: z.record(z.any()).optional(), - multimodal: z - .object({ - image: createImageProcessorOptionsValidator({ - supportedMimeTypes: [ - "image/png", - "image/jpeg", - "image/webp", - "image/avif", - "image/tiff", - "image/gif", - ], - preferredMimeType: "image/webp", - maxSizeInMB: Infinity, - maxWidth: 4096, - maxHeight: 4096, - }), - }) - .default({}), }); export async function endpointOai( input: z.input ): Promise { - const { - baseURL, - apiKey, - completion, - model, - defaultHeaders, - defaultQuery, - multimodal, - extraBody, - } = endpointOAIParametersSchema.parse(input); - - /* eslint-disable-next-line no-shadow */ + const { baseURL, apiKey, completion, model, defaultHeaders, defaultQuery, extraBody } = + endpointOAIParametersSchema.parse(input); let OpenAI; try { OpenAI = (await import("openai")).OpenAI; @@ -72,8 +40,6 @@ export async function endpointOai( defaultQuery, }); - const imageProcessor = makeImageProcessor(multimodal.image); - if (completion === "completions") { return async ({ messages, preprompt, continueMessage, generateSettings }) => { const prompt = await buildPrompt({ @@ -103,8 +69,10 @@ export async function endpointOai( }; } else if (completion === "chat_completions") { return async ({ messages, preprompt, generateSettings }) => { - let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = - await prepareMessages(messages, imageProcessor); + let messagesOpenAI = messages.map((message) => ({ + role: message.from, + content: message.content, + })); if (messagesOpenAI?.[0]?.role !== "system") { messagesOpenAI = [{ role: "system", content: "" }, ...messagesOpenAI]; @@ -136,39 +104,3 @@ export async function endpointOai( throw new Error("Invalid completion type"); } } - -async function prepareMessages( - messages: EndpointMessage[], - imageProcessor: ReturnType -): Promise { - return Promise.all( - messages.map(async (message) => { - if (message.from === "user") { - return { - role: message.from, - content: [ - ...(await prepareFiles(imageProcessor, message.files ?? [])), - { type: "text", text: message.content }, - ], - }; - } - return { - role: message.from, - content: message.content, - }; - }) - ); -} - -async function prepareFiles( - imageProcessor: ReturnType, - files: MessageFile[] -): Promise { - const processedFiles = await Promise.all(files.map(imageProcessor)); - return processedFiles.map((file) => ({ - type: "image_url" as const, - image_url: { - url: `data:${file.mime};base64,${file.image.toString("base64")}`, - }, - })); -} diff --git a/src/lib/server/endpoints/preprocessMessages.ts b/src/lib/server/endpoints/preprocessMessages.ts deleted file mode 100644 index e38dd9abc26..00000000000 --- a/src/lib/server/endpoints/preprocessMessages.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { Message } from "$lib/types/Message"; -import { format } from "date-fns"; -import type { EndpointMessage } from "./endpoints"; -import { downloadFile } from "../files/downloadFile"; -import type { ObjectId } from "mongodb"; - -export async function preprocessMessages( - messages: Message[], - webSearch: Message["webSearch"], - convId: ObjectId -): Promise { - return Promise.resolve(messages) - .then((msgs) => addWebSearchContext(msgs, webSearch)) - .then((msgs) => downloadFiles(msgs, convId)); -} - -function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) { - const webSearchContext = webSearch?.contextSources - .map(({ context }) => context.trim()) - .join("\n\n----------\n\n"); - - // No web search context available, skip - if (!webSearch || !webSearchContext?.trim()) return messages; - // No messages available, skip - if (messages.length === 0) return messages; - - const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? ""; - const previousQuestions = messages - .filter((el) => el.from === "user") - .slice(0, -1) - .map((el) => el.content); - const currentDate = format(new Date(), "MMMM d, yyyy"); - - const finalMessage = { - ...messages[messages.length - 1], - content: `I searched the web using the query: ${webSearch.searchQuery}. -Today is ${currentDate} and here are the results: -===================== -${webSearchContext} -===================== -${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""} -Answer the question: ${lastQuestion}`, - }; - - return [...messages.slice(0, -1), finalMessage]; -} - -async function downloadFiles(messages: Message[], convId: ObjectId): Promise { - return Promise.all( - messages.map>((message) => - Promise.all((message.files ?? []).map((file) => downloadFile(file.value, convId))).then( - (files) => ({ ...message, files }) - ) - ) - ); -} diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts index 53f69ca1e4b..aed06739722 100644 --- a/src/lib/server/endpoints/tgi/endpointTgi.ts +++ b/src/lib/server/endpoints/tgi/endpointTgi.ts @@ -1,13 +1,8 @@ import { env } from "$env/dynamic/private"; import { buildPrompt } from "$lib/buildPrompt"; import { textGenerationStream } from "@huggingface/inference"; -import type { Endpoint, EndpointMessage } from "../endpoints"; +import type { Endpoint } from "../endpoints"; import { z } from "zod"; -import { - createImageProcessorOptionsValidator, - makeImageProcessor, - type ImageProcessor, -} from "../images"; export const endpointTgiParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -16,32 +11,14 @@ export const endpointTgiParametersSchema = z.object({ url: z.string().url(), accessToken: z.string().default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN), authorization: z.string().optional(), - multimodal: z - .object({ - // Assumes IDEFICS - image: createImageProcessorOptionsValidator({ - supportedMimeTypes: ["image/jpeg", "image/webp"], - preferredMimeType: "image/webp", - maxSizeInMB: 5, - maxWidth: 224, - maxHeight: 224, - }), - }) - .default({}), }); export function endpointTgi(input: z.input): Endpoint { - const { url, accessToken, model, authorization, multimodal } = - endpointTgiParametersSchema.parse(input); - const imageProcessor = makeImageProcessor(multimodal.image); + const { url, accessToken, model, authorization } = endpointTgiParametersSchema.parse(input); return async ({ messages, preprompt, continueMessage, generateSettings }) => { - const messagesWithResizedFiles = await Promise.all( - messages.map((message) => prepareMessage(message, imageProcessor)) - ); - const prompt = await buildPrompt({ - messages: messagesWithResizedFiles, + messages, preprompt, model, continueMessage, @@ -71,23 +48,4 @@ export function endpointTgi(input: z.input): }; } -const whiteImage = { - mime: "image/png", - image: Buffer.from( - "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==", - "base64" - ), -}; - -async function prepareMessage( - message: EndpointMessage, - imageProcessor: ImageProcessor -): Promise { - const files = await Promise.all(message.files?.map(imageProcessor) ?? [whiteImage]); - const markdowns = files.map( - (file) => `![](data:${file.mime};base64,${file.image.toString("base64")})` - ); - const content = message.content + "\n" + markdowns.join("\n "); - - return { ...message, content }; -} +export default endpointTgi; diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts index dac5bbda848..91b430fc5d8 100644 --- a/src/lib/server/files/downloadFile.ts +++ b/src/lib/server/files/downloadFile.ts @@ -2,16 +2,15 @@ import { error } from "@sveltejs/kit"; import { collections } from "$lib/server/database"; import type { Conversation } from "$lib/types/Conversation"; import type { SharedConversation } from "$lib/types/SharedConversation"; -import type { MessageFile } from "$lib/types/Message"; export async function downloadFile( sha256: string, convId: Conversation["_id"] | SharedConversation["_id"] -): Promise { +) { const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` }); let mime = ""; - const buffer = await fileId.next().then(async (file) => { + const content = await fileId.next().then(async (file) => { if (!file) { throw error(404, "File not found"); } @@ -33,5 +32,5 @@ export async function downloadFile( return fileBuffer; }); - return { type: "base64", value: buffer.toString("base64"), mime }; + return { content, mime }; } diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts index 339a4b4ea85..34452245741 100644 --- a/src/lib/server/files/uploadFile.ts +++ b/src/lib/server/files/uploadFile.ts @@ -1,27 +1,21 @@ import type { Conversation } from "$lib/types/Conversation"; -import type { MessageFile } from "$lib/types/Message"; import { sha256 } from "$lib/utils/sha256"; -import { fileTypeFromBuffer } from "file-type"; import { collections } from "$lib/server/database"; -export async function uploadFile(file: File, conv: Conversation): Promise { +export async function uploadFile(file: Blob, conv: Conversation): Promise { const sha = await sha256(await file.text()); - const buffer = await file.arrayBuffer(); - - // Attempt to detect the mime type of the file, fallback to the uploaded mime - const mime = await fileTypeFromBuffer(buffer).then((fileType) => fileType?.mime ?? file.type); const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, { - metadata: { conversation: conv._id.toString(), mime }, + metadata: { conversation: conv._id.toString(), mime: "image/jpeg" }, }); upload.write((await file.arrayBuffer()) as unknown as Buffer); upload.end(); - // only return the filename when upload throws a finish event or a 20s time out occurs + // only return the filename when upload throws a finish event or a 10s time out occurs return new Promise((resolve, reject) => { - upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type })); + upload.once("finish", () => resolve(sha)); upload.once("error", reject); - setTimeout(() => reject(new Error("Upload timed out")), 20_000); + setTimeout(() => reject(new Error("Upload timed out")), 10000); }); } diff --git a/src/lib/server/generateFromDefaultEndpoint.ts b/src/lib/server/generateFromDefaultEndpoint.ts index 4f798f90f51..428e94a06f9 100644 --- a/src/lib/server/generateFromDefaultEndpoint.ts +++ b/src/lib/server/generateFromDefaultEndpoint.ts @@ -1,12 +1,12 @@ import { smallModel } from "$lib/server/models"; -import type { EndpointMessage } from "./endpoints/endpoints"; +import type { Conversation } from "$lib/types/Conversation"; export async function generateFromDefaultEndpoint({ messages, preprompt, generateSettings, }: { - messages: EndpointMessage[]; + messages: Omit[]; preprompt?: string; generateSettings?: Record; }): Promise { diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 78f0383331a..fb9604b625a 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -3,7 +3,7 @@ import type { ChatTemplateInput } from "$lib/types/Template"; import { compileTemplate } from "$lib/utils/template"; import { z } from "zod"; import endpoints, { endpointSchema, type Endpoint } from "./endpoints/endpoints"; -import { endpointTgi } from "./endpoints/tgi/endpointTgi"; +import endpointTgi from "./endpoints/tgi/endpointTgi"; import { sum } from "$lib/utils/sum"; import { embeddingModels, validateEmbeddingModelByName } from "./embeddingModels"; diff --git a/src/lib/server/preprocessMessages.ts b/src/lib/server/preprocessMessages.ts new file mode 100644 index 00000000000..c5ff2c585c0 --- /dev/null +++ b/src/lib/server/preprocessMessages.ts @@ -0,0 +1,61 @@ +import type { Conversation } from "$lib/types/Conversation"; +import type { Message } from "$lib/types/Message"; +import { format } from "date-fns"; +import { downloadFile } from "./files/downloadFile"; +import { logger } from "$lib/server/logger"; + +export async function preprocessMessages( + messages: Message[], + webSearch: Message["webSearch"], + multimodal: boolean, + id: Conversation["_id"] +): Promise { + return await Promise.all( + structuredClone(messages).map(async (message, idx) => { + const webSearchContext = webSearch?.contextSources + .map(({ context }) => context.trim()) + .join("\n\n----------\n\n"); + + // start by adding websearch to the last message + if (idx === messages.length - 1 && webSearch && webSearchContext?.trim()) { + const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? ""; + const previousQuestions = messages + .filter((el) => el.from === "user") + .slice(0, -1) + .map((el) => el.content); + const currentDate = format(new Date(), "MMMM d, yyyy"); + + message.content = `I searched the web using the query: ${webSearch.searchQuery}. +Today is ${currentDate} and here are the results: +===================== +${webSearchContext} +===================== +${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""} +Answer the question: ${lastQuestion}`; + } + // handle files if model is multimodal + if (multimodal) { + if (message.files && message.files.length > 0) { + const markdowns = await Promise.all( + message.files.map(async (hash) => { + try { + const { content: image, mime } = await downloadFile(hash, id); + const b64 = image.toString("base64"); + return `![](data:${mime};base64,${b64})})`; + } catch (e) { + logger.error(e); + } + }) + ); + message.content += markdowns.join("\n "); + } else { + // if no image, append an empty white image + message.content += + "\n![](data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==)"; + } + } + + return message; + }) + ); +} diff --git a/src/lib/server/summarize.ts b/src/lib/server/summarize.ts index 033e1ae7033..4cef6174dc9 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/summarize.ts @@ -1,6 +1,6 @@ import { env } from "$env/dynamic/private"; import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint"; -import type { EndpointMessage } from "./endpoints/endpoints"; +import type { Message } from "$lib/types/Message"; import { logger } from "$lib/server/logger"; export async function summarize(prompt: string) { @@ -8,7 +8,7 @@ export async function summarize(prompt: string) { return prompt.split(/\s+/g).slice(0, 5).join(" "); } - const messages: Array = [ + const messages: Array> = [ { from: "user", content: "Who is the president of Gabon?" }, { from: "assistant", content: "🇬🇦 President of Gabon" }, { from: "user", content: "Who is Julien Chaumond?" }, diff --git a/src/lib/server/websearch/search/generateQuery.ts b/src/lib/server/websearch/search/generateQuery.ts index c71841a8c17..b08a3df6717 100644 --- a/src/lib/server/websearch/search/generateQuery.ts +++ b/src/lib/server/websearch/search/generateQuery.ts @@ -1,6 +1,5 @@ import type { Message } from "$lib/types/Message"; import { format } from "date-fns"; -import type { EndpointMessage } from "../../endpoints/endpoints"; import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint"; export async function generateQuery(messages: Message[]) { @@ -10,7 +9,7 @@ export async function generateQuery(messages: Message[]) { const lastMessage = userMessages.slice(-1)[0]; - const convQuery: Array = [ + const convQuery: Array> = [ { from: "user", content: `Previous Questions: diff --git a/src/lib/types/Message.ts b/src/lib/types/Message.ts index 6791164febf..68f9d6b271c 100644 --- a/src/lib/types/Message.ts +++ b/src/lib/types/Message.ts @@ -11,11 +11,7 @@ export type Message = Partial & { webSearchId?: WebSearch["_id"]; // legacy version webSearch?: WebSearch; score?: -1 | 0 | 1; - /** - * Either contains the base64 encoded image data - * or the hash of the file stored on the server - **/ - files?: MessageFile[]; + files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading interrupted?: boolean; // needed for conversation trees @@ -24,9 +20,3 @@ export type Message = Partial & { // goes one level deep children?: Message["id"][]; }; - -export type MessageFile = { - type: "hash" | "base64"; - value: string; - mime: string; -}; diff --git a/src/lib/utils/messageUpdates.ts b/src/lib/utils/messageUpdates.ts index 83929255e01..82b8bb9a2ca 100644 --- a/src/lib/utils/messageUpdates.ts +++ b/src/lib/utils/messageUpdates.ts @@ -1,4 +1,3 @@ -import type { MessageFile } from "$lib/types/Message"; import type { MessageUpdate, TextStreamUpdate } from "$lib/types/MessageUpdate"; type MessageUpdateRequestOptions = { @@ -8,7 +7,7 @@ type MessageUpdateRequestOptions = { isRetry: boolean; isContinue: boolean; webSearch: boolean; - files?: MessageFile[]; + files?: string[]; }; export async function fetchMessageUpdates( conversationId: string, diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 806b30bd772..04888829a06 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -75,10 +75,20 @@ loading = true; pending = true; - const base64Files = await Promise.all( - (files ?? []).map((file) => - file2base64(file).then((value) => ({ type: "base64" as const, value, mime: file.type })) - ) + const module = await import("browser-image-resizer"); + // currently, only IDEFICS is supported by TGI + // the size of images is hardcoded to 224x224 in TGI + // this will need to be configurable when support for more models is added + const resizedImages = await Promise.all( + files.map(async (file) => { + return await module + .readAndCompressImage(file, { + maxHeight: 224, + maxWidth: 224, + quality: 1, + }) + .then(async (el) => await file2base64(el as File)); + }) ); let messageToWriteToId: Message["id"] | undefined = undefined; @@ -110,11 +120,7 @@ messages, rootMessageId: data.rootMessageId, }, - { - from: "user", - content: prompt, - files: messageToRetry.files, - }, + { from: "user", content: prompt }, messageId ); messageToWriteToId = addChildren( @@ -122,7 +128,7 @@ messages, rootMessageId: data.rootMessageId, }, - { from: "assistant", content: "" }, + { from: "assistant", content: "", files: resizedImages }, newUserMessageId ); } else if (messageToRetry?.from === "assistant") { @@ -148,7 +154,7 @@ { from: "user", content: prompt ?? "", - files: base64Files, + files: resizedImages, createdAt: new Date(), updatedAt: new Date(), }, @@ -175,7 +181,6 @@ } messages = [...messages]; - const userMessage = messages.find((message) => message.id === messageId); const messageToWriteTo = messages.find((message) => message.id === messageToWriteToId); if (!messageToWriteTo) { throw new Error("Message to write to not found"); @@ -193,7 +198,7 @@ isRetry, isContinue, webSearch: !hasAssistant && $webSearchParameters.useSearch, - files: isRetry ? userMessage?.files : base64Files, + files: isRetry ? undefined : resizedImages, }, messageUpdatesAbortController.signal ).catch((err) => { diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 0dc4913fe57..ebbedb945f9 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -13,13 +13,14 @@ import { runWebSearch } from "$lib/server/websearch/runWebSearch"; import { AbortedGenerations } from "$lib/server/abortedGenerations"; import { summarize } from "$lib/server/summarize"; import { uploadFile } from "$lib/server/files/uploadFile"; +import sizeof from "image-size"; import type { Assistant } from "$lib/types/Assistant"; import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation"; import { isMessageId } from "$lib/utils/tree/isMessageId"; import { buildSubtree } from "$lib/utils/tree/buildSubtree.js"; import { addChildren } from "$lib/utils/tree/addChildren.js"; import { addSibling } from "$lib/utils/tree/addSibling.js"; -import { preprocessMessages } from "$lib/server/endpoints/preprocessMessages.js"; +import { preprocessMessages } from "$lib/server/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; import { logger } from "$lib/server/logger.js"; @@ -133,7 +134,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: isRetry, is_continue: isContinue, web_search: webSearch, - files: inputFiles, + files: b64files, } = z .object({ id: z.string().uuid().refine(isMessageId).optional(), // parent message id to append to for a normal message, or the message id for a retry/continue @@ -146,43 +147,44 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: z.optional(z.boolean()), is_continue: z.optional(z.boolean()), web_search: z.optional(z.boolean()), - files: z.optional( - z.array( - z.object({ - type: z.literal("base64").or(z.literal("hash")), - value: z.string(), - mime: z.string(), - }) - ) - ), + files: z.optional(z.array(z.string())), }) .parse(json); if (usageLimits?.messageLength && (newPrompt?.length ?? 0) > usageLimits.messageLength) { throw error(400, "Message too long."); } + // files is an array of base64 strings encoding Blob objects + // we need to convert this array to an array of File objects - // each file is either: - // base64 string requiring upload to the server - // hash pointing to an existing file - const hashFiles = inputFiles?.filter((file) => file.type === "hash") ?? []; - const b64Files = - inputFiles - ?.filter((file) => file.type !== "hash") - .map((file) => { - const blob = Buffer.from(file.value, "base64"); - return new File([blob], "file", { type: file.mime }); - }) ?? []; + const files = b64files?.map((file) => { + const blob = Buffer.from(file, "base64"); + return new File([blob], "image.png"); + }); // check sizes - // todo: make configurable - if (b64Files.some((file) => file.size > 10 * 1024 * 1024)) { - throw error(413, "File too large, should be <10MB"); + if (files) { + const filechecks = await Promise.all( + files.map(async (file) => { + const dimensions = sizeof(Buffer.from(await file.arrayBuffer())); + return ( + file.size > 2 * 1024 * 1024 || + (dimensions.width ?? 0) > 224 || + (dimensions.height ?? 0) > 224 + ); + }) + ); + + if (filechecks.some((check) => check)) { + throw error(413, "File too large, should be <2MB and 224x224 max."); + } } - const uploadedFiles = await Promise.all(b64Files.map((file) => uploadFile(file, conv))).then( - (files) => [...files, ...hashFiles] - ); + let hashes: undefined | string[]; + + if (files) { + hashes = await Promise.all(files.map(async (file) => await uploadFile(file, conv))); + } // we will append tokens to the content of this message let messageToWriteToId: Message["id"] | undefined = undefined; @@ -214,13 +216,7 @@ export async function POST({ request, locals, params, getClientAddress }) { // add a children to that sibling, where we can write to const newUserMessageId = addSibling( conv, - { - from: "user", - content: newPrompt, - files: uploadedFiles, - createdAt: new Date(), - updatedAt: new Date(), - }, + { from: "user", content: newPrompt, createdAt: new Date(), updatedAt: new Date() }, messageId ); messageToWriteToId = addChildren( @@ -228,6 +224,7 @@ export async function POST({ request, locals, params, getClientAddress }) { { from: "assistant", content: "", + files: hashes, createdAt: new Date(), updatedAt: new Date(), }, @@ -253,7 +250,7 @@ export async function POST({ request, locals, params, getClientAddress }) { { from: "user", content: newPrompt ?? "", - files: uploadedFiles, + files: hashes, createdAt: new Date(), updatedAt: new Date(), }, @@ -414,9 +411,10 @@ export async function POST({ request, locals, params, getClientAddress }) { } // inject websearch result & optionally images into the messages - const processedMessages = preprocessMessages( + const processedMessages = await preprocessMessages( messagesForPrompt, messageToWriteTo.webSearch, + model.multimodal, convId ); @@ -431,7 +429,7 @@ export async function POST({ request, locals, params, getClientAddress }) { try { const endpoint = await model.getEndpoint(); for await (const output of await endpoint({ - messages: await processedMessages, + messages: processedMessages, preprompt, continueMessage: isContinue, generateSettings: assistant?.generateSettings, diff --git a/src/routes/conversation/[id]/output/[sha256]/+server.ts b/src/routes/conversation/[id]/output/[sha256]/+server.ts index 5b3ae84dcf8..79ae37b7585 100644 --- a/src/routes/conversation/[id]/output/[sha256]/+server.ts +++ b/src/routes/conversation/[id]/output/[sha256]/+server.ts @@ -39,9 +39,9 @@ export const GET: RequestHandler = async ({ locals, params }) => { } } - const { value, mime } = await downloadFile(sha256, params.id); + const { content, mime } = await downloadFile(sha256, params.id); - return new Response(Buffer.from(value, "base64"), { + return new Response(content, { headers: { "Content-Type": mime ?? "application/octet-stream", },