feat: vision modal

Jazee6 · May 30, 2024 · c4653f7 · c4653f7
1 parent 1543aad
commit c4653f7
Show file tree

Hide file tree

Showing 9 changed files with 204 additions and 85 deletions.
diff --git a/components/ChatInput.vue b/components/ChatInput.vue
@@ -1,6 +1,12 @@
 <script setup lang="ts">
+import {handleImgZoom} from "~/utils/tools";
+
 const input = ref('')
 const addHistory = ref(true)
+const fileList = ref<{
+  file: File
+  url: string
+}[]>([])
 const {openModelSelect} = useGlobalState()
 
 onMounted(() => {
@@ -14,7 +20,10 @@ const p = defineProps<{
   loading: boolean
   selectedModel: Model
 
-  handleSend: (input: string, addHistory: boolean) => void
+  handleSend: (input: string, addHistory: boolean, files: {
+    file: File
+    url: string
+  }[]) => void
 }>()
 
 function handleInput(e: KeyboardEvent) {
@@ -27,24 +36,69 @@ function handleInput(e: KeyboardEvent) {
 
   if (input.value.trim() === '') return
   if (p.loading) return
-  p.handleSend(input.value, addHistory.value)
+  p.handleSend(input.value, addHistory.value, fileList.value)
   input.value = ''
 }
+
+function handleAddFiles() {
+  const input = document.createElement('input')
+  input.type = 'file'
+  input.accept = 'image/*'
+  input.multiple = true
+  input.onchange = () => {
+    const files = Array.from(input.files || [])
+    files.forEach(file => {
+      if (file.type.indexOf('image') === -1) return
+      if (fileList.value.length >= 5) return
+      const url = URL.createObjectURL(file)
+      fileList.value.push({file, url})
+    })
+  }
+  input.click()
+}
+
+// TODO paste ?? size limit ?? tips
+
+onUnmounted(() => {
+  fileList.value.forEach(i => {
+    URL.revokeObjectURL(i.url)
+  })
+})
 </script>
 
 <template>
-  <div class="flex flex-col space-y-1">
-    <UButton class="self-center" color="white" @click="openModelSelect=!openModelSelect">
-      {{ selectedModel.name }}
-      <template #trailing>
-        <UIcon name="i-heroicons-chevron-down-solid"/>
-      </template>
-    </UButton>
+  <div class="relative">
+    <div class="absolute bottom-10 w-full flex flex-col">
+      <UButton class="self-center drop-shadow-xl mb-1" color="white" @click="openModelSelect=!openModelSelect">
+        {{ selectedModel.name }}
+        <template #trailing>
+          <UIcon name="i-heroicons-chevron-down-solid"/>
+        </template>
+      </UButton>
+      <ul v-if="selectedModel.type === 'vision'" style="margin: 0"
+          class="flex flex-wrap bg-white dark:bg-[#121212] rounded-t-md">
+        <li v-for="file in fileList" :key="file.url" class="relative group/img">
+          <button @click="fileList.splice(fileList.indexOf(file), 1)"
+                  class="absolute z-10 hidden group-hover/img:block rounded-full bg-neutral-100 right-0 hover:brightness-75 transition-all">
+            <svg xmlns="http://www.w3.org/2000/svg" class="w-5 h-5" viewBox="0 0 16 16">
+              <path fill="currentColor"
+                    d="M5.28 4.22a.75.75 0 0 0-1.06 1.06L6.94 8l-2.72 2.72a.75.75 0 1 0 1.06 1.06L8 9.06l2.72 2.72a.75.75 0 1 0 1.06-1.06L9.06 8l2.72-2.72a.75.75 0 0 0-1.06-1.06L8 6.94z"/>
+            </svg>
+          </button>
+          <img :src="file.url"
+               class="w-16 h-16 m-1 shadow-xl object-contain cursor-pointer group-hover/img:brightness-75 transition-all rounded-md"
+               alt="selected image" @click="handleImgZoom($event.target as HTMLImageElement)"/>
+        </li>
+      </ul>
+    </div>
     <div class="flex items-end">
       <UTooltip :text="addHistory?$t('with_history'):$t('without_history')">
         <UButton class="m-1" @click="addHistory = !addHistory" :color="addHistory?'primary':'gray'"
                  icon="i-heroicons-clock-solid"/>
       </UTooltip>
+      <UTooltip v-if="selectedModel.type === 'vision'" :text="$t('add_image')">
+        <UButton @click="handleAddFiles" color="white" class="m-1" icon="i-heroicons-paper-clip-16-solid"/>
+      </UTooltip>
       <UTextarea v-model="input" :placeholder="$t('please_input_text') + '...' "
                  @keydown.prevent.enter="handleInput($event)"
                  autofocus :rows="1" autoresize

diff --git a/components/ChatList.vue b/components/ChatList.vue
@@ -3,6 +3,7 @@ import MarkdownIt from "markdown-it"
 import markdownit from "markdown-it"
 import hljs from "highlight.js";
 import 'highlight.js/styles/github-dark-dimmed.min.css'
+import {handleImgZoom} from "~/utils/tools";
 
 defineProps<{
   history: HistoryItem[]
@@ -18,35 +19,6 @@ const md: MarkdownIt = markdownit({
     return `<pre class="hljs"><code>${hljs.highlightAuto(code).value}</code></pre>`;
   },
 })
-
-function handleZoom(e: MouseEvent) {
-  const img = e.target as HTMLImageElement
-  const container = document.createElement('div')
-  container.style.cssText = `
-    position: fixed;
-    inset: 0;
-    background-color: rgba(0, 0, 0, 0.8);
-    display: flex;
-    justify-content: center;
-    align-items: center;
-    transition: all 0.3s;
-    opacity: 0;
-    z-index: 9999;
-  `
-  const imgZoom = document.createElement('img')
-  imgZoom.src = img.src
-  container.appendChild(imgZoom)
-  document.body.appendChild(container)
-  container.addEventListener('click', () => {
-    container.style.opacity = '0'
-    setTimeout(() => {
-      document.body.removeChild(container)
-    }, 300)
-  })
-
-  imgZoom.height
-  container.style.opacity = '1'
-}
 </script>
 
 <template>
@@ -60,9 +32,12 @@ function handleZoom(e: MouseEvent) {
             :class="[i.role==='user'?'send':'reply-text', index+1===history.length && loading ?  'loading':'' ]"
             v-html="i.role === 'user'? i.content: md.render(i.content)"/>
         <li v-else-if="i.type === 'image'">
-          <img @click="handleZoom" class="chat-item slide-top cursor-pointer hover:brightness-75 transition-all"
-               :src="i.content"
-               :alt="history[index-1].content"/>
+          <template v-for="img_url in i.src_url" :key="img_url">
+            <img @click="handleImgZoom($event.target as HTMLImageElement)"
+                 class="img-item slide-top cursor-pointer hover:brightness-75 transition-all"
+                 :src="img_url"
+                 :alt="history[index-1].content"/>
+          </template>
         </li>
         <li v-else-if="i.type==='error'" class="chat-item slide-top reply-error">
           {{ i.content }}
@@ -82,6 +57,11 @@ function handleZoom(e: MouseEvent) {
   @apply break-words rounded-xl px-2 py-1.5
 }
 
+.img-item {
+  max-width: 80%;
+  @apply rounded-xl
+}
+
 .send {
   @apply self-end bg-green-500 text-white dark:bg-green-700 dark:text-gray-300
 }

diff --git a/components/ModelSelect.vue b/components/ModelSelect.vue
@@ -11,21 +11,29 @@ watch(selectedModel, v => {
   localStorage.setItem('selectedModel', v.id)
 })
 
-const groups = computed(() => [{
-  key: 'text generation',
-  label: t('text_generation'),
-  commands: textGenModels.map(i => ({
-    id: i.id,
-    label: i.name
-  }))
-}, {
-  key: 'image generation',
-  label: t('image_generation'),
-  commands: imageGenModels.map(i => ({
-    id: i.id,
-    label: i.name
-  }))
-}])
+const groups = computed(() => [
+  {
+    key: 'vision',
+    label: t('vision'),
+    commands: visionModals.map(i => ({
+      id: i.id,
+      label: i.name
+    }))
+  }, {
+    key: 'text generation',
+    label: t('text_generation'),
+    commands: textGenModels.map(i => ({
+      id: i.id,
+      label: i.name
+    }))
+  }, {
+    key: 'image generation',
+    label: t('image_generation'),
+    commands: imageGenModels.map(i => ({
+      id: i.id,
+      label: i.name
+    }))
+  }])
 
 function onSelect(option: { id: string }) {
   selectedModel.value = models.find(i => i.id === option.id) || textGenModels[0]

diff --git a/i18n.config.ts b/i18n.config.ts
@@ -13,13 +13,14 @@ export default defineI18nConfig(() => ({
             with_history: '发送时携带历史记录',
             without_history: '发送时不携带历史记录',
             please_input_text: '请输入文本',
-            // add_image: '添加图片',
+            add_image: '添加图片',
             // support_paste: '支持粘贴',
             send: '发送',
             img_gen_steps: '图片生成步数',
             text_generation: '文本生成',
             image_generation: '图像生成',
             system_prompt: '系统提示',
+            vision: '视觉',
         },
         en: {
             setting: 'Setting',
@@ -33,13 +34,14 @@ export default defineI18nConfig(() => ({
             with_history: 'Send with history',
             without_history: 'Send without history',
             please_input_text: 'Please input text',
-            // add_image: 'Add image',
+            add_image: 'Add image',
             // support_paste: 'Support paste',
             send: 'Send',
             img_gen_steps: 'Image generation steps',
             text_generation: 'Text generation',
             image_generation: 'Image generation',
             system_prompt: 'System prompt',
+            vision: 'Vision',
         }
     }
 }))
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cloudflare-ai-web",
-  "version": "2.2",
+  "version": "3.0.0beta",
   "private": true,
   "type": "module",
   "scripts": {
@@ -31,5 +31,6 @@
     "patchedDependencies": {
       "@google/generative-ai@0.3.1": "patches/@google__generative-ai@0.3.1.patch"
     }
-  }
+  },
+  "packageManager": "pnpm@9.1.2+sha512.127dc83b9ea10c32be65d22a8efb4a65fb952e8fefbdfded39bdc3c97efc32d31b48b00420df2c1187ace28c921c902f0cb5a134a4d032b8b5295cbfa2c681e2"
 }
diff --git a/pages/index.vue b/pages/index.vue
@@ -1,5 +1,6 @@
 <script setup lang="ts">
 import {useLocalStorage} from "@vueuse/core";
+import {deepClone} from "@vue/devtools-shared";
 
 const route = useRoute()
 const router = useRouter()
@@ -18,7 +19,8 @@ async function initDB() {
   session = await DB.addTab(t('new_chat')) as number
   tabs.value.unshift({
     id: session,
-    label: t('new_chat')
+    label: t('new_chat'),
+    created_at: Date.now()
   })
   selectedTab.value = session
   await router.push({query: {session}})
@@ -70,6 +72,7 @@ async function handleNewChat() {
 async function handleSwitchChat(e: MouseEvent) {
   if (loading.value) return
 
+  // TODO revokeObjectURL
   const target = e.target as HTMLElement
   const id = target.dataset.id
   if (!id) return
@@ -112,7 +115,10 @@ function basicDone() {
   DB.history.add(toRaw(history.value[history.value.length - 1]))
 }
 
-async function handleSend(input: string, addHistory: boolean) {
+async function handleSend(input: string, addHistory: boolean, files: {
+  file: File
+  url: string
+}[]) {
   loading.value = true
   const type = selectedModel.value.type
 
@@ -122,12 +128,18 @@ async function handleSend(input: string, addHistory: boolean) {
       tabs.value.find(i => i.id === session)!.label = label
     })
   }
-  const historyItem = {
+
+  if (files.length) {
+
+  }
+
+  const historyItem: HistoryItem = {
     session,
     role: 'user',
     content: input,
-    type: type === 'chat' ? 'text' : 'image-prompt'
-  } as HistoryItem
+    type: type === 'text-to-image' ? 'image-prompt' : 'text',
+    created_at: Date.now()
+  }
   const id = await DB.history.add(historyItem) as number
   history.value.push({
     id,
@@ -138,13 +150,14 @@ async function handleSend(input: string, addHistory: boolean) {
     session,
     role: 'assistant',
     content: '',
-    type: type === 'chat' ? 'text' : 'image'
+    type: type === 'chat' ? 'text' : 'image',
+    created_at: Date.now()
   })
 
   const chatList = document.getElementById('chatList') as HTMLElement
-  await nextTick(() => {
+  nextTick(() => {
     scrollToTop(chatList)
-  })
+  }).then(r => r)
 
   const req = {
     model: selectedModel.value.id,
@@ -172,14 +185,22 @@ async function handleSend(input: string, addHistory: boolean) {
         ...req,
         num_steps: settings.value.image_steps,
       }).then(res => {
-        history.value[history.value.length - 1].content = URL.createObjectURL(res as Blob)
-        history.value[history.value.length - 1].src = res as Blob
+        const blob = res as Blob
+        Object.assign(history.value[history.value.length - 1], {
+          content: input,
+          src: [blob],
+          src_url: [URL.createObjectURL(blob)]
+        })
 
         setTimeout(() => {
           scrollToTop(chatList)
+          basicFin()
         }, 100)
-        basicDone()
-      }).catch(basicCatch).finally(basicFin)
+      }).then(() => {
+        const store = {...toRaw(history.value[history.value.length - 1])}
+        delete store.src_url
+        DB.history.add(store)
+      }).catch(basicCatch)
       break
     case "google":
       geminiReq(req, text => {