Merge pull request #21 from clj-codes/feat/adds-fulltext-search-api

feat: Adds Fulltext search API

.clj-kondo/config.edn

@@ -9,4 +9,5 @@
                                       honeysql.helpers/from]}}
  :lint-as {honeysql.helpers/defhelper clj-kondo.lint-as/def-catch-all
            clojure.test.check.properties/for-all clojure.core/let
-           clojure.test.check.clojure-test/defspec clojure.test/deftest}}
+           clojure.test.check.clojure-test/defspec clojure.test/deftest
+           datalevin.interpret/inter-fn clojure.core/fn}}

.lsp/config.edn

@@ -5,4 +5,6 @@
                     flow                    [[:block 1]]
                     flow-with-defaults      [[:block 1]]
                     flow-as-of              [[:block 1]]
-                    flow-without-validation [[:block 1]]}}}
+                    flow-without-validation [[:block 1]]
+                    inter-fn                [[:inner 0]
+                                             [:inner 1]]}}}

resources/config.edn

@@ -1,5 +1,5 @@
 {:webserver/port #long #or [#env PORT 3001]
- :webserver/allowed-origins ["http://docs.cljs.codes" "https://docs.cljs.codes"
+ :webserver/allowed-origins ["http://docs.clj.codes" "https://docs.clj.codes"
                              "http://docs-frontend.fly.dev" "https://docs-frontend.fly.dev"]
 
  :database {:dbtype "postgres"
@@ -11,5 +11,5 @@
 
  :db-docs {:dir "target"
            :url "https://github.com/clj-codes/docs.extractor/releases/download"
-           :version "v0.1.2"
+           :version "v0.3.0"
            :file-name "docs-db.zip"}}

src/codes/clj/docs/backend/adapters/document.clj

@@ -102,3 +102,35 @@
                   schemas.wire.out.document/Definitions]}
   [definitions]
   (map definition->wire definitions))
+
+(defn search-result->wire
+  {:malli/schema [:=> [:cat schemas.model.document/SearchResult]
+                  schemas.wire.out.document/SearchResult]}
+  [search-result]
+  (cond
+    (:definition/id search-result)
+    (let [{:definition/keys [id name doc]} search-result]
+      (enc/assoc-some {:id   id
+                       :name name
+                       :type :definition}
+                      :doc doc))
+
+    (:namespace/id search-result)
+    (let [{:namespace/keys [id name doc]} search-result]
+      (enc/assoc-some {:id   id
+                       :name name
+                       :type :namespace}
+                      :doc doc))
+
+    (:project/id search-result)
+    (let [{:project/keys [id artifact group]} search-result]
+      (enc/assoc-some {:id   id
+                       :name artifact
+                       :type :project}
+                      :group group))))
+
+(defn search-results->wire
+  {:malli/schema [:=> [:cat schemas.model.document/SearchResults]
+                  schemas.wire.out.document/SearchResults]}
+  [search-results]
+  (map search-result->wire search-results))

src/codes/clj/docs/backend/components/db_docs.clj

@@ -63,7 +63,7 @@
   (conn [component]
     "Returns a database connection"))
 
-(defrecord DbDocs [schema config http conn]
+(defrecord DbDocs [schema opts config http conn]
   component/Lifecycle
   (start [this]
     (logs/log :info :datalevin :start)
@@ -75,7 +75,7 @@
         (logs/log :info :datalevin :db-not-found :downloading :end))
       (if conn
         this
-        (assoc this :conn (d/get-conn db-path schema)))))
+        (assoc this :conn (d/get-conn db-path schema opts)))))
   (stop [this]
     (logs/log :info :datalevin :stop)
     (if conn
@@ -91,10 +91,13 @@
   (conn [this]
     (:conn this)))
 
-(defn new-db-docs [schema]
-  (map->DbDocs {:schema schema}))
+(defn new-db-docs
+  ([schema opts]
+   (map->DbDocs {:schema schema :opts opts}))
+  ([schema]
+   (map->DbDocs {:schema schema :opts nil})))
 
-(defrecord DbDocsMock [schema conn db-path]
+(defrecord DbDocsMock [schema opts conn db-path]
   component/Lifecycle
   (start [this]
     (logs/log :info :datalevin :start)
@@ -104,7 +107,7 @@
       (if conn
         this
         (assoc this
-               :conn (d/get-conn db-path schema)
+               :conn (d/get-conn db-path schema opts)
                :db-path db-path))))
   (stop [this]
     (logs/log :info :datalevin :stop)
@@ -122,5 +125,8 @@
   (conn [this]
     (:conn this)))
 
-(defn new-db-docs-mock [schema]
-  (map->DbDocsMock {:schema schema}))
+(defn new-db-docs-mock
+  ([schema opts]
+   (map->DbDocsMock {:schema schema :opts opts}))
+  ([schema]
+   (map->DbDocsMock {:schema schema :opts nil})))

src/codes/clj/docs/backend/controllers/document.clj

@@ -26,3 +26,9 @@
                   [:maybe schemas.model.document/Definition]]}
   [definition-id {:keys [db-docs]}]
   (db/get-definition-by-id definition-id db-docs))
+
+(defn search-by-fulltext
+  {:malli/schema [:=> [:cat :string :int schemas.types/Components]
+                  schemas.model.document/SearchResults]}
+  [search top {:keys [db-docs]}]
+  (db/search-by-fulltext search top db-docs))

src/codes/clj/docs/backend/db/datalevin.clj

@@ -2,7 +2,26 @@
   (:require [codes.clj.docs.backend.components.db-docs :as component.db-docs]
             [codes.clj.docs.backend.schemas.model.document :as schemas.model.document]
             [codes.clj.docs.backend.schemas.types :as schemas.types]
-            [datalevin.core :as d]))
+            [datalevin.core :as d]
+            [datalevin.interpret :refer [inter-fn]]
+            [datalevin.search-utils :as su]))
+
+(defn merge-tokenizers
+  "Merges the results of tokenizer a and b into one sequence."
+  [tokenizer-a tokenizer-b]
+  (inter-fn [^String s]
+    (into (sequence (tokenizer-a s))
+      (sequence (tokenizer-b s)))))
+
+(def read-conn-opts
+  (let [query-analyzer (su/create-analyzer
+                        {:tokenizer (merge-tokenizers
+                                     (inter-fn [s] [[s 0 0]])
+                                     (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+"))
+                         :token-filters [su/lower-case-token-filter]})]
+    {:search-domains {"project-name" {:query-analyzer query-analyzer}
+                      "namespace-name" {:query-analyzer query-analyzer}
+                      "definition-name" {:query-analyzer query-analyzer}}}))
 
 (defn get-projects
   {:malli/schema [:=> [:cat schemas.types/DatalevinComponent]
@@ -49,3 +68,26 @@
            (component.db-docs/db db)
            definition-id)
       first))
+
+(defn search-by-fulltext
+  {:malli/schema [:=> [:cat :string :int schemas.types/DatalevinComponent]
+                  schemas.model.document/SearchResults]}
+  [search top db-component]
+  (let [db (component.db-docs/db db-component)]
+
+    (->> (d/fulltext-datoms db
+                            search
+                            {:top top
+                             :domains ["definition-name"
+                                       "namespace-name"
+                                       "project-name"]})
+         (map first)
+         (d/pull-many db '[:definition/id
+                           :definition/name
+                           :definition/doc
+                           :namespace/id
+                           :namespace/name
+                           :namespace/doc
+                           :project/id
+                           :project/artifact
+                           :project/group]))))

src/codes/clj/docs/backend/ports/http_in/document.clj

@@ -48,3 +48,10 @@
               :definition (adapters.document/definition->wire definition)}})
     {:status 404
      :body "not found"}))
+
+(defn search-by-fulltext
+  [{{{:keys [q top]} :query} :parameters
+    components :components}]
+  {:status 200
+   :body (-> (controllers.document/search-by-fulltext q (or top 20) components)
+             adapters.document/search-results->wire)})

src/codes/clj/docs/backend/routes.clj

@@ -17,10 +17,10 @@
    ["/api"
 
     ["/social"
+     {:swagger {:tags ["social"]}}
 
      ["/author"
-      {:swagger {:tags ["author" "social"]}}
-
+      {:swagger {:tags ["author"]}}
       ["/"
        {:post {:summary "create new author"
                :parameters {:body schemas.wire.in.social/NewAuthor}
@@ -41,8 +41,6 @@
               :handler ports.http-in.social/get-author}}]]
 
      ["/example"
-      {:swagger {:tags ["example" "social"]}}
-
       ["/"
        {:post {:summary "create new example"
                :parameters {:body schemas.wire.in.social/NewExample}
@@ -60,8 +58,6 @@
               :handler ports.http-in.social/update-example}}]]
 
      ["/note"
-      {:swagger {:tags ["note" "social"]}}
-
       ["/"
        {:post {:summary "create new note"
                :parameters {:body schemas.wire.in.social/NewNote}
@@ -79,8 +75,6 @@
               :handler ports.http-in.social/update-note}}]]
 
      ["/see-also"
-      {:swagger {:tags ["see-also" "social"]}}
-
       ["/"
        {:post {:summary "create new see-also"
                :parameters {:body schemas.wire.in.social/NewSeeAlso}
@@ -90,8 +84,6 @@
                :handler ports.http-in.social/insert-see-also}}]]
 
      ["/definition"
-      {:swagger {:tags ["definition" "social"]}}
-
       ["/{*definition-id}"
        {:get {:summary "get definition socials list by id"
               :parameters {:path {:definition-id :string}}
@@ -100,18 +92,15 @@
               :handler ports.http-in.social/get-by-definition}}]]]
 
     ["/document"
+     {:swagger {:tags ["document"]}}
 
      ["/projects"
-      {:swagger {:tags ["projects" "document"]}}
-
       ["/"
        {:get {:summary "get project list"
               :responses {200 {:body schemas.wire.out.document/Projects}}
               :handler ports.http-in.document/get-projects}}]]
 
      ["/namespaces"
-      {:swagger {:tags ["namespaces" "document"]}}
-
       ["/{*project-id}"
        {:get {:summary "get namespace list by project id"
               :parameters {:path {:project-id :string}}
@@ -120,8 +109,6 @@
               :handler ports.http-in.document/get-namespaces-by-project}}]]
 
      ["/definitions"
-      {:swagger {:tags ["definitions" "document"]}}
-
       ["/{*namespace-id}"
        {:get {:summary "get definitions list by namespace id"
               :parameters {:path {:namespace-id :string}}
@@ -130,11 +117,19 @@
               :handler ports.http-in.document/get-definitions-by-namespace}}]]
 
      ["/definition"
-      {:swagger {:tags ["definition" "document"]}}
-
       ["/{*definition-id}"
        {:get {:summary "get definition by id"
               :parameters {:path {:definition-id :string}}
               :responses {200 {:body schemas.wire.out.document/ProjectNamespaceDefinition}
                           404 {:body :string}}
-              :handler ports.http-in.document/get-definition-by-id}}]]]]])
+              :handler ports.http-in.document/get-definition-by-id}}]]
+
+     ["/search"
+      ["/"
+       {:get {:summary "search documents by fulltext index"
+              :parameters {:query [:map
+                                   [:q :string]
+                                   [:top {:optional true} :int]]}
+              :responses {200 {:body schemas.wire.out.document/SearchResults}
+                          404 {:body :string}}
+              :handler ports.http-in.document/search-by-fulltext}}]]]]])

src/codes/clj/docs/backend/schemas/model/document.clj

@@ -1,4 +1,5 @@
-(ns codes.clj.docs.backend.schemas.model.document)
+(ns codes.clj.docs.backend.schemas.model.document
+  (:require [malli.util :as mu]))
 
 (def Project
   [:map
@@ -72,3 +73,22 @@
 
 (def Definitions [:sequential Definition])
 
+(def DefinitionSearchResult
+  (mu/select-keys Definition [:definition/id
+                              :definition/name
+                              :definition/doc]))
+
+(def NamespaceSearchResult
+  (mu/select-keys Namespace [:namespace/id
+                             :namespace/name
+                             :namespace/doc]))
+
+(def ProjectSearchResult
+  (mu/select-keys Project [:project/id
+                           :project/artifact
+                           :project/group]))
+
+(def SearchResult
+  [:or DefinitionSearchResult NamespaceSearchResult ProjectSearchResult])
+
+(def SearchResults [:sequential SearchResult])

src/codes/clj/docs/backend/schemas/wire/out/document.clj

@@ -91,3 +91,13 @@
    [:project Project]
    [:namespace Namespace]
    [:definition Definition]])
+
+(def SearchResult
+  [:map
+   [:id :string]
+   [:name :string]
+   [:type [:enum :definition :namespace :project]]
+   [:group {:optional true} :string]
+   [:doc {:optional true} :string]])
+
+(def SearchResults [:sequential SearchResult])

src/codes/clj/docs/backend/server.clj

@@ -1,5 +1,6 @@
 (ns codes.clj.docs.backend.server
   (:require [codes.clj.docs.backend.components.db-docs :as components.db-docs]
+            [codes.clj.docs.backend.db.datalevin :refer [read-conn-opts]]
             [codes.clj.docs.backend.routes :as routes]
             [com.stuartsierra.component :as component]
             [parenthesin.components.config.aero :as config]
@@ -19,7 +20,7 @@
    :http (http/new-http)
    :router (router/new-router routes/routes)
    :database (component/using (database/new-database) [:config])
-   :db-docs (component/using (components.db-docs/new-db-docs {}) [:config :http])
+   :db-docs (component/using (components.db-docs/new-db-docs {} read-conn-opts) [:config :http])
    :webserver (component/using (webserver/new-webserver) [:config :http :router :database :db-docs])))
 
 (defn start-system! [system-map]

test/integration/codes/clj/docs/backend/db/datalevin_test.clj

@@ -82,4 +82,20 @@
 
   (flow "find definitions by its id in db"
     (match? fixtures.document/definition-clojure-pprint-print-table
-            (util.db.datalevin/get-definition-by-id "org.clojure/clojure/clojure.pprint/print-table/0"))))
+            (util.db.datalevin/get-definition-by-id "org.clojure/clojure/clojure.pprint/print-table/0")))
+
+  (flow "serach by fulltext in db"
+    (match? [#:definition{:id "org.clojure/clojure/clojure.pprint/pprint-logical-block/0"
+                          :doc #"^Execute the body as a pretty printing logical block"
+                          :name "pprint-logical-block"}
+             #:namespace{:name "clojure.pprint"
+                         :doc #"A Pretty Printer for Clojure"
+                         :id "org.clojure/clojure/clojure.pprint"}]
+            (util.db.datalevin/search-by-fulltext "pprint" 10))
+    (match? [#:namespace{:name "clojure.pprint"
+                         :doc #"^A Pretty Printer for Clojure"
+                         :id "org.clojure/clojure/clojure.pprint"}
+             #:project{:artifact "clojure"
+                       :group "org.clojure"
+                       :id "org.clojure/clojure"}]
+            (util.db.datalevin/search-by-fulltext "clojure" 10))))