From d2a42249958e033ecbdd3b7fab825a4b14a41b1b Mon Sep 17 00:00:00 2001 From: Rafael Delboni Date: Sat, 24 Feb 2024 21:59:07 -0300 Subject: [PATCH 1/5] feat: bump db ver, adds db query and boilerplate for tests --- resources/config.edn | 2 +- .../clj/docs/backend/components/db_docs.clj | 23 +++++++---- src/codes/clj/docs/backend/db/datalevin.clj | 34 +++++++++++++++- .../docs/backend/schemas/model/document.clj | 20 +++++++++- src/codes/clj/docs/backend/server.clj | 3 +- .../codes/clj/docs/backend/util.clj | 39 +++++++++++++++++-- 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/resources/config.edn b/resources/config.edn index a43fc57..84f2f19 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -11,5 +11,5 @@ :db-docs {:dir "target" :url "https://github.com/clj-codes/docs.extractor/releases/download" - :version "v0.1.2" + :version "v0.2.1" :file-name "docs-db.zip"}} diff --git a/src/codes/clj/docs/backend/components/db_docs.clj b/src/codes/clj/docs/backend/components/db_docs.clj index 4039f0f..a514179 100644 --- a/src/codes/clj/docs/backend/components/db_docs.clj +++ b/src/codes/clj/docs/backend/components/db_docs.clj @@ -63,7 +63,7 @@ (conn [component] "Returns a database connection")) -(defrecord DbDocs [schema config http conn] +(defrecord DbDocs [schema opts config http conn] component/Lifecycle (start [this] (logs/log :info :datalevin :start) @@ -75,7 +75,7 @@ (logs/log :info :datalevin :db-not-found :downloading :end)) (if conn this - (assoc this :conn (d/get-conn db-path schema))))) + (assoc this :conn (d/get-conn db-path schema opts))))) (stop [this] (logs/log :info :datalevin :stop) (if conn @@ -84,6 +84,7 @@ (assoc this :conn nil)) this)) + DbDocsProvider (db [this] (d/db (:conn this))) @@ -91,10 +92,13 @@ (conn [this] (:conn this))) -(defn new-db-docs [schema] - (map->DbDocs {:schema schema})) +(defn new-db-docs + ([schema opts] + (map->DbDocs {:schema schema :opts opts})) + ([schema] + (map->DbDocs {:schema schema :opts nil}))) -(defrecord DbDocsMock [schema conn db-path] +(defrecord DbDocsMock [schema opts conn db-path] component/Lifecycle (start [this] (logs/log :info :datalevin :start) @@ -104,7 +108,7 @@ (if conn this (assoc this - :conn (d/get-conn db-path schema) + :conn (d/get-conn db-path schema opts) :db-path db-path)))) (stop [this] (logs/log :info :datalevin :stop) @@ -122,5 +126,8 @@ (conn [this] (:conn this))) -(defn new-db-docs-mock [schema] - (map->DbDocsMock {:schema schema})) +(defn new-db-docs-mock + ([schema opts] + (map->DbDocsMock {:schema schema :opts opts})) + ([schema] + (map->DbDocsMock {:schema schema :opts nil}))) diff --git a/src/codes/clj/docs/backend/db/datalevin.clj b/src/codes/clj/docs/backend/db/datalevin.clj index 1b62934..97c9987 100644 --- a/src/codes/clj/docs/backend/db/datalevin.clj +++ b/src/codes/clj/docs/backend/db/datalevin.clj @@ -2,7 +2,16 @@ (:require [codes.clj.docs.backend.components.db-docs :as component.db-docs] [codes.clj.docs.backend.schemas.model.document :as schemas.model.document] [codes.clj.docs.backend.schemas.types :as schemas.types] - [datalevin.core :as d])) + [datalevin.core :as d] + [datalevin.search-utils :as su])) + +(def read-conn-opts + (let [query-analyzer (su/create-analyzer + {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+") + :token-filters [su/lower-case-token-filter]})] + {:search-domains {"project-name" {:query-analyzer query-analyzer} + "namespace-name" {:query-analyzer query-analyzer} + "definition-name" {:query-analyzer query-analyzer}}})) (defn get-projects {:malli/schema [:=> [:cat schemas.types/DatalevinComponent] @@ -49,3 +58,26 @@ (component.db-docs/db db) definition-id) first)) + +(defn search-by-fulltext + {:malli/schema [:=> [:cat :string :int schemas.types/DatalevinComponent] + schemas.model.document/SearchResult]} + [search top db-component] + (let [db (component.db-docs/db db-component)] + + (->> (d/fulltext-datoms db + search + {:top top + :domains ["definition-name" + "namespace-name" + "project-name"]}) + (map first) + (d/pull-many db '[:definition/id + :definition/name + :definition/doc + :namespace/id + :namespace/name + :namespace/doc + :project/id + :project/artifact + :project/group])))) diff --git a/src/codes/clj/docs/backend/schemas/model/document.clj b/src/codes/clj/docs/backend/schemas/model/document.clj index ab7bf13..26e7f82 100644 --- a/src/codes/clj/docs/backend/schemas/model/document.clj +++ b/src/codes/clj/docs/backend/schemas/model/document.clj @@ -1,4 +1,5 @@ -(ns codes.clj.docs.backend.schemas.model.document) +(ns codes.clj.docs.backend.schemas.model.document + (:require [malli.util :as mu])) (def Project [:map @@ -72,3 +73,20 @@ (def Definitions [:sequential Definition]) +(def DefinitionSearchResult + (mu/select-keys Definition [:definition/id + :definition/name + :definition/doc])) + +(def NamespaceSearchResult + (mu/select-keys Namespace [:namespace/id + :namespace/name + :namespace/doc])) + +(def ProjectSearchResult + (mu/select-keys Project [:project/id + :project/artifact + :project/group])) + +(def SearchResult + [:or DefinitionSearchResult NamespaceSearchResult ProjectSearchResult]) diff --git a/src/codes/clj/docs/backend/server.clj b/src/codes/clj/docs/backend/server.clj index f737870..ca9ef04 100644 --- a/src/codes/clj/docs/backend/server.clj +++ b/src/codes/clj/docs/backend/server.clj @@ -1,5 +1,6 @@ (ns codes.clj.docs.backend.server (:require [codes.clj.docs.backend.components.db-docs :as components.db-docs] + [codes.clj.docs.backend.db.datalevin :refer [read-conn-opts]] [codes.clj.docs.backend.routes :as routes] [com.stuartsierra.component :as component] [parenthesin.components.config.aero :as config] @@ -19,7 +20,7 @@ :http (http/new-http) :router (router/new-router routes/routes) :database (component/using (database/new-database) [:config]) - :db-docs (component/using (components.db-docs/new-db-docs {}) [:config :http]) + :db-docs (component/using (components.db-docs/new-db-docs {} read-conn-opts) [:config :http]) :webserver (component/using (webserver/new-webserver) [:config :http :router :database :db-docs]))) (defn start-system! [system-map] diff --git a/test/integration/codes/clj/docs/backend/util.clj b/test/integration/codes/clj/docs/backend/util.clj index 3d2f1f4..e211fda 100644 --- a/test/integration/codes/clj/docs/backend/util.clj +++ b/test/integration/codes/clj/docs/backend/util.clj @@ -1,7 +1,9 @@ (ns integration.codes.clj.docs.backend.util (:require [codes.clj.docs.backend.components.db-docs :as components.db-docs] + [codes.clj.docs.backend.db.datalevin :refer [read-conn-opts]] [codes.clj.docs.backend.server :as server] [com.stuartsierra.component :as component] + [datalevin.search-utils :as su] [parenthesin.components.http.clj-http :as components.http] [parenthesin.helpers.logs :as logs] [parenthesin.helpers.migrations :as migrations] @@ -10,23 +12,54 @@ (def minimal-schema {:project/id {:db/valueType :db.type/string :unique :db.unique/identity} + :project/group {:db/valueType :db.type/string + :db/fulltext true + :db.fulltext/domains ["project" + "project-group"]} + :project/artifact {:db/valueType :db.type/string + :db/fulltext true + :db.fulltext/domains ["project" + "project-name"]} :namespace/id {:db/valueType :db.type/string :unique :db.unique/identity} :namespace/project {:db/valueType :db.type/ref} + :namespace/name {:db/valueType :db.type/string + :db/fulltext true + :db.fulltext/domains ["namespace" + "namespace-name"]} :namespace/doc {:db/valueType :db.type/string - :db/fulltext true} + :db/fulltext true + :db.fulltext/autoDomain true + :db.fulltext/domains ["namespace" + "namespace-doc"]} :definition/id {:db/valueType :db.type/string :unique :db.unique/identity} :definition/namespace {:db/valueType :db.type/ref} + :definition/name {:db/valueType :db.type/string + :db/fulltext true + :db.fulltext/domains ["definition" + "definition-name"]} :definition/doc {:db/valueType :db.type/string - :db/fulltext true}}) + :db/fulltext true + :db.fulltext/domains ["definition" + "definition-doc"]}}) + +(def write-conn-opts + (let [analyzer (su/create-analyzer + {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+") + :token-filters [su/lower-case-token-filter + su/prefix-token-filter]})] + (-> read-conn-opts + (assoc-in [:search-domains "project-name" :analyzer] analyzer) + (assoc-in [:search-domains "namespace-name" :analyzer] analyzer) + (assoc-in [:search-domains "definition-name" :analyzer] analyzer)))) (defn create-and-start-components! [] (component/start-system (merge (server/base-system-map) (component/system-map :http (components.http/new-http-mock {}) - :db-docs (components.db-docs/new-db-docs-mock minimal-schema))))) + :db-docs (components.db-docs/new-db-docs-mock minimal-schema write-conn-opts))))) (defn start-system! ([] From bacc23723385d48ace1c6a004b2efd28c94d7ec9 Mon Sep 17 00:00:00 2001 From: Rafael Delboni Date: Mon, 26 Feb 2024 20:22:42 -0300 Subject: [PATCH 2/5] test: adds integration fulltext search db tests --- .../clj/docs/backend/components/db_docs.clj | 1 - .../clj/docs/backend/controllers/document.clj | 6 ++++++ src/codes/clj/docs/backend/db/datalevin.clj | 2 +- .../docs/backend/schemas/model/document.clj | 2 ++ .../clj/docs/backend/db/datalevin_test.clj | 18 +++++++++++++++++- .../clj/docs/backend/util/db/datalevin.clj | 8 ++++++++ 6 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/codes/clj/docs/backend/components/db_docs.clj b/src/codes/clj/docs/backend/components/db_docs.clj index a514179..bd6d6cd 100644 --- a/src/codes/clj/docs/backend/components/db_docs.clj +++ b/src/codes/clj/docs/backend/components/db_docs.clj @@ -84,7 +84,6 @@ (assoc this :conn nil)) this)) - DbDocsProvider (db [this] (d/db (:conn this))) diff --git a/src/codes/clj/docs/backend/controllers/document.clj b/src/codes/clj/docs/backend/controllers/document.clj index ea29ed7..0494858 100644 --- a/src/codes/clj/docs/backend/controllers/document.clj +++ b/src/codes/clj/docs/backend/controllers/document.clj @@ -26,3 +26,9 @@ [:maybe schemas.model.document/Definition]]} [definition-id {:keys [db-docs]}] (db/get-definition-by-id definition-id db-docs)) + +(defn search-by-fulltext + {:malli/schema [:=> [:cat :string :int schemas.types/DatalevinComponent] + schemas.model.document/SearchResults]} + [search top {:keys [db-docs]}] + (db/search-by-fulltext search top db-docs)) diff --git a/src/codes/clj/docs/backend/db/datalevin.clj b/src/codes/clj/docs/backend/db/datalevin.clj index 97c9987..56ee0f8 100644 --- a/src/codes/clj/docs/backend/db/datalevin.clj +++ b/src/codes/clj/docs/backend/db/datalevin.clj @@ -61,7 +61,7 @@ (defn search-by-fulltext {:malli/schema [:=> [:cat :string :int schemas.types/DatalevinComponent] - schemas.model.document/SearchResult]} + schemas.model.document/SearchResults]} [search top db-component] (let [db (component.db-docs/db db-component)] diff --git a/src/codes/clj/docs/backend/schemas/model/document.clj b/src/codes/clj/docs/backend/schemas/model/document.clj index 26e7f82..62d7781 100644 --- a/src/codes/clj/docs/backend/schemas/model/document.clj +++ b/src/codes/clj/docs/backend/schemas/model/document.clj @@ -90,3 +90,5 @@ (def SearchResult [:or DefinitionSearchResult NamespaceSearchResult ProjectSearchResult]) + +(def SearchResults [:sequential SearchResult]) diff --git a/test/integration/codes/clj/docs/backend/db/datalevin_test.clj b/test/integration/codes/clj/docs/backend/db/datalevin_test.clj index 3ca502c..07ec31f 100644 --- a/test/integration/codes/clj/docs/backend/db/datalevin_test.clj +++ b/test/integration/codes/clj/docs/backend/db/datalevin_test.clj @@ -82,4 +82,20 @@ (flow "find definitions by its id in db" (match? fixtures.document/definition-clojure-pprint-print-table - (util.db.datalevin/get-definition-by-id "org.clojure/clojure/clojure.pprint/print-table/0")))) + (util.db.datalevin/get-definition-by-id "org.clojure/clojure/clojure.pprint/print-table/0"))) + + (flow "serach by fulltext in db" + (match? [#:definition{:id "org.clojure/clojure/clojure.pprint/pprint-logical-block/0" + :doc #"^Execute the body as a pretty printing logical block" + :name "pprint-logical-block"} + #:namespace{:name "clojure.pprint" + :doc #"A Pretty Printer for Clojure" + :id "org.clojure/clojure/clojure.pprint"}] + (util.db.datalevin/search-by-fulltext "pprint" 10)) + (match? [#:namespace{:name "clojure.pprint" + :doc #"^A Pretty Printer for Clojure" + :id "org.clojure/clojure/clojure.pprint"} + #:project{:artifact "clojure" + :group "org.clojure" + :id "org.clojure/clojure"}] + (util.db.datalevin/search-by-fulltext "clojure" 10)))) diff --git a/test/integration/codes/clj/docs/backend/util/db/datalevin.clj b/test/integration/codes/clj/docs/backend/util/db/datalevin.clj index cc4005f..527a772 100644 --- a/test/integration/codes/clj/docs/backend/util/db/datalevin.clj +++ b/test/integration/codes/clj/docs/backend/util/db/datalevin.clj @@ -44,3 +44,11 @@ (->> database (db/get-definition-by-id definition-id) state-flow.api/return))) + +(defn search-by-fulltext + [search top] + (flow "search by fulltext index in document db" + [database (state-flow.api/get-state :db-docs)] + (->> database + (db/search-by-fulltext search top) + state-flow.api/return))) From a4d13010834a0659129ffa0a61f91e88ce215c92 Mon Sep 17 00:00:00 2001 From: Rafael Delboni Date: Mon, 26 Feb 2024 21:30:06 -0300 Subject: [PATCH 3/5] feat: adds adapters/tests and http in --- .../clj/docs/backend/adapters/document.clj | 32 ++++++++++++++++++ .../docs/backend/ports/http_in/document.clj | 7 ++++ src/codes/clj/docs/backend/routes.clj | 33 ++++++++----------- .../backend/schemas/wire/out/document.clj | 10 ++++++ .../docs/backend/adapters/document_test.clj | 9 +++++ 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/codes/clj/docs/backend/adapters/document.clj b/src/codes/clj/docs/backend/adapters/document.clj index 0667931..90a66f4 100644 --- a/src/codes/clj/docs/backend/adapters/document.clj +++ b/src/codes/clj/docs/backend/adapters/document.clj @@ -102,3 +102,35 @@ schemas.wire.out.document/Definitions]} [definitions] (map definition->wire definitions)) + +(defn search-result->wire + {:malli/schema [:=> [:cat schemas.model.document/SearchResult] + schemas.wire.out.document/SearchResult]} + [search-result] + (cond + (:definition/id search-result) + (let [{:definition/keys [id name doc]} search-result] + (enc/assoc-some {:id id + :name name + :type :definition} + :doc doc)) + + (:namespace/id search-result) + (let [{:namespace/keys [id name doc]} search-result] + (enc/assoc-some {:id id + :name name + :type :namespace} + :doc doc)) + + (:project/id search-result) + (let [{:project/keys [id artifact group]} search-result] + (enc/assoc-some {:id id + :name artifact + :type :project} + :group group)))) + +(defn search-results->wire + {:malli/schema [:=> [:cat schemas.model.document/SearchResults] + schemas.wire.out.document/SearchResults]} + [search-results] + (map search-result->wire search-results)) diff --git a/src/codes/clj/docs/backend/ports/http_in/document.clj b/src/codes/clj/docs/backend/ports/http_in/document.clj index e3b2866..45a5162 100644 --- a/src/codes/clj/docs/backend/ports/http_in/document.clj +++ b/src/codes/clj/docs/backend/ports/http_in/document.clj @@ -48,3 +48,10 @@ :definition (adapters.document/definition->wire definition)}}) {:status 404 :body "not found"})) + +(defn search-by-fulltext + [{{{:keys [q top]} :query} :parameters + components :components}] + {:status 200 + :body (-> (controllers.document/search-by-fulltext q (or top 20) components) + adapters.document/search-results->wire)}) diff --git a/src/codes/clj/docs/backend/routes.clj b/src/codes/clj/docs/backend/routes.clj index b94d2bc..c4be829 100644 --- a/src/codes/clj/docs/backend/routes.clj +++ b/src/codes/clj/docs/backend/routes.clj @@ -17,10 +17,10 @@ ["/api" ["/social" + {:swagger {:tags ["social"]}} ["/author" - {:swagger {:tags ["author" "social"]}} - + {:swagger {:tags ["author"]}} ["/" {:post {:summary "create new author" :parameters {:body schemas.wire.in.social/NewAuthor} @@ -41,8 +41,6 @@ :handler ports.http-in.social/get-author}}]] ["/example" - {:swagger {:tags ["example" "social"]}} - ["/" {:post {:summary "create new example" :parameters {:body schemas.wire.in.social/NewExample} @@ -60,8 +58,6 @@ :handler ports.http-in.social/update-example}}]] ["/note" - {:swagger {:tags ["note" "social"]}} - ["/" {:post {:summary "create new note" :parameters {:body schemas.wire.in.social/NewNote} @@ -79,8 +75,6 @@ :handler ports.http-in.social/update-note}}]] ["/see-also" - {:swagger {:tags ["see-also" "social"]}} - ["/" {:post {:summary "create new see-also" :parameters {:body schemas.wire.in.social/NewSeeAlso} @@ -90,8 +84,6 @@ :handler ports.http-in.social/insert-see-also}}]] ["/definition" - {:swagger {:tags ["definition" "social"]}} - ["/{*definition-id}" {:get {:summary "get definition socials list by id" :parameters {:path {:definition-id :string}} @@ -100,18 +92,15 @@ :handler ports.http-in.social/get-by-definition}}]]] ["/document" + {:swagger {:tags ["document"]}} ["/projects" - {:swagger {:tags ["projects" "document"]}} - ["/" {:get {:summary "get project list" :responses {200 {:body schemas.wire.out.document/Projects}} :handler ports.http-in.document/get-projects}}]] ["/namespaces" - {:swagger {:tags ["namespaces" "document"]}} - ["/{*project-id}" {:get {:summary "get namespace list by project id" :parameters {:path {:project-id :string}} @@ -120,8 +109,6 @@ :handler ports.http-in.document/get-namespaces-by-project}}]] ["/definitions" - {:swagger {:tags ["definitions" "document"]}} - ["/{*namespace-id}" {:get {:summary "get definitions list by namespace id" :parameters {:path {:namespace-id :string}} @@ -130,11 +117,19 @@ :handler ports.http-in.document/get-definitions-by-namespace}}]] ["/definition" - {:swagger {:tags ["definition" "document"]}} - ["/{*definition-id}" {:get {:summary "get definition by id" :parameters {:path {:definition-id :string}} :responses {200 {:body schemas.wire.out.document/ProjectNamespaceDefinition} 404 {:body :string}} - :handler ports.http-in.document/get-definition-by-id}}]]]]]) + :handler ports.http-in.document/get-definition-by-id}}]] + + ["/search" + ["/" + {:get {:summary "search documents by fulltext index" + :parameters {:query [:map + [:q :string] + [:top {:optional true} :int]]} + :responses {200 {:body schemas.wire.out.document/SearchResults} + 404 {:body :string}} + :handler ports.http-in.document/search-by-fulltext}}]]]]]) diff --git a/src/codes/clj/docs/backend/schemas/wire/out/document.clj b/src/codes/clj/docs/backend/schemas/wire/out/document.clj index bae5625..75f6af5 100644 --- a/src/codes/clj/docs/backend/schemas/wire/out/document.clj +++ b/src/codes/clj/docs/backend/schemas/wire/out/document.clj @@ -91,3 +91,13 @@ [:project Project] [:namespace Namespace] [:definition Definition]]) + +(def SearchResult + [:map + [:id :string] + [:name :string] + [:type [:enum :definition :namespace :project]] + [:group {:optional true} :string] + [:doc {:optional true} :string]]) + +(def SearchResults [:sequential SearchResult]) diff --git a/test/unit/codes/clj/docs/backend/adapters/document_test.clj b/test/unit/codes/clj/docs/backend/adapters/document_test.clj index 95b9f4f..4f17e71 100644 --- a/test/unit/codes/clj/docs/backend/adapters/document_test.clj +++ b/test/unit/codes/clj/docs/backend/adapters/document_test.clj @@ -38,3 +38,12 @@ (properties/for-all [definitions (mg/generator schemas.model.document/Definitions)] (m/validate schemas.wire.out.document/Definitions (adapters.document/definitions->wire definitions)))) + +(defspec search-result->wire 50 + (properties/for-all [search-result (mg/generator schemas.model.document/SearchResult)] + (m/validate schemas.wire.out.document/SearchResult + (adapters.document/search-result->wire search-result)))) +(defspec search-results->wire 50 + (properties/for-all [search-results (mg/generator schemas.model.document/SearchResults)] + (m/validate schemas.wire.out.document/SearchResults + (adapters.document/search-results->wire search-results)))) From 8b112bb567483921b1a6617f6b2c3ae496211ea0 Mon Sep 17 00:00:00 2001 From: Rafael Delboni Date: Mon, 26 Feb 2024 21:49:40 -0300 Subject: [PATCH 4/5] tests: adds http in integration tests --- .../clj/docs/backend/controllers/document.clj | 2 +- .../codes/clj/docs/backend/document_test.clj | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/codes/clj/docs/backend/controllers/document.clj b/src/codes/clj/docs/backend/controllers/document.clj index 0494858..7207fed 100644 --- a/src/codes/clj/docs/backend/controllers/document.clj +++ b/src/codes/clj/docs/backend/controllers/document.clj @@ -28,7 +28,7 @@ (db/get-definition-by-id definition-id db-docs)) (defn search-by-fulltext - {:malli/schema [:=> [:cat :string :int schemas.types/DatalevinComponent] + {:malli/schema [:=> [:cat :string :int schemas.types/Components] schemas.model.document/SearchResults]} [search top {:keys [db-docs]}] (db/search-by-fulltext search top db-docs)) diff --git a/test/integration/codes/clj/docs/backend/document_test.clj b/test/integration/codes/clj/docs/backend/document_test.clj index 9a7196c..70bd39f 100644 --- a/test/integration/codes/clj/docs/backend/document_test.clj +++ b/test/integration/codes/clj/docs/backend/document_test.clj @@ -229,4 +229,23 @@ (match? {:status 404 :body "not found"} (state-flow.server/request! {:method :get - :uri "/api/document/definition/golang/go/math/abs/0"}))))) + :uri "/api/document/definition/golang/go/math/abs/0"}))) + + (flow "should return search results" + (match? {:status 200 + :body [{:id "org.clojure/clojure/clojure.pprint/pprint-logical-block/0" + :name "pprint-logical-block" + :type "definition" + :doc #"^Execute the body as a pretty printing"} + {:id "org.clojure/clojure/clojure.pprint" + :name "clojure.pprint" + :type "namespace" + :doc #"^A Pretty Printer for Clojure"}]} + (state-flow.server/request! {:method :get + :uri "/api/document/search/?q=pprint"}))) + + (flow "should return empty search results" + (match? {:status 200 + :body []} + (state-flow.server/request! {:method :get + :uri "/api/document/search/?q=golang"}))))) From 02cbe6b7b7459e7735371ace64530ef7d54c43ba Mon Sep 17 00:00:00 2001 From: Rafael Delboni Date: Tue, 27 Feb 2024 17:30:28 -0300 Subject: [PATCH 5/5] chore: update to docs-db v0.3 --- .clj-kondo/config.edn | 3 ++- .lsp/config.edn | 4 +++- resources/config.edn | 4 ++-- src/codes/clj/docs/backend/db/datalevin.clj | 12 +++++++++++- test/integration/codes/clj/docs/backend/util.clj | 8 ++++++-- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index dfe96b7..2715665 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -9,4 +9,5 @@ honeysql.helpers/from]}} :lint-as {honeysql.helpers/defhelper clj-kondo.lint-as/def-catch-all clojure.test.check.properties/for-all clojure.core/let - clojure.test.check.clojure-test/defspec clojure.test/deftest}} + clojure.test.check.clojure-test/defspec clojure.test/deftest + datalevin.interpret/inter-fn clojure.core/fn}} diff --git a/.lsp/config.edn b/.lsp/config.edn index 5ec4bfe..abed646 100644 --- a/.lsp/config.edn +++ b/.lsp/config.edn @@ -5,4 +5,6 @@ flow [[:block 1]] flow-with-defaults [[:block 1]] flow-as-of [[:block 1]] - flow-without-validation [[:block 1]]}}} + flow-without-validation [[:block 1]] + inter-fn [[:inner 0] + [:inner 1]]}}} diff --git a/resources/config.edn b/resources/config.edn index 84f2f19..b9e84db 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -1,5 +1,5 @@ {:webserver/port #long #or [#env PORT 3001] - :webserver/allowed-origins ["http://docs.cljs.codes" "https://docs.cljs.codes" + :webserver/allowed-origins ["http://docs.clj.codes" "https://docs.clj.codes" "http://docs-frontend.fly.dev" "https://docs-frontend.fly.dev"] :database {:dbtype "postgres" @@ -11,5 +11,5 @@ :db-docs {:dir "target" :url "https://github.com/clj-codes/docs.extractor/releases/download" - :version "v0.2.1" + :version "v0.3.0" :file-name "docs-db.zip"}} diff --git a/src/codes/clj/docs/backend/db/datalevin.clj b/src/codes/clj/docs/backend/db/datalevin.clj index 56ee0f8..b1c0c0b 100644 --- a/src/codes/clj/docs/backend/db/datalevin.clj +++ b/src/codes/clj/docs/backend/db/datalevin.clj @@ -3,11 +3,21 @@ [codes.clj.docs.backend.schemas.model.document :as schemas.model.document] [codes.clj.docs.backend.schemas.types :as schemas.types] [datalevin.core :as d] + [datalevin.interpret :refer [inter-fn]] [datalevin.search-utils :as su])) +(defn merge-tokenizers + "Merges the results of tokenizer a and b into one sequence." + [tokenizer-a tokenizer-b] + (inter-fn [^String s] + (into (sequence (tokenizer-a s)) + (sequence (tokenizer-b s))))) + (def read-conn-opts (let [query-analyzer (su/create-analyzer - {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+") + {:tokenizer (merge-tokenizers + (inter-fn [s] [[s 0 0]]) + (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")) :token-filters [su/lower-case-token-filter]})] {:search-domains {"project-name" {:query-analyzer query-analyzer} "namespace-name" {:query-analyzer query-analyzer} diff --git a/test/integration/codes/clj/docs/backend/util.clj b/test/integration/codes/clj/docs/backend/util.clj index e211fda..2780fbd 100644 --- a/test/integration/codes/clj/docs/backend/util.clj +++ b/test/integration/codes/clj/docs/backend/util.clj @@ -1,8 +1,10 @@ (ns integration.codes.clj.docs.backend.util (:require [codes.clj.docs.backend.components.db-docs :as components.db-docs] - [codes.clj.docs.backend.db.datalevin :refer [read-conn-opts]] + [codes.clj.docs.backend.db.datalevin :refer [merge-tokenizers + read-conn-opts]] [codes.clj.docs.backend.server :as server] [com.stuartsierra.component :as component] + [datalevin.interpret :refer [inter-fn]] [datalevin.search-utils :as su] [parenthesin.components.http.clj-http :as components.http] [parenthesin.helpers.logs :as logs] @@ -46,7 +48,9 @@ (def write-conn-opts (let [analyzer (su/create-analyzer - {:tokenizer (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+") + {:tokenizer (merge-tokenizers + (inter-fn [s] [[s 0 0]]) + (su/create-regexp-tokenizer #"[\s:/\.;,!=?\"'()\[\]{}|<>&@#^*\\~`\-]+")) :token-filters [su/lower-case-token-filter su/prefix-token-filter]})] (-> read-conn-opts