From ed04fae7755c820f9715b167e6616749a2b11e2e Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 17:38:03 +0800 Subject: [PATCH 01/11] feat: support metabase 0.49.x --- .../databend/metabase-core/maven-metadata.xml | 5 +- .../metabase-core/maven-metadata.xml.md5 | 2 +- .../metabase-core/maven-metadata.xml.sha1 | 2 +- src/metabase/driver/databend.clj | 155 ++++++++---------- 4 files changed, 69 insertions(+), 95 deletions(-) diff --git a/repo/com/databend/metabase-core/maven-metadata.xml b/repo/com/databend/metabase-core/maven-metadata.xml index 94122b3..2284e58 100644 --- a/repo/com/databend/metabase-core/maven-metadata.xml +++ b/repo/com/databend/metabase-core/maven-metadata.xml @@ -3,10 +3,11 @@ com.databend metabase-core - 1.40 + 1.41 1.40 + 1.41 - 20240527063529 + 20240528085732 diff --git a/repo/com/databend/metabase-core/maven-metadata.xml.md5 b/repo/com/databend/metabase-core/maven-metadata.xml.md5 index 469bc13..fff64a5 100644 --- a/repo/com/databend/metabase-core/maven-metadata.xml.md5 +++ b/repo/com/databend/metabase-core/maven-metadata.xml.md5 @@ -1 +1 @@ -02403d0fcb09a53437e4ab2986dd388f \ No newline at end of file +f5ab9f9594372558da4228baacdbb718 \ No newline at end of file diff --git a/repo/com/databend/metabase-core/maven-metadata.xml.sha1 b/repo/com/databend/metabase-core/maven-metadata.xml.sha1 index 4760c94..31b174e 100644 --- a/repo/com/databend/metabase-core/maven-metadata.xml.sha1 +++ b/repo/com/databend/metabase-core/maven-metadata.xml.sha1 @@ -1 +1 @@ -516a620a9e4ba200bf6527030341c8c5eee491d7 \ No newline at end of file +344de6db540a64af1dd9f9d602aef2c1fccbc83f \ No newline at end of file diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index dd7ccdc..285a054 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -4,7 +4,7 @@ (:require [clojure.java.jdbc :as jdbc] [clojure.string :as str] [clojure.tools.logging :as log] - [honeysql [core :as hsql] [format :as hformat]] + [honey.sql :as sql] [java-time :as t] [medley.core :as m] [metabase [config :as config] [driver :as driver] [util :as u]] @@ -18,10 +18,8 @@ [metabase.driver.sql-jdbc.sync.common :as common] [metabase.driver.sql-jdbc.execute.legacy-impl :as legacy] [metabase.driver.sql-jdbc.sync.interface :as i] - [metabase.mbql.schema :as mbql.s] - [metabase.mbql.util :as mbql.u] + [metabase.util.honey-sql-2 :as h2x] [metabase.util.date-2 :as u.date] - [metabase.util.honeysql-extensions :as hx] [metabase.util.ssh :as ssh] [schema.core :as s]) @@ -78,10 +76,10 @@ (def ^:private default-connection-details {:classname "com.databend.jdbc.DatabendDriver", :subprotocol "databend", :user "root", :password "root", :dbname "default",:host "localhost", :port "8000", :ssl false}) -(def ^:private product-name "metabase/1.4.0") - -(defmethod sql-jdbc.conn/connection-details->spec :databend - [_ details] +(def ^:private product-name "metabase/1.4.1") +(defmethod driver/prettify-native-form :databend [_ native-form] + (sql.u/format-sql-and-fix-params :mysql native-form)) +(defn- connection-details->spec* [details] ;; ensure defaults merge on top of nils (let [details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details)))) default-connection-details @@ -99,11 +97,32 @@ (sql-jdbc.common/handle-additional-options details :separator-style :url)) )) +(defmethod sql-jdbc.conn/connection-details->spec :databend + [_ details] + (connection-details->spec* details)) ; Testing the databend database connection -(defmethod driver/can-connect? :databend [driver details] - (let [connection (sql-jdbc.conn/connection-details->spec driver (ssh/include-ssh-tunnel! details))] - (= 1 (first (vals (first (jdbc/query connection ["SELECT 1"]))))))) +(defmethod driver/can-connect? :databend + [driver details] + (if config/is-test? + (try + ;; Default SELECT 1 is not enough for Metabase test suite, + ;; as it works slightly differently than expected there + (let [spec (sql-jdbc.conn/connection-details->spec driver details) + db (or (:dbname details) (:db details) "default")] + (sql-jdbc.execute/do-with-connection-with-options + driver spec nil + (fn [^java.sql.Connection conn] + (let [stmt (.prepareStatement conn "SELECT count(*) > 0 FROM system.databases WHERE name = ?") + _ (.setString stmt 1 db) + rset (.executeQuery stmt)] + (when (.next rset) + (.getBoolean rset 1)))))) + (catch Throwable e + (log/error e "An exception during Databend connectivity check") + false)) + ;; During normal usage, fall back to the default implementation + (sql-jdbc.conn/can-connect? driver details))) (def ^:private allowed-table-types @@ -184,57 +203,54 @@ (defn- to-start-of-year [expr] - (hsql/call :to_start_of_year (hsql/call :TO_DATETIME expr))) + [:'to_start_of_year expr]) (defn- to-day-of-year [expr] - (hsql/call :to_day_of_year (hsql/call :TO_DATETIME expr))) + [:'to_day_of_year expr]) (defn- to-start-of-week [expr] ;; The first day of a week can be Sunday or Monday, which is specified by the argument mode. ;; Here we use Sunday as default - (hsql/call :to_start_of_week expr)) + [:'to_start_of_week expr]) (defn- to-start-of-minute [expr] - (hsql/call :to_start_of_minute (hsql/call :TO_DATETIME expr))) + [:'to_start_of_minute expr]) (defn- to-start-of-hour [expr] - (hsql/call :to_start_of_hour (hsql/call :TO_DATETIME expr))) + [:'to_start_of_hour expr]) -(defn- to-hour [expr] (hsql/call :to_hour (hsql/call :TO_DATETIME expr))) +(defn- to-hour [expr] [:'to_hour expr]) -(defn- to-minute [expr] (hsql/call :to_minute (hsql/call :TO_DATETIME expr))) +(defn- to-minute [expr] [:'to_minute expr]) -(defn- to-day [expr] (hsql/call :to_date expr)) +(defn- to-day [expr] [:'to_day expr]) (defmethod sql.qp/date [:databend :day-of-week] [_ _ expr] - (sql.qp/adjust-day-of-week :databend (hsql/call :to_day_of_week expr))) + (sql.qp/adjust-day-of-week :databend [:'to_day_of_week expr])) (defn- to-day-of-month [expr] - (hsql/call :to_day_of_month (hsql/call :TO_DATETIME expr))) + [:'to_day_of_month expr]) (defn- to-start-of-month [expr] - (hsql/call :to_start_of_month (hsql/call :TO_DATETIME expr))) + [:'to_start_of_month expr]) (defn- to-start-of-quarter [expr] - (hsql/call :to_start_of_quarter (hsql/call :TO_DATETIME expr))) + [:'to_start_of_quarter expr]) (defmethod sql.qp/date [:databend :default] [_ _ expr] expr) (defmethod sql.qp/date [:databend :minute] [_ _ expr] (to-start-of-minute expr)) -; Return an appropriate HoneySQL form for converting a Unix timestamp integer field or value to a proper SQL Timestamp. -;(defmethod sql.qp/unix-timestamp->honeysql [:databend :seconds] [_ _ expr] (hsql/call :to_timestamp expr)) - (defmethod sql.qp/date [:databend :minute-of-hour] [_ _ expr] (to-minute expr)) @@ -257,9 +273,9 @@ [_ _ expr] (to-start-of-quarter expr)) -;(defmethod sql.qp/unix-timestamp->honeysql [:databend :seconds] -; [_ _ expr] -; (hsql/call :TO_DATETIME expr)) +(defmethod sql.qp/unix-timestamp->honeysql [:databend :seconds] + [_ _ expr] + (h2x/->datetime expr)) (defmethod unprepare/unprepare-value [:databend LocalDate] [_ t] @@ -288,84 +304,34 @@ (defmethod sql.qp/->honeysql [:databend :log] [driver [_ field]] - (hsql/call :log10 (sql.qp/->honeysql driver field))) - -(defmethod hformat/fn-handler "quantile" - [_ field p] - (str "quantile(" (hformat/to-sql p) ")(" (hformat/to-sql field) ")")) + [:'log10 (sql.qp/->honeysql driver field)]) ; call REGEXP_SUBSTR function when regex-match-first is called (defmethod sql.qp/->honeysql [:databend :regex-match-first] [driver [_ arg pattern]] - (let [arg-sql (hformat/to-sql (sql.qp/->honeysql driver arg)) - pattern-sql (sql.u/escape-sql (sql.qp/->honeysql driver pattern) :ansi) - sql-string (str "REGEXP_SUBSTR(" arg-sql ", '" pattern-sql "')")] - (hsql/raw sql-string))) + [:'extract (sql.qp/->honeysql driver arg) pattern]) ;; metabase.query-processor-test.count-where-test ;; metabase.query-processor-test.share-test (defmethod sql.qp/->honeysql [:databend :count-where] [driver [_ pred]] - (hsql/call :case - (hsql/call :> - (hsql/call :count) 0) - (hsql/call :sum - (hsql/call :case (sql.qp/->honeysql driver pred) 1.0 :else 0.0)) - :else nil)) + [:case + [:> [:'count] 0] + [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]] + :else nil]) (defmethod sql.qp/quote-style :databend [_] :mysql) (defmethod sql.qp/add-interval-honeysql-form :databend [_ dt amount unit] - (hx/+ (hx/->timestamp dt) - (hsql/raw (format "INTERVAL %d %s" (int amount) (name unit))))) - -;; The following lines make sure we call lowerUTF8 instead of lower -(defn- databend-like-clause - [driver field value options] - (if (get options :case-sensitive true) - [:like field (sql.qp/->honeysql driver value)] - [:like (hsql/call :lowerUTF8 field) - (sql.qp/->honeysql driver (update value 1 str/lower-case))])) - -(s/defn ^:private update-string-value :- mbql.s/value - [value :- (s/constrained mbql.s/value #(string? (second %)) "string value") f] - (update value 1 f)) - -(defmethod sql.qp/->honeysql [:databendd :starts-with] - [driver [_ field value options]] - (databend-like-clause driver - (sql.qp/->honeysql driver field) - (update-string-value value #(str % \%)) - options)) - -(defmethod sql.qp/->honeysql [:databend :contains] - [driver [_ field value options]] - (databend-like-clause driver - (sql.qp/->honeysql driver field) - (update-string-value value #(str \% % \%)) - options)) - -(defmethod sql.qp/->honeysql [:databend :ends-with] - [driver [_ field value options]] - (databend-like-clause driver - (sql.qp/->honeysql driver field) - (update-string-value value #(str \% %)) - options)) + (h2x/+ dt [:raw (format "INTERVAL %d %s" (int amount) (name unit))])) (defmethod sql.qp/cast-temporal-byte [:databend :Coercion/ISO8601->Time] [_driver _special_type expr] - (hx/->timestamp expr)) + (h2x/->timestamp expr)) -;(defmethod sql-jdbc.execute/read-column-thunk [:databend Types/TIMESTAMP] -; [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] -; (fn [] -; (let [r (.getObject rs i LocalDateTime)] -; (cond (nil? r) nil -; (= (.toLocalDate r) (t/local-date 1970 1 1)) (.toLocalTime r) -; :else r)))) (defmethod sql-jdbc.execute/read-column-thunk [:databend Types/TIMESTAMP_WITH_TIMEZONE] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] @@ -405,10 +371,17 @@ (defmethod driver/display-name :databend [_] "Databend") -(defmethod driver/supports? [:databend :standard-deviation-aggregations] [_ _] true) -(defmethod driver/supports? [:databend :set-timezone] [_ _] true) -(defmethod driver/supports? [:databend :foreign-keys] [_ _] false) -(defmethod driver/supports? [:databend :test/jvm-timezone-setting] [_ _] false) +(doseq [[feature supported?] {:standard-deviation-aggregations true + :foreign-keys false + :set-timezone false + :convert-timezone false + :test/jvm-timezone-setting false + :connection-impersonation false + :schemas true + :datetime-diff true + :upload-with-auto-pk false}] + + (defmethod driver/database-supports? [:databend feature] [_driver _feature _db] supported?)) (defmethod sql-jdbc.sync/db-default-timezone :databend [_ spec] From f3ea6c5c159bda489124286d2dbae818d66c20da Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 17:40:12 +0800 Subject: [PATCH 02/11] remove to_day --- src/metabase/driver/databend.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index 285a054..d9470e5 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -228,7 +228,6 @@ (defn- to-minute [expr] [:'to_minute expr]) -(defn- to-day [expr] [:'to_day expr]) (defmethod sql.qp/date [:databend :day-of-week] [_ _ expr] From d4d58ef0cbf6969629e07766302f56c78eecf7c8 Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 17:41:10 +0800 Subject: [PATCH 03/11] fix --- src/metabase/driver/databend.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index d9470e5..2d1905b 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -77,8 +77,6 @@ (def ^:private default-connection-details {:classname "com.databend.jdbc.DatabendDriver", :subprotocol "databend", :user "root", :password "root", :dbname "default",:host "localhost", :port "8000", :ssl false}) (def ^:private product-name "metabase/1.4.1") -(defmethod driver/prettify-native-form :databend [_ native-form] - (sql.u/format-sql-and-fix-params :mysql native-form)) (defn- connection-details->spec* [details] ;; ensure defaults merge on top of nils (let [details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details)))) From cdd368788934fe3e01c7fbba586b24749ac734cc Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 17:44:34 +0800 Subject: [PATCH 04/11] upgrade metabase to 0.49 in CI --- .github/workflows/check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 6288c27..0b64007 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v2 with: repository: metabase/metabase - ref: v0.47.4 + ref: v0.49.12 - name: Checkout Driver Repo uses: actions/checkout@v2 From 47daa8ca91875fccb43d4589678bd63d035de25c Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 17:47:44 +0800 Subject: [PATCH 05/11] fix --- src/metabase/driver/databend.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index 2d1905b..a7941b2 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -262,7 +262,6 @@ (defmethod sql.qp/date [:databend :month] [_ _ expr] (to-start-of-month expr)) (defmethod sql.qp/date [:databend :year] [_ _ expr] (to-start-of-year expr)) -(defmethod sql.qp/date [:databend :day] [_ _ expr] (to-day expr)) (defmethod sql.qp/date [:databend :week] [driver _ expr] (sql.qp/adjust-start-of-week driver to-start-of-week expr)) From 285f616ccfde910803a9810f90c54a34c1e15291 Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 18:07:46 +0800 Subject: [PATCH 06/11] update reademe --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 0c12952..3ad89d3 100644 --- a/README.md +++ b/README.md @@ -60,3 +60,12 @@ You should see a message on startup similar to: 2019-05-07 23:27:32 INFO plugins.lazy-loaded-driver :: Registering lazy loading driver :databend... 2019-05-07 23:27:32 INFO metabase.driver :: Registered driver :databend (parents: #{:sql-jdbc}) 🚚 ``` + +## Choosing the Right Version + +| Metabase Release | Driver Version | +|------------------|----------------| +| 0.37.x | 0.0.1 | +| 0.38.1+ | 0.0.2 | +| 0.47.7+ | 0.0.7 | +| 0.49.x | 0.0.8 | \ No newline at end of file From bcf08b6ca3887a090d1a0b72e3bab196a3c1cfe2 Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 28 May 2024 18:24:24 +0800 Subject: [PATCH 07/11] update --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3ad89d3..8c9b52b 100644 --- a/README.md +++ b/README.md @@ -67,5 +67,9 @@ You should see a message on startup similar to: |------------------|----------------| | 0.37.x | 0.0.1 | | 0.38.1+ | 0.0.2 | +| 0.41.2 | 0.0.3 | +| 0.41.3.1 | 0.0.4 | +| 0.42.x | 0.0.5 | +| 0.44.x | 0.0.6 | | 0.47.7+ | 0.0.7 | | 0.49.x | 0.0.8 | \ No newline at end of file From 8d9c209c303142e3051146ca90592a058e0258ab Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 29 May 2024 10:23:22 +0800 Subject: [PATCH 08/11] add more honey sql --- src/metabase/driver/databend.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index a7941b2..ad315f3 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -307,6 +307,13 @@ [driver [_ arg pattern]] [:'extract (sql.qp/->honeysql driver arg) pattern]) +(defmethod sql.qp/->honeysql [:clickhouse :stddev] + [driver [_ field]] + [:'stddevPop (sql.qp/->honeysql driver field)]) + +(defmethod sql.qp/->honeysql [:clickhouse :median] + [driver [_ field]] + [:'median (sql.qp/->honeysql driver field)]) ;; metabase.query-processor-test.count-where-test ;; metabase.query-processor-test.share-test From 43f215793c5b0cc097482b5d81c0f2171564402d Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 29 May 2024 10:25:10 +0800 Subject: [PATCH 09/11] fix --- src/metabase/driver/databend.clj | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index ad315f3..6786080 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -307,14 +307,24 @@ [driver [_ arg pattern]] [:'extract (sql.qp/->honeysql driver arg) pattern]) -(defmethod sql.qp/->honeysql [:clickhouse :stddev] +(defmethod sql.qp/->honeysql [:databend :stddev] [driver [_ field]] [:'stddevPop (sql.qp/->honeysql driver field)]) -(defmethod sql.qp/->honeysql [:clickhouse :median] +(defmethod sql.qp/->honeysql [:databend :median] [driver [_ field]] [:'median (sql.qp/->honeysql driver field)]) +(defmethod sql.qp/->honeysql [:databend :substring] + [driver [_ arg start length]] + (let [str [:'toString (sql.qp/->honeysql driver arg)]] + (if length + [:'substring str + (sql.qp/->honeysql driver start) + (sql.qp/->honeysql driver length)] + [:'substring str + (sql.qp/->honeysql driver start)]))) + ;; metabase.query-processor-test.count-where-test ;; metabase.query-processor-test.share-test (defmethod sql.qp/->honeysql [:databend :count-where] From 260e691f1a26a53c12d639dcedcda217ba27a9ad Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 29 May 2024 15:01:10 +0800 Subject: [PATCH 10/11] update --- src/metabase/driver/databend.clj | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index 6786080..934be9d 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -9,6 +9,7 @@ [medley.core :as m] [metabase [config :as config] [driver :as driver] [util :as u]] [metabase.driver.ddl.interface :as ddl.i] + [metabase.driver.sql :as driver.sql] [metabase.driver.sql.util :as sql.u] [metabase.driver.sql-jdbc [common :as sql-jdbc.common] [connection :as sql-jdbc.conn] [execute :as sql-jdbc.execute] @@ -315,6 +316,13 @@ [driver [_ field]] [:'median (sql.qp/->honeysql driver field)]) +(defn- args->float64 + [args] + (map (fn [arg] [:'to_float64 (sql.qp/->honeysql :databend arg)]) args)) + +(defmethod sql.qp/->float :databend + [_ value] + [:'to_float64 value]) (defmethod sql.qp/->honeysql [:databend :substring] [driver [_ arg start length]] (let [str [:'toString (sql.qp/->honeysql driver arg)]] @@ -334,6 +342,7 @@ [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]] :else nil]) + (defmethod sql.qp/quote-style :databend [_] :mysql) (defmethod sql.qp/add-interval-honeysql-form :databend @@ -406,3 +415,7 @@ (defmethod ddl.i/format-name :databend [_ table-or-field-name] (str/replace table-or-field-name #"-" "_")) + +(defmethod driver.sql/set-role-statement :databend + [_ role] + (format "SET ROLE %s;" role)) \ No newline at end of file From 2feffb25fc5223e228509e917d1f250deddb7c01 Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 29 May 2024 15:28:15 +0800 Subject: [PATCH 11/11] fix substring --- src/metabase/driver/databend.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metabase/driver/databend.clj b/src/metabase/driver/databend.clj index 934be9d..050737a 100644 --- a/src/metabase/driver/databend.clj +++ b/src/metabase/driver/databend.clj @@ -327,10 +327,10 @@ [driver [_ arg start length]] (let [str [:'toString (sql.qp/->honeysql driver arg)]] (if length - [:'substring str + [:'substr str (sql.qp/->honeysql driver start) (sql.qp/->honeysql driver length)] - [:'substring str + [:'substr str (sql.qp/->honeysql driver start)]))) ;; metabase.query-processor-test.count-where-test