From 3648d3ef7ef53939c3bea43a1d40b308eb30da4f Mon Sep 17 00:00:00 2001 From: Mmx Date: Sat, 24 Aug 2024 23:09:13 +0800 Subject: [PATCH] feat: allow database fulltext parser --- drivers/139/driver.go | 4 ++-- drivers/139/types.go | 12 ++++++------ drivers/139/util.go | 2 +- drivers/aliyundrive/util.go | 2 +- drivers/crypt/meta.go | 4 ++-- drivers/google_photo/types.go | 4 ++-- drivers/google_photo/util.go | 16 ++++++++-------- drivers/lark.go | 1 + drivers/seafile/meta.go | 2 +- drivers/seafile/types.go | 2 +- drivers/seafile/util.go | 2 -- drivers/trainbit/meta.go | 18 +++++++++--------- drivers/trainbit/types.go | 2 +- internal/conf/config.go | 1 + internal/offline_download/tool/add.go | 2 +- internal/search/db/init.go | 11 +++++++++-- 16 files changed, 46 insertions(+), 39 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index d33c3d77ebf1..aca717af30eb 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -14,15 +14,15 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) type Yun139 struct { model.Storage Addition - cron *cron.Cron + cron *cron.Cron Account string } diff --git a/drivers/139/types.go b/drivers/139/types.go index f797196624b9..419f6405b9fb 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -236,10 +236,10 @@ type PersonalUploadResp struct { } type RefreshTokenResp struct { - XMLName xml.Name `xml:"root"` - Return string `xml:"return"` - Token string `xml:"token"` - Expiretime int32 `xml:"expiretime"` - AccessToken string `xml:"accessToken"` - Desc string `xml:"desc"` + XMLName xml.Name `xml:"root"` + Return string `xml:"return"` + Token string `xml:"token"` + Expiretime int32 `xml:"expiretime"` + AccessToken string `xml:"accessToken"` + Desc string `xml:"desc"` } diff --git a/drivers/139/util.go b/drivers/139/util.go index 5918e4c53053..409a02116ee4 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -13,9 +13,9 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" - "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" diff --git a/drivers/aliyundrive/util.go b/drivers/aliyundrive/util.go index 0e81b082bb91..36f2422d78d4 100644 --- a/drivers/aliyundrive/util.go +++ b/drivers/aliyundrive/util.go @@ -35,7 +35,7 @@ func (d *AliDrive) createSession() error { "refreshToken": d.RefreshToken, }) }, nil) - if err == nil{ + if err == nil { state.retry = 0 } return err diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index 180773a3f483..eec62a1502cc 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -20,9 +20,9 @@ type Addition struct { EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` - ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ diff --git a/drivers/google_photo/types.go b/drivers/google_photo/types.go index 1a53ae09bfc5..497af0bfece7 100644 --- a/drivers/google_photo/types.go +++ b/drivers/google_photo/types.go @@ -26,7 +26,7 @@ type MediaItem struct { CoverPhotoBaseUrl string `json:"coverPhotoBaseUrl,omitempty"` MimeType string `json:"mimeType,omitempty"` FileName string `json:"filename,omitempty"` - MediaMetadata MediaMetadata `json:"mediaMetadata,omitempty"` + MediaMetadata MediaMetadata `json:"mediaMetadata,omitempty"` } type MediaMetadata struct { @@ -44,7 +44,7 @@ type Video struct { } func fileToObj(f MediaItem) *model.ObjThumb { - if !reflect.DeepEqual(f.MediaMetadata, MediaMetadata{}){ + if !reflect.DeepEqual(f.MediaMetadata, MediaMetadata{}) { return &model.ObjThumb{ Object: model.Object{ ID: f.Id, diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 0fd271b9bb47..818a483882cb 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -11,9 +11,9 @@ import ( // do others that not defined in Driver interface const ( - FETCH_ALL = "all" - FETCH_ALBUMS = "albums" - FETCH_ROOT = "root" + FETCH_ALL = "all" + FETCH_ALBUMS = "albums" + FETCH_ROOT = "root" FETCH_SHARE_ALBUMS = "share_albums" ) @@ -89,15 +89,15 @@ func (d *GooglePhoto) getFiles(id string) ([]MediaItem, error) { func (d *GooglePhoto) getFakeRoot() ([]MediaItem, error) { return []MediaItem{ { - Id: FETCH_ALL, + Id: FETCH_ALL, Title: "全部媒体", }, { - Id: FETCH_ALBUMS, + Id: FETCH_ALBUMS, Title: "全部影集", }, { - Id: FETCH_SHARE_ALBUMS, + Id: FETCH_SHARE_ALBUMS, Title: "共享影集", }, }, nil @@ -131,7 +131,7 @@ func (d *GooglePhoto) getMedias(albumId string) ([]MediaItem, error) { map[string]string{ "fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken", "pageSize": "100", - "albumId": albumId, + "albumId": albumId, "pageToken": "first", }, http.MethodPost) } @@ -163,7 +163,7 @@ func (d *GooglePhoto) getMedia(id string) (MediaItem, error) { return resp, nil } -func (d *GooglePhoto) fetchItems(url string, query map[string]string, method string) ([]MediaItem, error){ +func (d *GooglePhoto) fetchItems(url string, query map[string]string, method string) ([]MediaItem, error) { res := make([]MediaItem, 0) for query["pageToken"] != "" { if query["pageToken"] == "first" { diff --git a/drivers/lark.go b/drivers/lark.go index d50700786516..d255f99105c1 100644 --- a/drivers/lark.go +++ b/drivers/lark.go @@ -1,3 +1,4 @@ +//go:build (linux || darwin || windows) && (amd64 || arm64) // +build linux darwin windows // +build amd64 arm64 diff --git a/drivers/seafile/meta.go b/drivers/seafile/meta.go index fd5255f592b7..0393ee37fbc9 100644 --- a/drivers/seafile/meta.go +++ b/drivers/seafile/meta.go @@ -11,7 +11,7 @@ type Addition struct { Address string `json:"address" required:"true"` UserName string `json:"username" required:"false"` Password string `json:"password" required:"false"` - Token string `json:"token" required:"false"` + Token string `json:"token" required:"false"` RepoId string `json:"repoId" required:"false"` RepoPwd string `json:"repoPwd" required:"false"` } diff --git a/drivers/seafile/types.go b/drivers/seafile/types.go index 47cb322df4af..9918739d6f7e 100644 --- a/drivers/seafile/types.go +++ b/drivers/seafile/types.go @@ -41,4 +41,4 @@ type LibraryInfo struct { LibraryItemResp decryptedTime time.Time decryptedSuccess bool -} \ No newline at end of file +} diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 89b7b0fc39f3..755c2bc556d3 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -174,5 +174,3 @@ func (d *Seafile) decryptLibrary(repo *LibraryInfo) (err error) { repo.decryptedSuccess = true return nil } - - diff --git a/drivers/trainbit/meta.go b/drivers/trainbit/meta.go index 59c09d77e1c6..4009268f2880 100644 --- a/drivers/trainbit/meta.go +++ b/drivers/trainbit/meta.go @@ -8,18 +8,18 @@ import ( type Addition struct { driver.RootID AUSHELLPORTAL string `json:"AUSHELLPORTAL" required:"true"` - ApiKey string `json:"apikey" required:"true"` + ApiKey string `json:"apikey" required:"true"` } var config = driver.Config{ - Name: "Trainbit", - LocalSort: false, - OnlyLocal: false, - OnlyProxy: false, - NoCache: false, - NoUpload: false, - NeedMs: false, - DefaultRoot: "0_000", + Name: "Trainbit", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0_000", } func init() { diff --git a/drivers/trainbit/types.go b/drivers/trainbit/types.go index 4de1a0abdf39..1b30f4c5545a 100644 --- a/drivers/trainbit/types.go +++ b/drivers/trainbit/types.go @@ -1 +1 @@ -package trainbit \ No newline at end of file +package trainbit diff --git a/internal/conf/config.go b/internal/conf/config.go index c5dc9c521bf2..6d7d793d2146 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -18,6 +18,7 @@ type Database struct { TablePrefix string `json:"table_prefix" env:"TABLE_PREFIX"` SSLMode string `json:"ssl_mode" env:"SSL_MODE"` DSN string `json:"dsn" env:"DSN"` + Parser string `json:"parser" env:"PARSER"` } type Meilisearch struct { diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index c7c5c781f710..d1dfb7d06cb5 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -77,7 +77,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { // 防止将下载好的文件删除 deletePolicy = DeleteNever } - + t := &DownloadTask{ Url: args.URL, DstDirPath: args.DstDirPath, diff --git a/internal/search/db/init.go b/internal/search/db/init.go index b7d0288f9474..9f6ddaddba5d 100644 --- a/internal/search/db/init.go +++ b/internal/search/db/init.go @@ -19,19 +19,26 @@ var config = searcher.Config{ func init() { searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { db := db.GetDb() + var parser string switch conf.Conf.Database.Type { case "mysql": + if conf.Conf.Database.Parser != "" { + parser = fmt.Sprintf(" WITH PARSER %s", parser) + } tableName := fmt.Sprintf("%ssearch_nodes", conf.Conf.Database.TablePrefix) - tx := db.Exec(fmt.Sprintf("CREATE FULLTEXT INDEX idx_%s_name_fulltext ON %s(name);", tableName, tableName)) + tx := db.Exec(fmt.Sprintf("CREATE FULLTEXT INDEX idx_%s_name_fulltext%s ON %s(name);", tableName, parser, tableName)) if err := tx.Error; err != nil && !strings.Contains(err.Error(), "Error 1061 (42000)") { // duplicate error log.Errorf("failed to create full text index: %v", err) return nil, err } case "postgres": + if conf.Conf.Database.Parser != "" { + parser = fmt.Sprintf(" gin_%s", parser) + } db.Exec("CREATE EXTENSION pg_trgm;") db.Exec("CREATE EXTENSION btree_gin;") tableName := fmt.Sprintf("%ssearch_nodes", conf.Conf.Database.TablePrefix) - tx := db.Exec(fmt.Sprintf("CREATE INDEX idx_%s_name ON %s USING GIN (name);", tableName, tableName)) + tx := db.Exec(fmt.Sprintf("CREATE INDEX idx_%s_name ON %s USING GIN (name%s);", tableName, parser, tableName)) if err := tx.Error; err != nil && !strings.Contains(err.Error(), "SQLSTATE 42P07") { log.Errorf("failed to create index using GIN: %v", err) return nil, err