diff --git a/go.mod b/go.mod index 81f71d5291f..a2953ad78cc 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/maruel/natural v1.1.1 + github.com/meilisearch/meilisearch-go v0.26.1 github.com/minio/sio v0.3.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 @@ -73,6 +74,7 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.2.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -133,7 +135,9 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -142,6 +146,7 @@ require ( github.com/libp2p/go-libp2p v0.27.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -189,6 +194,8 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect diff --git a/go.sum b/go.sum index f3781212a81..9ad983a75b7 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -247,6 +249,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -254,6 +258,10 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= @@ -284,6 +292,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -301,6 +311,8 @@ github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOj github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= +github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= @@ -449,8 +461,13 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -478,6 +495,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -497,6 +515,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -524,6 +543,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 0aee410aab5..4fd99ca2383 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -145,7 +145,7 @@ func InitialSettings() []model.SettingItem { // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, + {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX}, {Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`}, diff --git a/internal/conf/config.go b/internal/conf/config.go index b4664562035..0f1e0048c75 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -1,10 +1,9 @@ package conf import ( - "path/filepath" - "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/pkg/utils/random" + "path/filepath" ) type Database struct { @@ -20,6 +19,12 @@ type Database struct { DSN string `json:"dsn" env:"DSN"` } +type Meilisearch struct { + Host string `json:"host" env:"HOST"` + APIKey string `json:"api_key" env:"API_KEY"` + IndexPrefix string `json:"index_prefix" env:"INDEX_PREFIX"` +} + type Scheme struct { Address string `json:"address" env:"ADDR"` HttpPort int `json:"http_port" env:"HTTP_PORT"` @@ -65,6 +70,7 @@ type Config struct { JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` Database Database `json:"database" envPrefix:"DB_"` + Meilisearch Meilisearch `json:"meilisearch" env:"MEILISEARCH"` Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` @@ -101,6 +107,9 @@ func DefaultConfig() *Config { TablePrefix: "x_", DBFile: dbPath, }, + Meilisearch: Meilisearch{ + Host: "http://localhost:7700", + }, BleveDir: indexDir, Log: LogConfig{ Enable: true, diff --git a/internal/search/import.go b/internal/search/import.go index 822ae3f793f..a34c36f9a34 100644 --- a/internal/search/import.go +++ b/internal/search/import.go @@ -4,4 +4,5 @@ import ( _ "github.com/alist-org/alist/v3/internal/search/bleve" _ "github.com/alist-org/alist/v3/internal/search/db" _ "github.com/alist-org/alist/v3/internal/search/db_non_full_text" + _ "github.com/alist-org/alist/v3/internal/search/meilisearch" ) diff --git a/internal/search/meilisearch/init.go b/internal/search/meilisearch/init.go new file mode 100644 index 00000000000..8f5f24733ee --- /dev/null +++ b/internal/search/meilisearch/init.go @@ -0,0 +1,89 @@ +package meilisearch + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/meilisearch/meilisearch-go" +) + +var config = searcher.Config{ + Name: "meilisearch", + AutoUpdate: true, +} + +func init() { + searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { + m := Meilisearch{ + Client: meilisearch.NewClient(meilisearch.ClientConfig{ + Host: conf.Conf.Meilisearch.Host, + APIKey: conf.Conf.Meilisearch.APIKey, + }), + IndexUid: conf.Conf.Meilisearch.IndexPrefix + "alist", + FilterableAttributes: []string{"parent", "is_dir", "name"}, + SearchableAttributes: []string{"name"}, + } + + _, err := m.Client.GetIndex(m.IndexUid) + if err != nil { + var mErr *meilisearch.Error + ok := errors.As(err, &mErr) + if ok && mErr.MeilisearchApiError.Code == "index_not_found" { + task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{ + Uid: m.IndexUid, + PrimaryKey: "id", + }) + if err != nil { + return nil, err + } + forTask, err := m.Client.WaitForTask(task.TaskUID) + if err != nil { + return nil, err + } + if forTask.Status != meilisearch.TaskStatusSucceeded { + return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status) + } + } else { + return nil, err + } + } + attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes) + if err != nil { + return nil, err + } + } + + attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes) + if err != nil { + return nil, err + } + } + + pagination, err := m.Client.Index(m.IndexUid).GetPagination() + if err != nil { + return nil, err + } + if pagination.MaxTotalHits != int64(model.MaxInt) { + _, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{ + MaxTotalHits: int64(model.MaxInt), + }) + if err != nil { + return nil, err + } + } + return &m, nil + }) +} diff --git a/internal/search/meilisearch/search.go b/internal/search/meilisearch/search.go new file mode 100644 index 00000000000..1516306b75f --- /dev/null +++ b/internal/search/meilisearch/search.go @@ -0,0 +1,227 @@ +package meilisearch + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/google/uuid" + "github.com/meilisearch/meilisearch-go" + "path" + "strings" + "time" +) + +type searchDocument struct { + ID string `json:"id"` + model.SearchNode +} + +type Meilisearch struct { + Client *meilisearch.Client + IndexUid string + FilterableAttributes []string + SearchableAttributes []string +} + +func (m *Meilisearch) Config() searcher.Config { + return config +} + +func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + mReq := &meilisearch.SearchRequest{ + AttributesToSearchOn: m.SearchableAttributes, + Page: int64(req.Page), + HitsPerPage: int64(req.PerPage), + } + if req.Scope != 0 { + mReq.Filter = fmt.Sprintf("is_dir = %v", req.Scope == 1) + } + search, err := m.Client.Index(m.IndexUid).Search(req.Keywords, mReq) + if err != nil { + return nil, 0, err + } + nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) { + srcMap := src.(map[string]any) + return model.SearchNode{ + Parent: srcMap["parent"].(string), + Name: srcMap["name"].(string), + IsDir: srcMap["is_dir"].(bool), + Size: int64(srcMap["size"].(float64)), + }, nil + }) + if err != nil { + return nil, 0, err + } + return nodes, search.TotalHits, nil +} + +func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { + return m.BatchIndex(ctx, []model.SearchNode{node}) +} + +func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { + documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + + return &searchDocument{ + ID: uuid.NewString(), + SearchNode: src, + }, nil + }) + + _, err := m.Client.Index(m.IndexUid).AddDocuments(documents) + if err != nil { + return err + } + + //// Wait for the task to complete and check + //forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{ + // Context: ctx, + // Interval: time.Millisecond * 50, + //}) + //if err != nil { + // return err + //} + //if forTask.Status != meilisearch.TaskStatusSucceeded { + // return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status) + //} + return nil +} + +func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) { + var result meilisearch.DocumentsResult + err := m.Client.Index(m.IndexUid).GetDocuments(&meilisearch.DocumentsQuery{ + Filter: fmt.Sprintf("parent = '%s'", strings.ReplaceAll(parent, "'", "\\'")), + Limit: int64(model.MaxInt), + }, &result) + if err != nil { + return nil, err + } + return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) { + return &searchDocument{ + ID: src["id"].(string), + SearchNode: model.SearchNode{ + Parent: src["parent"].(string), + Name: src["name"].(string), + IsDir: src["is_dir"].(bool), + Size: int64(src["size"].(float64)), + }, + }, nil + }) +} + +func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + result, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) { + return src.SearchNode, nil + }) + +} + +func (m *Meilisearch) getParentsByPrefix(ctx context.Context, parent string) ([]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + parents := []string{parent} + get, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + for _, node := range get { + if node.IsDir { + arr, err := m.getParentsByPrefix(ctx, path.Join(node.Parent, node.Name)) + if err != nil { + return nil, err + } + parents = append(parents, arr...) + } + } + return parents, nil + } +} + +func (m *Meilisearch) DelDirChild(ctx context.Context, prefix string) error { + dfs, err := m.getParentsByPrefix(ctx, utils.FixAndCleanPath(prefix)) + if err != nil { + return err + } + utils.SliceReplace(dfs, func(src string) string { + return "'" + strings.ReplaceAll(src, "'", "\\'") + "'" + }) + s := fmt.Sprintf("parent IN [%s]", strings.Join(dfs, ",")) + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilter(s) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Del(ctx context.Context, prefix string) error { + prefix = utils.FixAndCleanPath(prefix) + dir, name := path.Split(prefix) + get, err := m.getDocumentsByParent(ctx, dir[:len(dir)-1]) + if err != nil { + return err + } + var document *searchDocument + for _, v := range get { + if v.Name == name { + document = v + break + } + } + if document == nil { + // Defensive programming. Document may be the folder, try deleting Child + return m.DelDirChild(ctx, prefix) + } + if document.IsDir { + err = m.DelDirChild(ctx, prefix) + if err != nil { + return err + } + } + task, err := m.Client.Index(m.IndexUid).DeleteDocument(document.ID) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Release(ctx context.Context) error { + return nil +} + +func (m *Meilisearch) Clear(ctx context.Context) error { + _, err := m.Client.Index(m.IndexUid).DeleteAllDocuments() + return err +} + +func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) { + forTask, err := m.Client.WaitForTask(taskUID, meilisearch.WaitParams{ + Context: ctx, + Interval: time.Second, + }) + if err != nil { + return meilisearch.TaskStatusUnknown, err + } + return forTask.Status, nil +} diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go index 73bac93b4d0..842995daaf1 100644 --- a/pkg/utils/slice.go +++ b/pkg/utils/slice.go @@ -29,6 +29,20 @@ func SliceContains[T comparable](arr []T, v T) bool { return false } +// SliceAllContains check if slice all contains elements +func SliceAllContains[T comparable](arr []T, vs ...T) bool { + vsMap := make(map[T]struct{}) + for _, v := range arr { + vsMap[v] = struct{}{} + } + for _, v := range vs { + if _, ok := vsMap[v]; !ok { + return false + } + } + return true +} + // SliceConvert convert slice to another type slice func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) { res := make([]D, 0, len(srcS)) @@ -79,3 +93,9 @@ func SliceFilter[T any](arr []T, filter func(src T) bool) []T { } return res } + +func SliceReplace[T any](arr []T, replace func(src T) T) { + for i, src := range arr { + arr[i] = replace(src) + } +}