From 0ec7485bb08f61544ffa0fe84fcb83838ef83781 Mon Sep 17 00:00:00 2001 From: Sjoerd Riemersma Date: Tue, 11 May 2021 14:04:50 +0200 Subject: [PATCH] storing jwt session on disk - saves on approving each time. dowloading files now supports recursive downloading. fixed an issue where a response would panic when an jwt expired --- cmd/afosto/files/download.go | 117 +++++++++++++++++++--------------- cmd/afosto/files/upload.go | 5 ++ cmd/afosto/template/render.go | 5 ++ pkg/auth/listener.go | 21 ++++++ pkg/client/client.go | 33 +++++++++- pkg/data/user.go | 30 +++++++++ 6 files changed, 158 insertions(+), 53 deletions(-) diff --git a/cmd/afosto/files/download.go b/cmd/afosto/files/download.go index d01feaa..1fe16b8 100644 --- a/cmd/afosto/files/download.go +++ b/cmd/afosto/files/download.go @@ -1,23 +1,26 @@ package files import ( - "bytes" "github.com/afosto/cli/pkg/auth" "github.com/afosto/cli/pkg/client" "github.com/afosto/cli/pkg/data" "github.com/afosto/cli/pkg/logging" "github.com/gen2brain/dlgs" "github.com/spf13/cobra" - "io" + "io/ioutil" "net/url" "os" - "path/filepath" "strings" "sync" ) func download(cmd *cobra.Command, args []string) { user := auth.GetUser() + + if user == nil { + user = auth.LoadFromStorage() + } + if user == nil { user = auth.GetImplicitUser([]string{ "openid", @@ -28,89 +31,99 @@ func download(cmd *cobra.Command, args []string) { } ac := client.GetClient(user.TenantID, user.GetAccessToken()) + source, err := cmd.Flags().GetString("source") if err != nil { logging.Log.Fatal(err) } - if source == "" { - selectedSource, ok, err := dlgs.File("Select source path", "", true) + source = strings.Trim(source, "/") + + destination, err := cmd.Flags().GetString("destination") + + if destination == "" { + selectedDestination, ok, err := dlgs.File("Select the download directory", "", true) if err != nil { logging.Log.Fatal(err) } if !ok { - logging.Log.Fatal("failed to select a source path") + logging.Log.Fatal("Failed to select a directory") } - source = selectedSource + destination = selectedDestination } - destination, err := cmd.Flags().GetString("destination") - if err != nil { - logging.Log.Fatal(err) - } destination = strings.TrimRight(destination, "/") - if _, err := os.Stat(destination); os.IsNotExist(err) { - err := os.Mkdir(destination, 0755) - if err != nil { - logging.Log.Fatal("Destination path [" + destination + "] does not yet exist and could not create it") - } - } - downloadQueue := make(chan data.File, 10) + var wg sync.WaitGroup - go downloadHandler(downloadQueue, ac, destination, &wg) - cursor := "" - var files []data.File - for { - files, cursor, err = ac.ListDirectory(source, cursor) - if err != nil { - logging.Log.Fatal(err) - } - for _, file := range files { - wg.Add(1) - downloadQueue <- file - } - if len(files) < 25 { - break + go downloadHandler(downloadQueue, ac, source, destination, &wg) + logging.Log.Infof("✔ Started listing Directories`") + directories, err := ac.ListDirectories(source) + if err != nil { + logging.Log.Fatal(err) + } + logging.Log.Infof("✔ Finished listing directories`") + + for _, directory := range directories { + var files []data.File + cursor := "" + for { + files, cursor, err = ac.ListDirectory(strings.TrimRight(directory, "/"), cursor) + if err != nil { + logging.Log.Fatal(err) + } + for _, file := range files { + wg.Add(1) + downloadQueue <- file + } + if len(files) < 25 { + break + } } } + wg.Wait() logging.Log.Infof("✔ Downloaded all files from `%s` to `%s`", source, destination) } -func downloadHandler(downloadQueue <-chan data.File, ac *client.AfostoClient, destination string, wg *sync.WaitGroup) { +func downloadHandler(downloadQueue <-chan data.File, ac *client.AfostoClient, source string, destination string, wg *sync.WaitGroup) { for file := range downloadQueue { - go func(file data.File, wg *sync.WaitGroup) { - u, err := url.Parse(file.Url) + go func(file data.File, source string, destination string, wg *sync.WaitGroup) { + defer wg.Done() + fileUri, err := url.Parse(file.Url) + if err != nil { logging.Log.Error(err) - + return } - b, err := ac.Download(u) - logging.Log.Infof("✔ Downloaded `%s` on from `%s`", file.Filename, file.Url) - path := destination + "/" + file.Dir + "/" + file.Filename - targetPath := filepath.Dir(path) - if _, err := os.Stat(targetPath); os.IsNotExist(err) { - err := os.Mkdir(targetPath, 0755) - if err != nil { - logging.Log.Error("Destination path does not yet exist and could not create it") - - } + trimmedPath := source + if idx := strings.LastIndex(trimmedPath, "/"); idx != -1 { + trimmedPath = trimmedPath[0 : idx+1] } - out, err := os.Create(path) + + destinationDir := destination + strings.TrimLeft(file.Dir, source) + + b, err := ac.Download(fileUri) if err != nil { logging.Log.Error(err) + return } - defer out.Close() - _, err = io.Copy(out, bytes.NewReader(b)) - if err != nil { + + if err := os.MkdirAll(destinationDir, 0755); err != nil { + logging.Log.Error("✗ Destination path does not yet exist and could not create it") + return + + } + + if err := ioutil.WriteFile(destinationDir+"/"+file.Filename, b, 0664); err != nil { logging.Log.Error(err) } - wg.Done() - }(file, wg) + logging.Log.Infof("✔ Downloaded `%s` on from `%s`", file.Filename, file.Url) + + }(file, source, destination, wg) } } diff --git a/cmd/afosto/files/upload.go b/cmd/afosto/files/upload.go index c7e97a0..3fb1fc9 100644 --- a/cmd/afosto/files/upload.go +++ b/cmd/afosto/files/upload.go @@ -48,6 +48,11 @@ func GetCommands() []*cobra.Command { func upload(cmd *cobra.Command, args []string) { user := auth.GetUser() + + if user == nil { + user = auth.LoadFromStorage() + } + if user == nil { user = auth.GetImplicitUser([]string{ "openid", diff --git a/cmd/afosto/template/render.go b/cmd/afosto/template/render.go index 30b43ac..3bd1565 100644 --- a/cmd/afosto/template/render.go +++ b/cmd/afosto/template/render.go @@ -29,6 +29,11 @@ func GetCommands() []*cobra.Command { func Render(cmd *cobra.Command, args []string) { user := auth.GetUser() + + if user == nil { + user = auth.LoadFromStorage() + } + if user == nil { user = auth.GetImplicitUser([]string{ "openid", diff --git a/pkg/auth/listener.go b/pkg/auth/listener.go index 6eb3366..6ec0a37 100644 --- a/pkg/auth/listener.go +++ b/pkg/auth/listener.go @@ -2,11 +2,13 @@ package auth import ( "context" + "encoding/json" "github.com/afosto/cli/pkg/client" "github.com/afosto/cli/pkg/data" "github.com/afosto/cli/pkg/logging" "github.com/dgrijalva/jwt-go" "github.com/pkg/browser" + "io/ioutil" "net/http" "sync" ) @@ -27,6 +29,21 @@ func GetUser() *data.User { return user } +func LoadFromStorage() *data.User { + if data2, err := ioutil.ReadFile("user.json"); err == nil { + loadedUser := data.User{} + if err := json.Unmarshal(data2, &loadedUser); err == nil { + claims := jwt.StandardClaims{} + parser := jwt.Parser{} + parser.ParseUnverified(loadedUser.GetAccessToken(), &claims) + if err := claims.Valid(); err == nil { + return &loadedUser + } + } + } + return nil +} + func GetImplicitUser(permissions []string) *data.User { err := browser.OpenURL(client.GetAuthorizationURL(permissions)) if err != nil { @@ -38,6 +55,10 @@ func GetImplicitUser(permissions []string) *data.User { resource.await() user = resource.user + + if userData, err := json.Marshal(user); err == nil { + _ = ioutil.WriteFile("user.json", userData, 0644) + } return user } diff --git a/pkg/client/client.go b/pkg/client/client.go index 7048a2f..d634790 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -71,7 +71,7 @@ func GetClient(tenantID string, accessToken string) *AfostoClient { c: cache.New(time.Minute*5, time.Minute), } cl.client = &http.Client{ - Timeout: time.Second * 10, + Timeout: time.Second * 30, Transport: &tripper{ accessToken: accessToken, tenantID: tenantID, @@ -157,6 +157,33 @@ func (ac *AfostoClient) ListDirectory(dir string, cursor string) ([]data.File, s return files, cursorResponse, nil } +func (ac *AfostoClient) ListDirectories(dir string) ([]string, error) { + + req, _ := http.NewRequest("GET", fmt.Sprintf("%s/%s", BaseApiUrl, "cnt/directories"), nil) + + directories := struct { + Directories []string `json:"directories"` + }{} + + resp, err := ac.client.Do(req) + + _ = resp + b, _, err := handle(resp, err) + if err != nil { + return nil, err + } + _ = json.Unmarshal(b, &directories) + + list := []string{} + + for _, directory := range directories.Directories { + if strings.HasPrefix(directory, dir) { + list = append(list, directory) + } + } + return list, nil +} + func (ac *AfostoClient) Upload(sourceFilePath string, labelFilename string, signature string) (*data.File, error) { file, err := os.Open(sourceFilePath) if err != nil { @@ -233,6 +260,10 @@ func jsonPayload(payload interface{}) *bytes.Reader { } func handle(res *http.Response, err error) ([]byte, map[string][]string, error) { + if err != nil { + return nil, nil, err + } + defer res.Body.Close() b, err := ioutil.ReadAll(res.Body) if err != nil { diff --git a/pkg/data/user.go b/pkg/data/user.go index ef23f69..e44228f 100644 --- a/pkg/data/user.go +++ b/pkg/data/user.go @@ -1,5 +1,9 @@ package data +import ( + "encoding/json" +) + type User struct { ID string `json:"id"` Name string `json:"name"` @@ -16,3 +20,29 @@ func (u *User) SetAccessToken(token string) { func (u *User) GetAccessToken() string { return u.accessToken } + +func (u *User) MarshalJSON() ([]byte, error) { + type Alias User + return json.Marshal(&struct { + AccessToken string `json:"token"` + *Alias + }{ + AccessToken: u.GetAccessToken(), + Alias: (*Alias)(u), + }) +} + +func (u *User) UnmarshalJSON(data []byte) error { + type Alias User + aux := &struct { + AccessToken string `json:"token"` + *Alias + }{ + Alias: (*Alias)(u), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + u.accessToken = aux.AccessToken + return nil +}