move GetGitcommitsummary to not query too much at once

in local testing it never goes about 2s, but the real server slams into 30s+, so, uh, maybe don't load too many commits at once, heh.
Comfy-Org · Sep 14, 2024 · 352769e · 352769e
1 parent ef094c9
commit 352769e
Showing 1 changed file with 82 additions and 56 deletions.
diff --git a/server/implementation/cicd.go b/server/implementation/cicd.go
@@ -126,60 +126,95 @@ func (impl *DripStrictServerImplementation) GetGitcommit(ctx context.Context, re
 func (impl *DripStrictServerImplementation) GetGitcommitsummary(ctx context.Context, request drip.GetGitcommitsummaryRequestObject) (drip.GetGitcommitsummaryResponseObject, error) {
 	log.Ctx(ctx).Info().Msg("Getting git commit summary")
 
-	// TODO: This is implementing as a dense processing at time of request, but this should really be preprocessed in advance
-	// probably even stored in the database
-	query := impl.Client.CIWorkflowResult.Query().
-		WithGitcommit().
-		WithStorageFile()
-
-	// Apply filters
-	repoName := "comfyanonymous/ComfyUI"
-	if request.Params.RepoName != nil {
-		repoName = *request.Params.RepoName
-	}
-	repoName = strings.ToLower(repoName)
-	if request.Params.RepoName != nil {
-		log.Ctx(ctx).Info().Msgf("Filtering git commit by repo name %s", repoName)
-		query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.RepoNameEQ(repoName)))
+	// Prep relevant vars
+	page := 1
+	pageSize := 10
+	if request.Params.Page != nil {
+		page = *request.Params.Page
 	}
-	if request.Params.BranchName != nil {
-		log.Ctx(ctx).Info().Msgf("Filtering git commit by branch name %s", *request.Params.BranchName)
-		query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.BranchNameEQ(*request.Params.BranchName)))
+	if request.Params.PageSize != nil {
+		pageSize = *request.Params.PageSize
 	}
-	query.Order(ciworkflowresult.ByGitcommitField(gitcommit.FieldCommitTimestamp, sql.OrderDesc()))
+	targetCount := (page + 2) * pageSize // Note: +1 for current page, +1 more to avoid partial result at end
+	summaries := make(map[string]*drip.GitCommitSummary)
+	commitsThusFar := 0
+	readCount := 4096
 
-	// Execute the query to get all commits
-	commits, err := query.All(ctx)
-	if err != nil {
-		log.Ctx(ctx).Error().Msgf("Error retrieving git commits w/ err: %v", err)
-		message := fmt.Sprintf("Error retrieving git commits: %v", err)
-		return drip.GetGitcommitsummary500JSONResponse{
-			Message: &message,
-		}, nil
-	}
-	log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits))
+	for len(summaries) < targetCount {
 
-	// Create summaries
-	summaries := make(map[string]*drip.GitCommitSummary)
-	for _, commit := range commits {
-		summary, exists := summaries[commit.Edges.Gitcommit.CommitHash]
-		if !exists {
-			summary = &drip.GitCommitSummary{
-				CommitHash:    &commit.Edges.Gitcommit.CommitHash,
-				Timestamp:     &commit.Edges.Gitcommit.CommitTimestamp,
-				Author:        &commit.Edges.Gitcommit.Author,
-				CommitName:    &commit.Edges.Gitcommit.CommitMessage,
-				BranchName:    &commit.Edges.Gitcommit.BranchName,
-				StatusSummary: &map[string]string{},
+		// TODO: This is implementing as a dense processing at time of request, but this should really be preprocessed in advance
+		// probably even stored in the database
+		query := impl.Client.CIWorkflowResult.Query().
+			WithGitcommit().
+			WithStorageFile()
+
+		// Apply filters
+		repoName := "comfyanonymous/ComfyUI"
+		if request.Params.RepoName != nil {
+			repoName = *request.Params.RepoName
+		}
+		repoName = strings.ToLower(repoName)
+		if request.Params.RepoName != nil {
+			if commitsThusFar == 0 {
+				log.Ctx(ctx).Info().Msgf("Filtering git commit by repo name %s", repoName)
+			}
+			query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.RepoNameEQ(repoName)))
+		}
+		if request.Params.BranchName != nil {
+			if commitsThusFar == 0 {
+				log.Ctx(ctx).Info().Msgf("Filtering git commit by branch name %s", *request.Params.BranchName)
 			}
-			summaries[commit.Edges.Gitcommit.CommitHash] = summary
+			query.Where(ciworkflowresult.HasGitcommitWith(gitcommit.BranchNameEQ(*request.Params.BranchName)))
+		}
+		query.Order(ciworkflowresult.ByGitcommitField(gitcommit.FieldCommitTimestamp, sql.OrderDesc()))
+
+		query.Offset(commitsThusFar).Limit(readCount)
+
+		// Execute the query to get all commits
+		commits, err := query.All(ctx)
+		if err != nil {
+			log.Ctx(ctx).Error().Msgf("Error retrieving git commits w/ err: %v", err)
+			message := fmt.Sprintf("Error retrieving git commits: %v", err)
+			return drip.GetGitcommitsummary500JSONResponse{
+				Message: &message,
+			}, nil
+		}
+		log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits))
+		if len(commits) == 0 {
+			break
 		}
-		_, exists = (*summary.StatusSummary)[commit.OperatingSystem]
-		if !exists {
-			(*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status)
-		} else if commit.Status == schema.WorkflowRunStatusTypeFailed {
-			(*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status)
+
+		// Create summaries
+		for _, commit := range commits {
+			summary, exists := summaries[commit.Edges.Gitcommit.CommitHash]
+			if !exists {
+				summary = &drip.GitCommitSummary{
+					CommitHash:    &commit.Edges.Gitcommit.CommitHash,
+					Timestamp:     &commit.Edges.Gitcommit.CommitTimestamp,
+					Author:        &commit.Edges.Gitcommit.Author,
+					CommitName:    &commit.Edges.Gitcommit.CommitMessage,
+					BranchName:    &commit.Edges.Gitcommit.BranchName,
+					StatusSummary: &map[string]string{},
+				}
+				summaries[commit.Edges.Gitcommit.CommitHash] = summary
+			}
+			_, exists = (*summary.StatusSummary)[commit.OperatingSystem]
+			if !exists {
+				(*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status)
+			} else if commit.Status == schema.WorkflowRunStatusTypeFailed {
+				(*summary.StatusSummary)[commit.OperatingSystem] = string(commit.Status)
+			}
 		}
+		// TODO: This heuristic hack is technically valid, but scaling up the database transfer hurts
+		// And this is all just a placeholder hack anyway so meh
+		/*
+			if commitsThusFar == 0 {
+				// Approximation of how many need to avoid too many loops
+				heuristic := (targetCount * len(commits)) / len(summaries)
+				readCount = max(1, min(10, heuristic+1)) * 4096
+			}*/
+		commitsThusFar += len(commits)
+		log.Ctx(ctx).Info().Msgf("Retrieved %d commits to summarize", len(commits))
 	}
 
 	// Convert map to slice for pagination
@@ -196,15 +231,6 @@ func (impl *DripStrictServerImplementation) GetGitcommitsummary(ctx context.Cont
 	})
 
 	// Pagination
-	page := 1
-	pageSize := 10
-	if request.Params.Page != nil {
-		page = *request.Params.Page
-	}
-	if request.Params.PageSize != nil {
-		pageSize = *request.Params.PageSize
-	}
-
 	start := (page - 1) * pageSize
 	end := start + pageSize
 	if end > len(summarySlice) {