WIP: Issue/718 create worker for scanning

assets/js/Components/App.js

@@ -8,6 +8,7 @@ import AboutModal from './AboutModal'
 import { View } from '@instructure/ui-view'
 import Api from '../Services/Api'
 import MessageTray from './MessageTray'
+import CourseUpdateProgress from './CourseUpdateProgress'
 import FilesPage from './FilesPage'
 import SummaryBar from './SummaryBar'
 
@@ -20,14 +21,20 @@ class App extends React.Component {
     this.settings = props.settings
     this.reportHistory = []
     this.messages = props.messages
-    this.newReportInterval = 5000
 
     this.state = {
       report: this.initialReport,
       navigation: 'welcome',
       modal: null,
       syncComplete: false,
       hasNewReport: false,
+      progress: 0,
+      totalIssues: 0,
+      scanCompleted: false,
+      contentUpdateCompleted: false,
+      title: "",
+      updateInterval: 0,
+      scanInterval: 0,
     }
 
     this.handleNavigation = this.handleNavigation.bind(this)
@@ -41,6 +48,11 @@ class App extends React.Component {
     this.handleCourseRescan = this.handleCourseRescan.bind(this)
     this.handleNewReport = this.handleNewReport.bind(this)
     this.resizeFrame = this.resizeFrame.bind(this)
+    this.pollBackgroundWorker = this.pollBackgroundWorker.bind(this)
+    this.fetchLoadData = this.fetchLoadData.bind(this)
+    this.fetchUpdateData = this.fetchUpdateData.bind(this)
+    this.getReport = this.getReport.bind(this)
+    this.getLatestReport = this.getLatestReport.bind(this)
   }
 
   render() {
@@ -60,7 +72,12 @@ class App extends React.Component {
         }
 
         <MessageTray messages={this.messages} t={this.t} clearMessages={this.clearMessages} hasNewReport={this.state.syncComplete} />
-
+
+        <CourseUpdateProgress
+          progress={this.state.progress}
+          contentUpdateCompleted={this.state.contentUpdateCompleted} 
+          title = {this.state.title}/>
+
         <main role="main">
           {('welcome' === this.state.navigation) &&
             <WelcomePage
@@ -127,14 +144,32 @@ class App extends React.Component {
 
     this.scanCourse()
       .then((response) => response.json())
-      .then(this.handleNewReport)
+      .then(() => this.getContentUpdate())
+      .then(() => this.pollBackgroundWorker())
+      .then(() => this.getReport())
 
       // update iframe height on resize
       window.addEventListener("resize", this.resizeFrame);
 
       this.resizeFrame();
   }
 
+  getReport() {
+    if(this.state.scanCompleted === true) {
+      this.getLatestReport()
+        .then((response) => response.json())
+        .then(this.handleNewReport)
+    }
+    else {
+      setTimeout(this.getReport, 500);
+    }
+  }
+
+  getLatestReport() {
+    let api = new Api(this.settings)
+    return api.getLatestReport(this.settings.course.id)
+  }
+
   componentWillUnmount() {
     window.removeEventListener('resize', this.resizeFrame);
   }
@@ -154,14 +189,88 @@ class App extends React.Component {
 
   handleCourseRescan() {
     if (this.state.hasNewReport) {
-      this.setState({ hasNewReport: false, syncComplete: false })
+      this.setState({hasNewReport: false, 
+                     syncComplete: false, 
+                     contentUpdateCompleted: false,
+                     scanCompleted: false, 
+                     progress: 0})
       this.scanCourse()
         .then((response) => response.json())
-        .then(this.handleNewReport)
+        .then(() => this.getContentUpdate())
+        .then(() => this.pollBackgroundWorker())
+        .then(() => this.getReport())
     }
     this.forceUpdate()
   }
 
+  // As the worker process updates UDOIT with the latest content items, this method checks on whether or not
+  // This process has been completed
+  fetchUpdateData() {
+    fetch(`http://${window.location.hostname}:8000/udoit3/api/progress`)
+      .then(response => response.json())
+      .then(data => {
+        this.setState({ 
+          progress: data.progress, 
+          title: data.title
+        }, () => { 
+          const progress = this.state.progress
+          const interval = this.state.updateInterval
+          const title = this.state.title
+          if(title != "Getting content from Canvas") {
+            clearInterval(interval);
+            this.state.updateInterval = null
+            this.state.contentUpdateCompleted = true
+          }
+        });
+      })
+      .catch(error => {
+        console.error('Error:', error);
+      });
+  }
+
+  fetchLoadData() {
+    fetch(`http://${window.location.hostname}:8000/udoit3/api/progress`)
+      .then(response => response.json())
+      .then(data => {
+        this.setState({ 
+          progress: data.progress, 
+          total: data.total, 
+          title: data.title
+        }, () => { 
+          const progress = this.state.progress
+          const total = this.state.total
+          const interval = this.state.scanInterval
+          if(progress === total) {
+            clearInterval(interval);
+            this.state.scanInterval = null
+            this.state.scanCompleted = true
+          }
+        });
+      })
+      .catch(error => {
+        console.error('Error:', error);
+      });
+  }
+
+  getContentUpdate() {
+    const interval = setInterval(() => {
+      this.fetchUpdateData();
+    }, 1000)
+    this.setState({updateInterval: interval})
+  }
+
+  pollBackgroundWorker() {
+    if (this.state.contentUpdateCompleted) {
+      const interval = setInterval(() => {
+        this.fetchLoadData()
+      }, 1000)
+      this.setState({scanInterval: interval})
+    }
+    else {
+      setTimeout(this.pollBackgroundWorker, 500);
+    }
+  }
+
   handleNewReport(data) {
     let report = this.state.report
     let hasNewReport = this.state.hasNewReport

assets/js/Components/CourseUpdateProgress.js

@@ -0,0 +1,41 @@
+import React from 'react'
+import { Spinner } from '@instructure/ui-spinner'
+import { Alert } from '@instructure/ui-alerts'
+
+import Classes from '../../css/app.css'
+
+class CourseUpdateProgress extends React.Component {
+    constructor(props) {
+        super(props)
+        this.checkProgress = this.checkProgress.bind(this)
+        this.state = {
+            completed: false,
+        }
+    }
+
+    render() {
+        return (
+            this.checkProgress() && !this.props.contentUpdateCompleted &&
+            <div>
+                <Alert
+                    variant="info"
+                    margin="small large"
+                >
+                    Updated {this.props.progress} items so far...
+                    <Spinner size="x-small" margin="0 small" renderTitle="Loading" />
+                </Alert>
+            </div>
+        )
+    }
+
+    checkProgress() {
+        if(this.props.title != "Getting content from Canvas") {
+            setTimeout(() => {
+                this.props.contentUpdateCompleted = false
+            }, 2000);
+        }
+        return true
+    }
+}
+
+export default CourseUpdateProgress

assets/js/Services/Api.js

@@ -4,6 +4,7 @@ export default class Api {
         this.apiUrl = `https://${window.location.hostname}`;
         this.endpoints = {
             getReport: '/api/courses/{course}/reports/{report}',
+            getLatestReport: '/api/sync/getLatestReport/{course}',
             getReportHistory: '/api/courses/{course}/reports',
             saveIssue: '/api/issues/{issue}/save',
             resolveIssue: '/api/issues/{issue}/resolve',
@@ -233,6 +234,21 @@ export default class Api {
         })
     }
 
+    getLatestReport(courseId)
+    {
+        const authToken = this.getAuthToken()
+        let url = `${this.apiUrl}${this.endpoints.getLatestReport}`
+        url = url.replace('{course}', courseId)
+
+        return fetch(url, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-AUTH-TOKEN': authToken,
+            },
+        })
+    }
+
     scanContent(contentId)
     {
         const authToken = this.getAuthToken()

build/nginx/entrypoint.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 service nginx start
-php-fpm
+php-fpm

composer.json

@@ -63,7 +63,7 @@
         "symfony/browser-kit": "6.0.*",
         "symfony/css-selector": "6.0.*",
         "symfony/debug-bundle": "6.0.*",
-        "symfony/maker-bundle": "^1.0",
+        "symfony/maker-bundle": "^1.49",
         "symfony/phpunit-bridge": "^6.0",
         "symfony/stopwatch": "6.0.*",
         "symfony/web-profiler-bundle": "6.0.*"

config/packages/messenger.yaml

@@ -25,3 +25,4 @@ framework:
             #'App\Message\BackgroundQueueItem': async_priority_low
             'App\Message\PriorityQueueItem': sync
             'App\Message\BackgroundQueueItem': sync
+            'App\Message\ScanMessage': async_priority_high

docker-compose.nginx.yml

@@ -43,6 +43,7 @@ services:
         - type: bind
           source: ./build/nginx/php-custom.ini
           target: /usr/local/etc/php/conf.d/php-custom.ini
+      image: php
       env_file:
         - .env
   composer:
@@ -66,6 +67,15 @@ services:
         yarn install && 
         yarn build'
 
+  messenger-consume:
+    image: php
+    volumes:
+      - ./:/var/www/html
+    command: >
+      bash -c "sleep 15 && bin/console messenger:consume async_priority_high"
+    depends_on:
+      - db
+
 volumes:
   web:
   dbdata: