Add processed descriptions to db

client/src/modules/Admin/Components/Admin.tsx

@@ -23,7 +23,7 @@ export const Admin = () => {
   const [doubleClick, setDoubleClick] = useState<boolean>(false)
 
   const [updating, setUpdating] = useState<boolean>(false);
-  type updatedStates = 'empty' | 'semester' | 'profsReset' | 'profsUpdate' | 'subjects' | 'database' | 'description';
+  type updatedStates = 'empty' | 'semester' | 'profsReset' | 'profsUpdate' | 'subjects' | 'database' | 'description' | 'processed';
   const [updated, setUpdated] = useState<updatedStates>('empty');
   const successMessages = {
     'empty': '',
@@ -32,7 +32,8 @@ export const Admin = () => {
     'profsUpdate': "Professor data successfully updated",
     'subjects': "Subject full name data successfully updated",
     'database': "Database successfully initialized",
-    'description': "Course description data successfully added"
+    'description': "Course description data successfully added",
+    'processed': "Processed course descriptino data successfully added"
   };
   const [updatingField, setUpdatingField] = useState<string>("");
 
@@ -284,6 +285,20 @@ export const Admin = () => {
     }
   }
 
+  async function updateProcessedDescriptions() {
+    console.log('Updating processed course descriptions')
+    setUpdating(true)
+    setUpdatingField("processed course descriptions")
+    const response = await axios.post('/api/admin/rec/desc', { token: token });
+    if (response.status === 200) {
+      console.log('Updated all processed course descriptions')
+      setUpdating(false)
+      setUpdated('processed')
+    } else {
+      console.log('Error at updateProcessedDescriptions')
+    }
+  }
+
   /**
    * Handle the first click to the "Initialize Database" button. Show an alert
    * and update state to remember the next click will be a double click.
@@ -388,6 +403,14 @@ export const Admin = () => {
               >
                 Update Subjects
               </button>
+              <button
+                disabled={updating}
+                type="button"
+                className={styles.adminButtons}
+                onClick={() => updateProcessedDescriptions()}
+              >
+                Update Processed Descriptions
+              </button>
               {renderInitButton(doubleClick)}
             </div>
           </div>

server/db/schema.ts

@@ -158,7 +158,7 @@ const RecommendationMetadataSchema = new Schema({
   _id: { type: String },
   classSub: { type: String },
   classNum: { type: String },
-  processedDescriptions: { type: String },
+  processedDescription: { type: String },
   tfidfVector: { type: Map, of: Number },
 });
 

server/scripts/index.ts

@@ -4,6 +4,7 @@ export {
   addNewSemester,
   addAllCourses,
   addAllDescriptions,
+  addAllProcessedDescriptions
 } from './populate-courses';
 
 export { addAllProfessors, resetProfessors } from './populate-professors';

server/scripts/populate-courses.ts

@@ -6,10 +6,11 @@
 import axios from 'axios';
 import shortid from 'shortid';
 import { ScrapingSubject, ScrapingClass } from './types';
-import { Classes, Professors, Subjects } from '../db/schema';
+import { Classes, Professors, Subjects, RecommendationMetadata } from '../db/schema';
 import { extractProfessors } from './populate-professors';
 import { fetchSubjects } from './populate-subjects';
 import { addStudentReview } from '../src/review/review.controller';
+import { preprocess } from '../src/course/course.recalgo';
 
 /**
  * Adds all possible crosslisted classes retrieved from Course API to crosslisted list in Courses database for all semesters.
@@ -568,4 +569,83 @@ export const addCourseDescription = async (course): Promise<boolean> => {
   removeCourse(course);
   console.log(`Error in adding description to course ${subject} ${courseNum}`);
   return false;
-}
+}
+
+export const addAllProcessedDescriptions = async (): Promise<boolean> => {
+  try {
+    const courses = await Classes.find().exec();
+    if (courses) {
+      for (const course of courses) {
+        await addProcessedDescription(course);
+      }
+    }
+    return true;
+  } catch (err) {
+    console.log(`Error in adding processed descriptions: ${err}`);
+  }
+}
+
+const addProcessedDescription = async (course): Promise<boolean> => {
+  const courseId = course._id;
+  const description = course.classDescription;
+  const processed = preprocess(description);
+  const subject = course.classSub;
+  const num = course.classNum;
+  try {
+    console.log(`${subject} ${num}: ${processed}`)
+    const rec = await RecommendationMetadata.findOne({ _id: courseId });
+    if (rec) {
+      await RecommendationMetadata.updateOne(
+        { _id: courseId },
+        { $set: { processedDescription: processed } }
+      );
+    } else {
+      const res = await new RecommendationMetadata({
+        _id: courseId,
+        classSub: subject,
+        classNum: num,
+        processedDescription: processed
+      })
+        .save()
+        .catch((err) => {
+          console.log(err);
+          return null;
+        });
+      if (!res) {
+        throw new Error();
+      }
+    }
+    return true;
+  } catch (err) {
+    console.log(`Error in adding processed description for ${subject} ${num}: ${err}`);
+  }
+}
+
+// export const addAllSimilarityData = async (): Promise<boolean> => {
+//   try {
+//     const courses = await Classes.find().exec();
+//     if (courses) {
+//       for (const course of courses) {
+//         await addSimilarityData(courses, course);
+//       }
+//     }
+//     return true;
+//   } catch (err) {
+//     console.log(`Error in adding similarity data: ${err}`);
+//   }
+// }
+
+// const addSimilarityData = async (courses, course): Promise<boolean> => {
+//   try {
+//     const courseId = course._id;
+//     const description = course.classDescription;
+//     for (const c of courses) {
+//       if (c._id !== courseId) {
+
+//       }
+//     }
+//     return true;
+//   } catch (err) {
+
+//   }
+// }

server/src/admin/admin.controller.ts

@@ -41,6 +41,7 @@ import {
   addCrossList,
   addNewSemester,
   addAllDescriptions,
+  addAllProcessedDescriptions,
 } from '../../scripts';
 import { fetchAddSubjects } from '../../scripts/populate-subjects';
 
@@ -419,3 +420,13 @@ export const addCourseDescriptionsDb = async ({ auth }: VerifyAdminType) => {
   const descriptionResult = await addAllDescriptions();
   return descriptionResult;
 }
+
+export const addProcessedDescriptionsDb = async ({ auth }: VerifyAdminType) => {
+  const userIsAdmin = verifyTokenAdmin({ auth });
+  if (!userIsAdmin) {
+    return null;
+  }
+
+  const descriptionResult = await addAllProcessedDescriptions();
+  return descriptionResult;
+}

server/src/admin/admin.router.ts

@@ -26,7 +26,8 @@ import {
   removeAdmin,
   addAdmin,
   approveReviews,
-  addCourseDescriptionsDb
+  addCourseDescriptionsDb,
+  addProcessedDescriptionsDb
 } from './admin.controller';
 
 export const adminRouter = express.Router();
@@ -508,3 +509,25 @@ adminRouter.post('/db/initialize', async (req, res) => {
     return res.status(500).json({ error: `Internal Server Error: ${err}` });
   }
 });
+
+adminRouter.post('/rec/desc', async (req, res) => {
+  const { token }: AdminRequestType = req.body;
+  try {
+    const auth = new Auth({ token });
+    const result = await addProcessedDescriptionsDb({ auth });
+    console.log(result)
+
+    if (result) {
+      res.status(200);
+      res.set('Connection', 'close');
+      res.json({ message: 'Processed course descriptions added!' });
+      return res;
+    }
+
+    return res
+      .status(400)
+      .json({ error: 'Processed course descriptions were unable to be added!' });
+  } catch (err) {
+    return res.status(500).json({ error: `Internal Server Error: ${err}` });
+  }
+});

server/src/course/course.controller.ts

@@ -1,4 +1,4 @@
-import { findCourseById, findCourseByInfo } from './course.data-access';
+import { findCourseById, findCourseByInfo, findRecommendationByInfo } from './course.data-access';
 import { CourseIdRequestType, CourseInfoRequestType, CourseDescriptionRequestType } from './course.type';
 import { preprocess, tfidf, cosineSimilarity, idf } from './course.recalgo';
 
@@ -81,6 +81,15 @@ export const getReviewsCrossListOR = async ({
   return null;
 };
 
+export const getRecommendationData = async (
+  { number,
+    subject,
+  }: CourseInfoRequestType
+) => {
+  const course = await findRecommendationByInfo(number, subject.toLowerCase());
+  return course;
+}
+
 export const getProcessedDescription = (text) => {
   const processed = preprocess(text);
   return processed;

server/src/course/course.data-access.ts

@@ -1,4 +1,4 @@
-import { Classes } from "../../db/schema";
+import { Classes, RecommendationMetadata } from "../../db/schema";
 
 export const findCourseById = async (courseId: string) => await Classes.findOne({ _id: courseId }).exec();
 
@@ -9,3 +9,11 @@ export const findCourseByInfo = async (
   classSub: courseSubject,
   classNum: courseNumber,
 }).exec();
+
+export const findRecommendationByInfo = async (
+  courseNumber: string,
+  courseSubject: string,
+) => await RecommendationMetadata.findOne({
+  classSub: courseSubject,
+  classNum: courseNumber,
+}).exec();

server/src/course/course.router.ts

@@ -1,7 +1,7 @@
 import express from 'express';
 
 import { CourseIdRequestType, CourseInfoRequestType, CourseDescriptionRequestType } from './course.type';
-import { getCourseByInfo, getReviewsCrossListOR, getProcessedDescription, getSimilarity } from './course.controller';
+import { getCourseByInfo, getReviewsCrossListOR, getRecommendationData, getProcessedDescription, getSimilarity } from './course.controller';
 
 import { getCourseById } from '../utils';
 
@@ -70,6 +70,25 @@ courseRouter.post('/get-reviews', async (req, res) => {
   }
 });
 
+courseRouter.post('/getRecData', async (req, res) => {
+  try {
+    const { number, subject }: CourseInfoRequestType = req.body;
+    const course = await getRecommendationData({ number, subject });
+
+    if (!course) {
+      return res.status(404).json({
+        error: `Recommendation data could not be found for ${subject} ${number}`,
+      });
+    }
+
+    return res.status(200).json({ result: course });
+  } catch (err) {
+    return res
+      .status(500)
+      .json({ error: `Internal Server Error: ${err.message}` });
+  }
+});
+
 /** Reachable at POST /api/courses/getPreDesc
  * @body description: a course description
  * Gets the processed description to use for the similarity algorithm