diff --git a/client/src/modules/Admin/Components/Admin.tsx b/client/src/modules/Admin/Components/Admin.tsx index 7d0dff18..7c4eb6d1 100644 --- a/client/src/modules/Admin/Components/Admin.tsx +++ b/client/src/modules/Admin/Components/Admin.tsx @@ -23,7 +23,7 @@ export const Admin = () => { const [doubleClick, setDoubleClick] = useState(false) const [updating, setUpdating] = useState(false); - type updatedStates = 'empty' | 'semester' | 'profsReset' | 'profsUpdate' | 'subjects' | 'database' | 'description'; + type updatedStates = 'empty' | 'semester' | 'profsReset' | 'profsUpdate' | 'subjects' | 'database' | 'description' | 'processed'; const [updated, setUpdated] = useState('empty'); const successMessages = { 'empty': '', @@ -32,7 +32,8 @@ export const Admin = () => { 'profsUpdate': "Professor data successfully updated", 'subjects': "Subject full name data successfully updated", 'database': "Database successfully initialized", - 'description': "Course description data successfully added" + 'description': "Course description data successfully added", + 'processed': "Processed course descriptino data successfully added" }; const [updatingField, setUpdatingField] = useState(""); @@ -284,6 +285,20 @@ export const Admin = () => { } } + async function updateProcessedDescriptions() { + console.log('Updating processed course descriptions') + setUpdating(true) + setUpdatingField("processed course descriptions") + const response = await axios.post('/api/admin/rec/desc', { token: token }); + if (response.status === 200) { + console.log('Updated all processed course descriptions') + setUpdating(false) + setUpdated('processed') + } else { + console.log('Error at updateProcessedDescriptions') + } + } + /** * Handle the first click to the "Initialize Database" button. Show an alert * and update state to remember the next click will be a double click. @@ -388,6 +403,14 @@ export const Admin = () => { > Update Subjects + {renderInitButton(doubleClick)} diff --git a/server/db/schema.ts b/server/db/schema.ts index 013c4004..12798a93 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -158,7 +158,7 @@ const RecommendationMetadataSchema = new Schema({ _id: { type: String }, classSub: { type: String }, classNum: { type: String }, - processedDescriptions: { type: String }, + processedDescription: { type: String }, tfidfVector: { type: Map, of: Number }, }); diff --git a/server/scripts/index.ts b/server/scripts/index.ts index d5a6e812..5c1e5dd8 100644 --- a/server/scripts/index.ts +++ b/server/scripts/index.ts @@ -4,6 +4,7 @@ export { addNewSemester, addAllCourses, addAllDescriptions, + addAllProcessedDescriptions } from './populate-courses'; export { addAllProfessors, resetProfessors } from './populate-professors'; diff --git a/server/scripts/populate-courses.ts b/server/scripts/populate-courses.ts index a06d6fd5..8a55b1a8 100644 --- a/server/scripts/populate-courses.ts +++ b/server/scripts/populate-courses.ts @@ -6,10 +6,11 @@ import axios from 'axios'; import shortid from 'shortid'; import { ScrapingSubject, ScrapingClass } from './types'; -import { Classes, Professors, Subjects } from '../db/schema'; +import { Classes, Professors, Subjects, RecommendationMetadata } from '../db/schema'; import { extractProfessors } from './populate-professors'; import { fetchSubjects } from './populate-subjects'; import { addStudentReview } from '../src/review/review.controller'; +import { preprocess } from '../src/course/course.recalgo'; /** * Adds all possible crosslisted classes retrieved from Course API to crosslisted list in Courses database for all semesters. @@ -568,4 +569,83 @@ export const addCourseDescription = async (course): Promise => { removeCourse(course); console.log(`Error in adding description to course ${subject} ${courseNum}`); return false; -} \ No newline at end of file +} + +export const addAllProcessedDescriptions = async (): Promise => { + try { + const courses = await Classes.find().exec(); + if (courses) { + for (const course of courses) { + await addProcessedDescription(course); + } + } + return true; + } catch (err) { + console.log(`Error in adding processed descriptions: ${err}`); + } +} + +const addProcessedDescription = async (course): Promise => { + const courseId = course._id; + const description = course.classDescription; + const processed = preprocess(description); + const subject = course.classSub; + const num = course.classNum; + try { + console.log(`${subject} ${num}: ${processed}`) + const rec = await RecommendationMetadata.findOne({ _id: courseId }); + if (rec) { + await RecommendationMetadata.updateOne( + { _id: courseId }, + { $set: { processedDescription: processed } } + ); + } else { + const res = await new RecommendationMetadata({ + _id: courseId, + classSub: subject, + classNum: num, + processedDescription: processed + }) + .save() + .catch((err) => { + console.log(err); + return null; + }); + if (!res) { + throw new Error(); + } + } + return true; + } catch (err) { + console.log(`Error in adding processed description for ${subject} ${num}: ${err}`); + } +} + +// export const addAllSimilarityData = async (): Promise => { +// try { +// const courses = await Classes.find().exec(); +// if (courses) { +// for (const course of courses) { +// await addSimilarityData(courses, course); +// } +// } +// return true; +// } catch (err) { +// console.log(`Error in adding similarity data: ${err}`); +// } +// } + +// const addSimilarityData = async (courses, course): Promise => { +// try { +// const courseId = course._id; +// const description = course.classDescription; +// for (const c of courses) { +// if (c._id !== courseId) { + +// } +// } +// return true; +// } catch (err) { + +// } +// } \ No newline at end of file diff --git a/server/src/admin/admin.controller.ts b/server/src/admin/admin.controller.ts index d716470b..86c1c09a 100644 --- a/server/src/admin/admin.controller.ts +++ b/server/src/admin/admin.controller.ts @@ -41,6 +41,7 @@ import { addCrossList, addNewSemester, addAllDescriptions, + addAllProcessedDescriptions, } from '../../scripts'; import { fetchAddSubjects } from '../../scripts/populate-subjects'; @@ -419,3 +420,13 @@ export const addCourseDescriptionsDb = async ({ auth }: VerifyAdminType) => { const descriptionResult = await addAllDescriptions(); return descriptionResult; } + +export const addProcessedDescriptionsDb = async ({ auth }: VerifyAdminType) => { + const userIsAdmin = verifyTokenAdmin({ auth }); + if (!userIsAdmin) { + return null; + } + + const descriptionResult = await addAllProcessedDescriptions(); + return descriptionResult; +} \ No newline at end of file diff --git a/server/src/admin/admin.router.ts b/server/src/admin/admin.router.ts index 54c97acf..0dea19c3 100644 --- a/server/src/admin/admin.router.ts +++ b/server/src/admin/admin.router.ts @@ -26,7 +26,8 @@ import { removeAdmin, addAdmin, approveReviews, - addCourseDescriptionsDb + addCourseDescriptionsDb, + addProcessedDescriptionsDb } from './admin.controller'; export const adminRouter = express.Router(); @@ -508,3 +509,25 @@ adminRouter.post('/db/initialize', async (req, res) => { return res.status(500).json({ error: `Internal Server Error: ${err}` }); } }); + +adminRouter.post('/rec/desc', async (req, res) => { + const { token }: AdminRequestType = req.body; + try { + const auth = new Auth({ token }); + const result = await addProcessedDescriptionsDb({ auth }); + console.log(result) + + if (result) { + res.status(200); + res.set('Connection', 'close'); + res.json({ message: 'Processed course descriptions added!' }); + return res; + } + + return res + .status(400) + .json({ error: 'Processed course descriptions were unable to be added!' }); + } catch (err) { + return res.status(500).json({ error: `Internal Server Error: ${err}` }); + } +}); diff --git a/server/src/course/course.controller.ts b/server/src/course/course.controller.ts index 4e2de3f7..f4a09623 100644 --- a/server/src/course/course.controller.ts +++ b/server/src/course/course.controller.ts @@ -1,4 +1,4 @@ -import { findCourseById, findCourseByInfo } from './course.data-access'; +import { findCourseById, findCourseByInfo, findRecommendationByInfo } from './course.data-access'; import { CourseIdRequestType, CourseInfoRequestType, CourseDescriptionRequestType } from './course.type'; import { preprocess, tfidf, cosineSimilarity, idf } from './course.recalgo'; @@ -81,6 +81,15 @@ export const getReviewsCrossListOR = async ({ return null; }; +export const getRecommendationData = async ( + { number, + subject, + }: CourseInfoRequestType +) => { + const course = await findRecommendationByInfo(number, subject.toLowerCase()); + return course; +} + export const getProcessedDescription = (text) => { const processed = preprocess(text); return processed; diff --git a/server/src/course/course.data-access.ts b/server/src/course/course.data-access.ts index 25743789..a62b4516 100644 --- a/server/src/course/course.data-access.ts +++ b/server/src/course/course.data-access.ts @@ -1,4 +1,4 @@ -import { Classes } from "../../db/schema"; +import { Classes, RecommendationMetadata } from "../../db/schema"; export const findCourseById = async (courseId: string) => await Classes.findOne({ _id: courseId }).exec(); @@ -9,3 +9,11 @@ export const findCourseByInfo = async ( classSub: courseSubject, classNum: courseNumber, }).exec(); + +export const findRecommendationByInfo = async ( + courseNumber: string, + courseSubject: string, +) => await RecommendationMetadata.findOne({ + classSub: courseSubject, + classNum: courseNumber, +}).exec(); \ No newline at end of file diff --git a/server/src/course/course.router.ts b/server/src/course/course.router.ts index 98c64813..494ae9c6 100644 --- a/server/src/course/course.router.ts +++ b/server/src/course/course.router.ts @@ -1,7 +1,7 @@ import express from 'express'; import { CourseIdRequestType, CourseInfoRequestType, CourseDescriptionRequestType } from './course.type'; -import { getCourseByInfo, getReviewsCrossListOR, getProcessedDescription, getSimilarity } from './course.controller'; +import { getCourseByInfo, getReviewsCrossListOR, getRecommendationData, getProcessedDescription, getSimilarity } from './course.controller'; import { getCourseById } from '../utils'; @@ -70,6 +70,25 @@ courseRouter.post('/get-reviews', async (req, res) => { } }); +courseRouter.post('/getRecData', async (req, res) => { + try { + const { number, subject }: CourseInfoRequestType = req.body; + const course = await getRecommendationData({ number, subject }); + + if (!course) { + return res.status(404).json({ + error: `Recommendation data could not be found for ${subject} ${number}`, + }); + } + + return res.status(200).json({ result: course }); + } catch (err) { + return res + .status(500) + .json({ error: `Internal Server Error: ${err.message}` }); + } +}); + /** Reachable at POST /api/courses/getPreDesc * @body description: a course description * Gets the processed description to use for the similarity algorithm