-
Notifications
You must be signed in to change notification settings - Fork 1
/
bitspilani-website.js
114 lines (101 loc) · 3.78 KB
/
bitspilani-website.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
import { launch } from "puppeteer";
import fs from "fs";
async function run() {
const browser = await launch({
headless: false,
defaultViewport: null,
args: ["--start-maximized"],
});
const page = await browser.newPage();
// Find maximum number of pages
let allFaculties = [];
let facultyDetails = [];
let maxPage = 2;
for (let i = 1; i <= maxPage; i++) {
await page.goto(
"https://www.bits-pilani.ac.in/faculty/page/" +
i +
"?&campus=hyderabad&department=humanities-and-social-sciences",
{
waitUntil: "networkidle2",
waitUntil: "domcontentloaded",
}
);
await page.waitForTimeout(4000);
// simulate cursror scroll
await page.evaluate(() => {
window.scrollBy(0, window.innerHeight);
});
const faculty = await page.$$(".faculty-item");
/// Find apply-btn acc_link for each faculty
for (let i = 0; i < faculty.length; i++) {
let facultyItem = faculty[i];
// get a tag which is inside div tag which has class apply-btn
let facultyLink = await facultyItem.$eval(
".apply-btn a",
(node) => node.href
);
allFaculties.push(facultyLink);
}
}
for (let i = 0; i < allFaculties.length; i++) {
await page.goto(allFaculties[i], {
waitUntil: "networkidle2",
waitUntil: "domcontentloaded",
});
await page.waitForTimeout(4000);
// We wait for faculty page to load then we get the details\
let facultyInfo = await page.$$(".facuty-info");
let facultyName = await page.$(".faculty-profile-txt");
let facultyDetailsObject = {
name: "",
email: "",
phone: "",
};
facultyDetailsObject.name = await facultyName.$eval("h2", (node) =>
node.innerText.trim()
);
for (let i = 0; i < facultyInfo.length; i++) {
let faculty = facultyInfo[i];
let facultyDetails = await faculty.evaluate((node) =>
node.innerText.trim()
);
if (facultyDetails.includes("@")) {
facultyDetailsObject.email = facultyDetails.trim();
}
if (facultyDetails.includes("+") || facultyDetails.includes("40")) {
facultyDetailsObject.phone = facultyDetails.trim();
}
}
let secondaryNavbar = await page.$$(".sec_nav_a");
if (secondaryNavbar == null) {
facultyDetailsObject["profileStatus"] = "No Tabs Found";
} else {
facultyDetailsObject["profileStatus"] =
secondaryNavbar.length + " Tabs Found";
}
let overview = await page.$(".ex_space");
if (overview != null) {
let overviewText = await overview.evaluate((node) =>
node.innerText.trim()
);
facultyDetailsObject["overview"] = overviewText;
if (overviewText == "") {
facultyDetailsObject["overviewStatus"] = "No Overview Found";
} else {
facultyDetailsObject["overviewStatus"] = "Overview Found";
}
}
facultyDetails.push(facultyDetailsObject);
}
// Write all this into cscv format
let csv = "Name,Email,Phone,Profile Status,Overview Status\n";
for (let i = 0; i < facultyDetails.length; i++) {
let faculty = facultyDetails[i];
csv += `${faculty.name},${faculty.email},${faculty.phone},${faculty.profileStatus},${faculty.overviewStatus}\n`;
}
// Write to file
fs.writeFileSync("faculty.csv", csv);
console.log("File written successfully");
}
run();