KeshavAnandCode/spaceward-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shuold be fixed hopefully

Browse Source
This commit is contained in:
Keshav Anand
2025-12-20 18:54:48 -06:00
parent d5f45dba8f
commit 4571094527
1 changed files with 104 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
index.ts
View File

@@ -1,7 +1,7 @@
import { firefox } from 'playwright';
import type { Browser, BrowserContext, Page } from 'playwright';
import { existsSync, mkdirSync, writeFileSync } from 'fs';
import pino from 'pino';
import { Page } from 'playwright';
import express from 'express';
const app = express();
@@ -19,6 +19,31 @@ const logger = pino({
const SCREENSHOTS_DIR = 'skyward_screenshots';
const TARGET_CATEGORIES = ['NW1', 'NW2', 'SE1', 'NW3', 'NW4', 'SE2'];
// Browser pool to reuse browser instances
let sharedBrowser: Browser | null = null;
const activeSessions = new Set<string>();
async function getSharedBrowser(): Promise<Browser> {
if (!sharedBrowser) {
logger.info('Creating shared browser instance...');
sharedBrowser = await firefox.launch({
headless: true,
firefoxUserPrefs: {
'dom.webdriver.enabled': false,
'media.peerconnection.enabled': false,
'useAutomationExtension': false,
},
});
// Handle browser disconnection
sharedBrowser.on('disconnected', () => {
logger.warn('Browser disconnected, clearing shared instance');
sharedBrowser = null;
});
}
return sharedBrowser;
}
// API endpoint to fetch grades
app.post('/fetch-grades', async (req, res) => {
const { username, password } = req.body;
@@ -30,8 +55,20 @@ app.post('/fetch-grades', async (req, res) => {
});
}
// Check if user already has an active session
if (activeSessions.has(username)) {
return res.status(429).json({
success: false,
error: 'Request already in progress',
message: 'Please wait for your previous request to complete'
});
}
logger.info(`Received request to fetch grades for user: ${username}`);
// Mark session as active
activeSessions.add(username);
try {
const grades = await runSkywardBot(username, password);
@@ -49,12 +86,19 @@ app.post('/fetch-grades', async (req, res) => {
error: errorMsg,
message: 'Failed to fetch grades from Skyward'
});
} finally {
// Remove session from active sessions
activeSessions.delete(username);
}
});
// Health check endpoint
app.get('/health', (req, res) => {
res.json({ status: 'ok', message: 'Skyward API is running' });
res.json({
status: 'ok',
message: 'Skyward API is running',
activeSessions: activeSessions.size
});
});
const PORT = process.env.PORT || 3000;
@@ -74,31 +118,42 @@ app.post('/check-auth', async (req, res) => {
});
}
const browser = await firefox.launch({ headless: true });
const context = await browser.newContext();
const page = await context.newPage();
let context: BrowserContext | null = null;
let page: Page | null = null;
try {
const authenticated = await checkAuth(page, username, password, context, []);
try {
const browser = await getSharedBrowser();
context = await browser.newContext({
userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:117.0) Gecko/20100101 Firefox/117.0',
viewport: { width: 1280, height: 720 },
});
page = await context.newPage();
if (!authenticated) {
const authenticated = await checkAuth(page, username, password, context, []);
if (!authenticated) {
return res.status(401).json({
success: false,
error: 'Authentication failed',
});
}
return res.json({ success: true });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return res.status(401).json({
success: false,
error: 'Authentication failed',
error: msg,
});
} finally {
if (page && !page.isClosed()) {
await page.close().catch(() => {});
}
if (context) {
await context.close().catch(() => {});
}
}
return res.json({ success: true });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return res.status(401).json({
success: false,
error: msg,
});
} finally {
await browser.close();
}
});
const runSkywardBot = async (username: string, password: string) => {
@@ -123,20 +178,13 @@ const runSkywardBot = async (username: string, password: string) => {
mkdirSync(SCREENSHOTS_DIR);
}
const browser = await firefox.launch({
headless: true,
firefoxUserPrefs: {
'dom.webdriver.enabled': false,
'media.peerconnection.enabled': false,
'useAutomationExtension': false,
},
});
const browser = await getSharedBrowser();
let context: BrowserContext | null = null;
let page: Page | null = null;
try {
const context = await browser.newContext({
userAgent:
'Mozilla/5.0 (X11; Linux x86_64; rv:117.0) Gecko/20100101 Firefox/117.0',
context = await browser.newContext({
userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:117.0) Gecko/20100101 Firefox/117.0',
viewport: { width: 1280, height: 720 },
});
@@ -151,17 +199,16 @@ const runSkywardBot = async (username: string, password: string) => {
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
logger.error(`Fatal error in main: ${errorMsg}`);
if (page) {
// try {
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/FATAL_ERROR.png` });
// } catch (screenshotError) {
// logger.warn('Could not take error screenshot');
// }
}
throw error;
} finally {
await browser.close();
logger.info('Browser closed, done.');
// Clean up page and context but NOT the browser
if (page && !page.isClosed()) {
await page.close().catch((err) => logger.warn('Error closing page:', err));
}
if (context) {
await context.close().catch((err) => logger.warn('Error closing context:', err));
}
logger.info('Session cleaned up, browser kept alive.');
}
};
@@ -181,7 +228,6 @@ async function checkAuth(
timeout: 30000,
});
await page.waitForTimeout(2000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/01-launchpad-page.png` });
// Step 2: Click "PISD Network Login" button
logger.info('Step 2: Looking for PISD Network Login button...');
@@ -216,7 +262,6 @@ async function checkAuth(
await pisdButton.click();
logger.info('Clicked PISD Network Login button');
await page.waitForTimeout(2000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/02-after-pisd-button-click.png` });
// Step 3: Enter username
logger.info('Step 3: Entering username...');
@@ -226,7 +271,6 @@ async function checkAuth(
await page.waitForTimeout(500);
await page.type(usernameSelector, username, { delay: 100 });
logger.info('Username entered');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/03-username-entered.png` });
// Step 4: Enter password
logger.info('Step 4: Entering password...');
@@ -236,12 +280,9 @@ async function checkAuth(
await page.waitForTimeout(500);
await page.type(passwordSelector, password, { delay: 100 });
logger.info('Password entered');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/04-password-entered.png` });
// Step 5: Click submit button
logger.info('Step 5: Clicking submit button...');
// First wait a bit for the page to be ready
await page.waitForTimeout(1000);
const submitSelectors = [
@@ -313,18 +354,16 @@ async function checkAuth(
if (!submitButton) {
logger.error('Could not find submit button');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/05-submit-not-found.png`, fullPage: true });
throw new Error('Submit button not found');
}
await submitButton.click();
logger.info('Submit button clicked');
await page.waitForTimeout(3000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/05-after-submit.png` });
// Wait for redirect back to launchpad
logger.info('Waiting for redirect to launchpad...');
await page.waitForTimeout(3000); // Wait a bit for redirect to happen
await page.waitForTimeout(3000);
const currentUrl = page.url();
logger.info(`Current URL after redirect: ${currentUrl}`);
@@ -332,16 +371,12 @@ async function checkAuth(
try {
await page.waitForURL('**/myapps.classlink.com/**', { timeout: 10000 });
logger.info('Successfully matched myapps.classlink.com URL pattern');
return true;
return true;
} catch (err) {
logger.warn(`URL pattern didn't match, but we're at: ${page.url()}`);
logger.warn('Continuing anyway...');
return false;
return false;
}
}
async function loginToSkyward(page: Page, username: string, password: string, context: any, allClassGrades: any[]) {
@@ -354,7 +389,6 @@ async function loginToSkyward(page: Page, username: string, password: string, co
timeout: 30000,
});
await page.waitForTimeout(2000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/01-launchpad-page.png` });
// Step 2: Click "PISD Network Login" button
logger.info('Step 2: Looking for PISD Network Login button...');
@@ -389,7 +423,6 @@ async function loginToSkyward(page: Page, username: string, password: string, co
await pisdButton.click();
logger.info('Clicked PISD Network Login button');
await page.waitForTimeout(2000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/02-after-pisd-button-click.png` });
// Step 3: Enter username
logger.info('Step 3: Entering username...');
@@ -399,7 +432,6 @@ async function loginToSkyward(page: Page, username: string, password: string, co
await page.waitForTimeout(500);
await page.type(usernameSelector, username, { delay: 100 });
logger.info('Username entered');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/03-username-entered.png` });
// Step 4: Enter password
logger.info('Step 4: Entering password...');
@@ -409,12 +441,9 @@ async function loginToSkyward(page: Page, username: string, password: string, co
await page.waitForTimeout(500);
await page.type(passwordSelector, password, { delay: 100 });
logger.info('Password entered');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/04-password-entered.png` });
// Step 5: Click submit button
logger.info('Step 5: Clicking submit button...');
// First wait a bit for the page to be ready
await page.waitForTimeout(1000);
const submitSelectors = [
@@ -453,7 +482,6 @@ async function loginToSkyward(page: Page, username: string, password: string, co
}
}
// If still not found, try getting all buttons and find one with "Sign in" text
if (!submitButton) {
logger.info('Trying to find button by text content...');
const allButtons = await page.$$('button, input[type="submit"]');
@@ -486,18 +514,15 @@ async function loginToSkyward(page: Page, username: string, password: string, co
if (!submitButton) {
logger.error('Could not find submit button');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/05-submit-not-found.png`, fullPage: true });
throw new Error('Submit button not found');
}
await submitButton.click();
logger.info('Submit button clicked');
await page.waitForTimeout(3000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/05-after-submit.png` });
// Wait for redirect back to launchpad
logger.info('Waiting for redirect to launchpad...');
await page.waitForTimeout(3000); // Wait a bit for redirect to happen
await page.waitForTimeout(3000);
const currentUrl = page.url();
logger.info(`Current URL after redirect: ${currentUrl}`);
@@ -511,17 +536,14 @@ async function loginToSkyward(page: Page, username: string, password: string, co
}
await page.waitForTimeout(2000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/06-back-at-launchpad.png` });
await openSkywardFromLaunchpad(page, context, allClassGrades);
}
async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades: any[]) {
try {
// Step 6: Search for Skyward
logger.info('Step 6: Searching for Skyward app...');
// Look for search input
const searchSelectors = [
'input[aria-label="Search All Apps"]',
'input#searchbar-filter-input',
@@ -551,7 +573,6 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
if (!searchInput) {
logger.error('Could not find search input');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/07-search-not-found.png`, fullPage: true });
throw new Error('Search input not found');
}
@@ -560,14 +581,11 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
await searchInput.type('Skyward', { delay: 100 });
logger.info('Typed "Skyward" into search');
await page.waitForTimeout(1000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/07-skyward-search.png` });
// Step 7: Press Enter to select Skyward (opens in new tab)
logger.info('Step 7: Pressing Enter to select Skyward...');
// Set up listener for new tab BEFORE pressing Enter
const newPagePromise = new Promise<Page>((resolve) => {
context.once('page', async (newPage) => {
context.once('page', async (newPage: Page) => {
logger.info(`New tab opened with URL: ${newPage.url()}`);
resolve(newPage);
});
@@ -576,7 +594,6 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
await searchInput.press('Enter');
logger.info('✓ Pressed Enter');
// Wait for new tab to open
logger.info('Waiting for new tab to open...');
const newPage = await Promise.race([
newPagePromise,
@@ -585,21 +602,17 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
if (!newPage) {
logger.error('No new tab opened');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/08-no-new-tab.png` });
throw new Error('New tab did not open');
}
logger.info(`New tab URL: ${newPage.url()}`);
// Wait for the new page to load
await newPage.waitForLoadState('domcontentloaded');
await newPage.waitForTimeout(3000);
const finalUrl = newPage.url();
logger.info(`Final URL in new tab: ${finalUrl}`);
// await newPage.screenshot({ path: `${SCREENSHOTS_DIR}/08-new-tab-skyward.png` });
// Check if we're on Skyward
if (finalUrl.includes('skyward')) {
logger.info('✓ Successfully opened Skyward in new tab');
} else {
@@ -607,7 +620,6 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
logger.info('Continuing anyway...');
}
// Now navigate to grading on the NEW page
await navigateToGrading(newPage, allClassGrades);
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
@@ -616,14 +628,11 @@ async function openSkywardFromLaunchpad(page: Page, context: any, allClassGrades
}
}
async function navigateToGrading(page: Page, allClassGrades: any[]) {
logger.info('Step 8: Navigating to Grading section...');
// Wait for page to fully load
await page.waitForTimeout(3000);
// Look for element with exact text "Grading"
const gradingSelectors = [
'text="Grading"',
'a:has-text("Grading")',
@@ -638,7 +647,6 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
try {
const elements = await page.$$(selector);
// Check each element to find one with EXACT text "Grading"
for (const element of elements) {
const text = (await element.textContent())?.trim();
if (text === 'Grading') {
@@ -659,55 +667,50 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
if (!gradingElement) {
logger.error('Could not find Grading element with exact text');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/10-grading-not-found.png`, fullPage: true });
throw new Error('Grading element not found');
}
// Scroll element into view
await gradingElement.scrollIntoViewIfNeeded();
await page.waitForTimeout(500);
await gradingElement.click();
logger.info('Clicked Grading element');
await page.waitForTimeout(3000);
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/10-grading-page.png`, fullPage: true });
logger.info('Successfully navigated to Grading page');
// Now find the grades table
logger.info('Step 9: Finding grades table...');
// Wait for the table to load
await page.waitForTimeout(2000);
// Find the table body
const tableBody = await page.$('.unlockedBrowseBody.dynamicHeight');
if (!tableBody) {
logger.error('Could not find grades table with class "unlockedBrowseBody dynamicHeight"');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/11-table-not-found.png`, fullPage: true });
throw new Error('Grades table not found');
}
logger.info('✓ Found grades table');
// Find all clickable grade cells (links with numbers)
const gradeCells = await tableBody.$$('a.linkPanelButton');
logger.info(`Found ${gradeCells.length} clickable grade cells`);
if (gradeCells.length === 0) {
logger.warn('No clickable grade cells found');
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/11-no-grade-cells.png`, fullPage: true });
return;
}
// Click each grade cell
for (let i = 0; i < gradeCells.length; i++) {
try {
// Check if page is still open
if (page.isClosed()) {
logger.error('Page was closed during processing');
break;
}
logger.info(`\n=== Processing grade cell ${i + 1}/${gradeCells.length} ===`);
// Re-query the cell each time (in case DOM has changed)
const allCells = await tableBody.$$('a.linkPanelButton');
const cell = allCells[i];
@@ -716,7 +719,6 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
continue;
}
// Get cell details
const gradeText = (await cell.textContent())?.trim();
const isVisible = await cell.isVisible();
const dataRow = await cell.getAttribute('data-row');
@@ -733,17 +735,14 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
continue;
}
// Scroll into view
await cell.scrollIntoViewIfNeeded();
await page.waitForTimeout(500);
// Click the cell
await cell.click();
logger.info(` ✓ Clicked grade cell`);
await page.waitForTimeout(2000);
// Check for the category label - look inside SegmentedAssignmentsInformationCell span (not the one with "left")
const categoryContainer = await page.$('span.SegmentedAssignmentsInformationCell:not(.left) label.labelTag.visualLabel');
if (!categoryContainer) {
@@ -752,11 +751,9 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
const categoryText = (await categoryContainer.textContent())?.trim();
logger.info(` Category label found: "${categoryText}"`);
// Check if this is one of our target categories
if (categoryText && TARGET_CATEGORIES.includes(categoryText)) {
logger.info(` ✓ Target category "${categoryText}" detected - reading grades`);
// Get the class information from the left span
const classInfoSpan = await page.$('span.SegmentedAssignmentsInformationCell.left');
let className = '';
let teacher = '';
@@ -782,7 +779,6 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
}
}
// Find the details panel with grades
const detailsPanel = await page.$('#detailsPanel_SegmentedGradeBucketAssignments_unlockedBody');
if (!detailsPanel) {
@@ -790,22 +786,19 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
} else {
logger.info(` ✓ Found details panel, extracting grades...`);
// Get all rows in the table
const rows = await detailsPanel.$$('tr');
logger.info(` Found ${rows.length} total rows`);
const grades = [];
let currentGradeType = null; // 'major' or 'minor'
let currentGradeType = null;
for (let j = 0; j < rows.length; j++) {
const row = rows[j];
if (!row) continue;
try {
// Check if this row is a grade type header (Major Grades / Minor Grades)
const gradeTypeSpan = await row.$('span.anchorText');
if (gradeTypeSpan) {
const gradeTypeText = (await gradeTypeSpan.textContent())?.trim();
logger.info(` Row ${j}: Found span with text "${gradeTypeText}"`);
@@ -825,15 +818,12 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
}
}
if (!currentGradeType) {
logger.info(` Row ${j}: Skipping (no grade type set yet)`);
currentGradeType = 'minor'; // default
currentGradeType = 'minor';
continue;
}
// Try to extract grade data from this row
// Look for any link in the row (could be linkDetails or other classes)
const nameLink = await row.$('a');
if (!nameLink) {
logger.info(` Row ${j}: No link found, skipping`);
@@ -843,23 +833,20 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
const name = (await nameLink.textContent())?.trim() || '';
logger.info(` Row ${j}: Found assignment "${name}"`);
let dueDate = '';
let dueDate = '';
let score = '';
let pointsEarned = '';
let attempts = '';
// Get all td cells
const tdCells = await row.$$('td');
logger.info(` Row ${j}: Found ${tdCells.length} td cells`);
// Due Date is typically in the second column (index 1)
if (tdCells && tdCells.length > 1 && tdCells[1]) {
dueDate = (await tdCells[1].textContent())?.trim() || '';
}
// Score and Points are in disabled-link-column spans
const disabledSpans = await row.$$('span.disabled-link-column');
logger.info(` Row ${j}: Found ${disabledSpans.length} disabled-link-column spans`);
logger.info(` Row ${j}: Found ${disabledSpans ? disabledSpans.length : 0} disabled-link-column spans`);
if (disabledSpans && disabledSpans.length >= 1 && disabledSpans[0]) {
score = (await disabledSpans[0].textContent())?.trim() || '';
@@ -868,7 +855,6 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
pointsEarned = (await disabledSpans[1].textContent())?.trim() || '';
}
// Attempts - look for any number link in last columns
const allLinks = await row.$$('a');
const lastLink = allLinks && allLinks.length > 1 ? allLinks[allLinks.length - 1] : null;
if (lastLink) {
@@ -894,7 +880,6 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
logger.info(` ✓ Extracted ${grades.length} grades for ${categoryText}`);
// Add to the global collection
allClassGrades.push({
className: className,
teacher: teacher,
@@ -904,21 +889,12 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
});
logger.info(` ✓ Added grades to collection (Total classes: ${allClassGrades.length})`);
// Take screenshot with category-based name
// const screenshotName = `${categoryText}_${className.replace(/\s+/g, '_')}_row${dataRow}.png`.replace(/\s+/g, '_');
// await page.screenshot({
// path: `${SCREENSHOTS_DIR}/${screenshotName}`,
// fullPage: true
// });
// logger.info(` 📸 Screenshot saved: ${screenshotName}`);
}
} else {
logger.info(` ℹ Category "${categoryText}" not in target list, skipping`);
}
}
// Close the modal/panel
const closeSelectors = [
'button[aria-label="Close"]',
'button[aria-label*="close"]',
@@ -949,29 +925,17 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
}
if (!closed) {
// Try pressing Escape key
logger.info(` Trying Escape key to close panel`);
await page.keyboard.press('Escape');
await page.waitForTimeout(1000);
}
// Small delay between cells
await page.waitForTimeout(500);
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
logger.error(` ✗ Error processing grade cell ${i + 1}: ${errorMsg}`);
// try {
// await page.screenshot({
// path: `${SCREENSHOTS_DIR}/grade_${i + 1}_ERROR.png`,
// fullPage: true
// });
// } catch (screenshotErr) {
// logger.warn(` Could not take error screenshot: ${screenshotErr}`);
// }
// Try to close any open modals before continuing
try {
await page.keyboard.press('Escape');
await page.waitForTimeout(1000);
@@ -979,20 +943,12 @@ async function navigateToGrading(page: Page, allClassGrades: any[]) {
logger.warn(` Could not close modal: ${closeErr}`);
}
// Continue to next cell instead of crashing
continue;
}
}
logger.info(`\n🎉 Finished processing all grade cells!`);
// try {
// await page.screenshot({ path: `${SCREENSHOTS_DIR}/12-all-grades-complete.png`, fullPage: true });
// } catch (err) {
// logger.warn(`Could not take final screenshot: ${err}`);
// }
// Write all grades to a JSON file
try {
const gradesFilePath = `${SCREENSHOTS_DIR}/all_grades.json`;
writeFileSync(gradesFilePath, JSON.stringify(allClassGrades, null, 2));