1019 lines
38 KiB
JavaScript
1019 lines
38 KiB
JavaScript
/**
|
||
* Websurfer Web Dashboard - Reactive Client Controller
|
||
*/
|
||
|
||
class WebsurferDashboard {
|
||
constructor() {
|
||
this.ws = null;
|
||
this.wsReconnectInterval = 2000;
|
||
this.status = 'idle';
|
||
this.elapsedSeconds = 0;
|
||
this.timerInterval = null;
|
||
this.errors = [];
|
||
this.logs = [];
|
||
|
||
// DOM Elements Cache
|
||
this.elements = {
|
||
// Status & Header
|
||
wsStatus: document.getElementById('wsStatus'),
|
||
crawlStatusBadge: document.getElementById('crawlStatusBadge'),
|
||
elapsedTimer: document.getElementById('elapsedTimer'),
|
||
progressMessage: document.getElementById('progressMessage'),
|
||
iterationBadge: document.getElementById('iterationBadge'),
|
||
progressBar: document.getElementById('progressBar'),
|
||
|
||
// Form Controls
|
||
configForm: document.getElementById('configForm'),
|
||
startUrl: document.getElementById('startUrl'),
|
||
numTabs: document.getElementById('numTabs'),
|
||
numTabsVal: document.getElementById('numTabsVal'),
|
||
numIterations: document.getElementById('numIterations'),
|
||
iterModeAll: document.getElementById('iterModeAll'),
|
||
iterModeVisits: document.getElementById('iterModeVisits'),
|
||
labelIterModeAll: document.getElementById('labelIterModeAll'),
|
||
labelIterModeVisits: document.getElementById('labelIterModeVisits'),
|
||
iterVisitsField: document.getElementById('iterVisitsField'),
|
||
maxVisitsPerIteration: document.getElementById('maxVisitsPerIteration'),
|
||
randomizeQueue: document.getElementById('randomizeQueue'),
|
||
headless: document.getElementById('headless'),
|
||
acceptCookies: document.getElementById('acceptCookies'),
|
||
useTor: document.getElementById('useTor'),
|
||
torOptionsGroup: document.getElementById('torOptionsGroup'),
|
||
torModeIteration: document.getElementById('torModeIteration'),
|
||
torModePages: document.getElementById('torModePages'),
|
||
labelTorModeIteration: document.getElementById('labelTorModeIteration'),
|
||
labelTorModePages: document.getElementById('labelTorModePages'),
|
||
torPagesField: document.getElementById('torPagesField'),
|
||
torCircuitPages: document.getElementById('torCircuitPages'),
|
||
torMaxConsecutiveTimeouts: document.getElementById('torMaxConsecutiveTimeouts'),
|
||
timeoutMs: document.getElementById('timeoutMs'),
|
||
slowMo: document.getElementById('slowMo'),
|
||
torSocksPort: document.getElementById('torSocksPort'),
|
||
torControlPort: document.getElementById('torControlPort'),
|
||
userAgentSelect: document.getElementById('userAgentSelect'),
|
||
userAgentCustom: document.getElementById('userAgentCustom'),
|
||
|
||
// Buttons
|
||
btnStart: document.getElementById('btnStart'),
|
||
btnStop: document.getElementById('btnStop'),
|
||
btnSaveEnv: document.getElementById('btnSaveEnv'),
|
||
btnReset: document.getElementById('btnReset'),
|
||
|
||
// European Exit Node Selector
|
||
countryChipsGrid: document.getElementById('countryChipsGrid'),
|
||
exitNodesCountBadge: document.getElementById('exitNodesCountBadge'),
|
||
exitNodesSummaryText: document.getElementById('exitNodesSummaryText'),
|
||
btnExitMajor: document.getElementById('btnExitMajor'),
|
||
btnExitAll: document.getElementById('btnExitAll'),
|
||
btnExitClear: document.getElementById('btnExitClear'),
|
||
|
||
// Metric Tiles
|
||
statVisited: document.getElementById('statVisited'),
|
||
statInternal: document.getElementById('statInternal'),
|
||
statExternal: document.getElementById('statExternal'),
|
||
statCookies: document.getElementById('statCookies'),
|
||
statQueue: document.getElementById('statQueue'),
|
||
statErrors: document.getElementById('statErrors'),
|
||
statTotalVisitedAll: document.getElementById('statTotalVisitedAll'),
|
||
statTotalErrorsAll: document.getElementById('statTotalErrorsAll'),
|
||
|
||
// Tor Identity Panel
|
||
torCard: document.getElementById('torCard'),
|
||
torStatusBadge: document.getElementById('torStatusBadge'),
|
||
torCircuitHint: document.getElementById('torCircuitHint'),
|
||
torIpVal: document.getElementById('torIpVal'),
|
||
torCountryVal: document.getElementById('torCountryVal'),
|
||
torCityVal: document.getElementById('torCityVal'),
|
||
torOrgVal: document.getElementById('torOrgVal'),
|
||
torStrategyVal: document.getElementById('torStrategyVal'),
|
||
torRotationsVal: document.getElementById('torRotationsVal'),
|
||
torFilterVal: document.getElementById('torFilterVal'),
|
||
|
||
// Error Table
|
||
errorsTableBody: document.getElementById('errorsTableBody'),
|
||
errorCountBadge: document.getElementById('errorCountBadge'),
|
||
errorSearchInput: document.getElementById('errorSearchInput'),
|
||
btnExportCsv: document.getElementById('btnExportCsv'),
|
||
btnExportJson: document.getElementById('btnExportJson'),
|
||
|
||
// Log Terminal
|
||
logOutput: document.getElementById('logOutput'),
|
||
terminalContainer: document.getElementById('terminalContainer'),
|
||
logCountBadge: document.getElementById('logCountBadge'),
|
||
autoScrollCheck: document.getElementById('autoScrollCheck'),
|
||
logLevelSelect: document.getElementById('logLevelSelect'),
|
||
logSearchInput: document.getElementById('logSearchInput'),
|
||
btnClearLogs: document.getElementById('btnClearLogs'),
|
||
btnCopyLogs: document.getElementById('btnCopyLogs'),
|
||
|
||
// Toasts
|
||
toastContainer: document.getElementById('toastContainer'),
|
||
};
|
||
|
||
this.selectedExitNodes = new Set();
|
||
this.availableCountries = [];
|
||
|
||
this.init();
|
||
}
|
||
|
||
async init() {
|
||
await this.checkAuth();
|
||
this.bindEvents();
|
||
await this.loadCountries();
|
||
await this.loadConfig();
|
||
this.connectWebSocket();
|
||
}
|
||
|
||
async checkAuth() {
|
||
try {
|
||
const res = await fetch('/api/auth/me');
|
||
if (res.status === 401) {
|
||
window.location.href = '/login';
|
||
return;
|
||
}
|
||
const data = await res.json();
|
||
if (!data.authenticated && data.auth_enabled) {
|
||
window.location.href = '/login';
|
||
return;
|
||
}
|
||
if (data.auth_enabled && data.username) {
|
||
const profile = document.getElementById('userProfile');
|
||
const nameSpan = document.getElementById('userName');
|
||
const btnLogout = document.getElementById('btnLogout');
|
||
if (profile && nameSpan) {
|
||
nameSpan.textContent = data.username;
|
||
profile.style.display = 'flex';
|
||
}
|
||
if (btnLogout) {
|
||
btnLogout.addEventListener('click', async () => {
|
||
await fetch('/api/logout', { method: 'POST' });
|
||
window.location.href = '/login';
|
||
});
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.warn('Auth check warning:', e);
|
||
}
|
||
}
|
||
|
||
bindEvents() {
|
||
// Sync slider value badge
|
||
this.elements.numTabs.addEventListener('input', (e) => {
|
||
this.elements.numTabsVal.textContent = e.target.value;
|
||
});
|
||
|
||
// Form submission (Start Crawling)
|
||
this.elements.configForm.addEventListener('submit', (e) => {
|
||
e.preventDefault();
|
||
this.startCrawl();
|
||
});
|
||
|
||
// Stop button
|
||
this.elements.btnStop.addEventListener('click', () => {
|
||
this.stopCrawl();
|
||
});
|
||
|
||
// Save to .env button
|
||
this.elements.btnSaveEnv.addEventListener('click', () => {
|
||
this.saveConfigToEnv();
|
||
});
|
||
|
||
// Reset button
|
||
this.elements.btnReset.addEventListener('click', () => {
|
||
this.loadConfig();
|
||
this.showToast('Form reset to saved .env settings', 'info');
|
||
});
|
||
|
||
// Use Tor toggle change
|
||
this.elements.useTor.addEventListener('change', () => {
|
||
this.updateTorVisibility();
|
||
if (!this.elements.useTor.checked) {
|
||
this.updateTorIdentity(null);
|
||
} else {
|
||
this.elements.torStatusBadge.className = 'tor-pill tor-on';
|
||
this.elements.torStatusBadge.textContent = 'Tor Enabled (Pending Run)';
|
||
}
|
||
});
|
||
|
||
// Tor circuit mode radio changes
|
||
const onTorModeChange = () => {
|
||
this.updateTorModeUI();
|
||
};
|
||
if (this.elements.torModeIteration) {
|
||
this.elements.torModeIteration.addEventListener('change', onTorModeChange);
|
||
}
|
||
if (this.elements.torModePages) {
|
||
this.elements.torModePages.addEventListener('change', onTorModeChange);
|
||
}
|
||
if (this.elements.torCircuitPages) {
|
||
this.elements.torCircuitPages.addEventListener('input', () => {
|
||
this.updateTorModeUI();
|
||
});
|
||
}
|
||
|
||
// Iteration mode radio changes
|
||
const onIterModeChange = () => {
|
||
this.updateIterationModeUI();
|
||
};
|
||
if (this.elements.iterModeAll) {
|
||
this.elements.iterModeAll.addEventListener('change', onIterModeChange);
|
||
}
|
||
if (this.elements.iterModeVisits) {
|
||
this.elements.iterModeVisits.addEventListener('change', onIterModeChange);
|
||
}
|
||
|
||
// User-Agent Selector Toggle
|
||
if (this.elements.userAgentSelect) {
|
||
this.elements.userAgentSelect.addEventListener('change', (e) => {
|
||
const val = e.target.value;
|
||
if (val === 'custom') {
|
||
this.elements.userAgentCustom.style.display = 'block';
|
||
this.elements.userAgentCustom.focus();
|
||
} else {
|
||
this.elements.userAgentCustom.style.display = 'none';
|
||
this.elements.userAgentCustom.value = val;
|
||
}
|
||
});
|
||
}
|
||
|
||
// Exit Node Action Presets
|
||
if (this.elements.btnExitMajor) {
|
||
this.elements.btnExitMajor.addEventListener('click', () => {
|
||
this.selectedExitNodes = new Set(['de', 'nl', 'fr', 'ch']);
|
||
this.updateExitNodesUI();
|
||
});
|
||
}
|
||
if (this.elements.btnExitAll) {
|
||
this.elements.btnExitAll.addEventListener('click', () => {
|
||
this.selectedExitNodes = new Set(this.availableCountries.map(c => c.code));
|
||
this.updateExitNodesUI();
|
||
});
|
||
}
|
||
if (this.elements.btnExitClear) {
|
||
this.elements.btnExitClear.addEventListener('click', () => {
|
||
this.selectedExitNodes.clear();
|
||
this.updateExitNodesUI();
|
||
});
|
||
}
|
||
|
||
// Error table search/filter
|
||
this.elements.errorSearchInput.addEventListener('input', () => {
|
||
this.renderErrorsTable();
|
||
});
|
||
|
||
// Error export buttons
|
||
this.elements.btnExportCsv.addEventListener('click', () => {
|
||
window.location.href = '/api/crawl/export-errors?format=csv';
|
||
});
|
||
|
||
this.elements.btnExportJson.addEventListener('click', () => {
|
||
window.open('/api/crawl/export-errors?format=json', '_blank');
|
||
});
|
||
|
||
// Log terminal controls
|
||
this.elements.logLevelSelect.addEventListener('change', () => this.filterLogs());
|
||
this.elements.logSearchInput.addEventListener('input', () => this.filterLogs());
|
||
|
||
this.elements.btnClearLogs.addEventListener('click', async () => {
|
||
this.logs = [];
|
||
this.elements.logOutput.innerHTML = '';
|
||
this.updateLogCount();
|
||
try {
|
||
await fetch('/api/crawl/clear-logs', { method: 'POST' });
|
||
} catch (err) {
|
||
console.error('Failed to clear server logs:', err);
|
||
}
|
||
});
|
||
|
||
this.elements.btnCopyLogs.addEventListener('click', () => {
|
||
const allText = this.logs.map(l => `[${this.formatTime(l.timestamp)}] [${l.level.toUpperCase()}] ${l.message}`).join('\n');
|
||
navigator.clipboard.writeText(allText).then(() => {
|
||
this.showToast('All logs copied to clipboard!', 'success');
|
||
});
|
||
});
|
||
}
|
||
|
||
// --- WebSocket Connection ---
|
||
connectWebSocket() {
|
||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||
const wsUrl = `${protocol}//${window.location.host}/ws/stream`;
|
||
|
||
this.updateWsStatus('connecting');
|
||
|
||
try {
|
||
this.ws = new WebSocket(wsUrl);
|
||
|
||
this.ws.onopen = () => {
|
||
this.updateWsStatus('connected');
|
||
};
|
||
|
||
this.ws.onmessage = (event) => {
|
||
try {
|
||
const payload = JSON.parse(event.data);
|
||
this.handleEvent(payload);
|
||
} catch (err) {
|
||
console.error('Error parsing WebSocket message:', err);
|
||
}
|
||
};
|
||
|
||
this.ws.onclose = () => {
|
||
this.updateWsStatus('disconnected');
|
||
setTimeout(() => this.connectWebSocket(), this.wsReconnectInterval);
|
||
};
|
||
|
||
this.ws.onerror = (error) => {
|
||
console.error('WebSocket Error:', error);
|
||
this.ws.close();
|
||
};
|
||
} catch (e) {
|
||
this.updateWsStatus('disconnected');
|
||
setTimeout(() => this.connectWebSocket(), this.wsReconnectInterval);
|
||
}
|
||
}
|
||
|
||
updateWsStatus(status) {
|
||
const pill = this.elements.wsStatus;
|
||
pill.className = `ws-pill ${status}`;
|
||
const textSpan = pill.querySelector('.ws-text');
|
||
if (status === 'connected') {
|
||
textSpan.textContent = 'Live Connected';
|
||
} else if (status === 'connecting') {
|
||
textSpan.textContent = 'Connecting...';
|
||
} else {
|
||
textSpan.textContent = 'Disconnected';
|
||
}
|
||
}
|
||
|
||
// --- Event Handling ---
|
||
handleEvent(event) {
|
||
const { type, data } = event;
|
||
|
||
if (type === 'init') {
|
||
this.hydrateSnapshot(data);
|
||
} else if (type === 'status') {
|
||
this.updateStatus(data.status, data.message, data.elapsed);
|
||
} else if (type === 'stats') {
|
||
this.updateStats(data);
|
||
} else if (type === 'tor_identity') {
|
||
this.updateTorIdentity(data);
|
||
} else if (type === 'error') {
|
||
this.addError(data);
|
||
} else if (type === 'log') {
|
||
this.addLog(data);
|
||
} else if (type === 'iteration') {
|
||
const iterText = `Iteration ${data.current} / ${data.total}`;
|
||
const modeText = data.iteration_mode === 'visits' && data.max_visits_per_iteration ? ` (${data.max_visits_per_iteration} visits/iter)` : '';
|
||
this.elements.iterationBadge.textContent = `${iterText}${modeText}`;
|
||
}
|
||
}
|
||
|
||
hydrateSnapshot(data) {
|
||
this.updateStatus(data.status, data.status_message, data.elapsed_seconds);
|
||
this.updateStats({
|
||
stats: data.stats,
|
||
queue_size: data.queue_size,
|
||
visited_count: data.visited_count,
|
||
error_count: data.error_count,
|
||
total_visited_all: data.total_visited_all !== undefined ? data.total_visited_all : (data.visited_count || 0),
|
||
total_errors_all: data.total_errors_all !== undefined ? data.total_errors_all : (data.error_count || (data.errors ? data.errors.length : 0)),
|
||
});
|
||
const iterText = `Iteration ${data.current_iteration || 1} / ${data.total_iterations || 1}`;
|
||
const modeText = data.iteration_mode === 'visits' && data.max_visits_per_iteration ? ` (${data.max_visits_per_iteration} visits/iter)` : '';
|
||
this.elements.iterationBadge.textContent = `${iterText}${modeText}`;
|
||
|
||
if (data.tor_identity) {
|
||
this.updateTorIdentity(data.tor_identity);
|
||
} else {
|
||
this.updateTorIdentity(null);
|
||
}
|
||
|
||
if (data.errors) {
|
||
this.errors = data.errors;
|
||
this.renderErrorsTable();
|
||
}
|
||
|
||
if (data.logs) {
|
||
this.logs = data.logs;
|
||
this.elements.logOutput.innerHTML = '';
|
||
this.logs.forEach(log => this.renderLogLine(log));
|
||
this.updateLogCount();
|
||
}
|
||
}
|
||
|
||
updateTorVisibility() {
|
||
const isTor = this.elements.useTor.checked;
|
||
if (this.elements.torOptionsGroup) {
|
||
this.elements.torOptionsGroup.style.display = isTor ? 'block' : 'none';
|
||
}
|
||
this.updateTorModeUI();
|
||
this.updateIterationModeUI();
|
||
this.updateExitNodesUI();
|
||
}
|
||
|
||
updateIterationModeUI() {
|
||
const isVisits = this.elements.iterModeVisits && this.elements.iterModeVisits.checked;
|
||
if (this.elements.labelIterModeAll) {
|
||
this.elements.labelIterModeAll.classList.toggle('active', !isVisits);
|
||
}
|
||
if (this.elements.labelIterModeVisits) {
|
||
this.elements.labelIterModeVisits.classList.toggle('active', isVisits);
|
||
}
|
||
if (this.elements.iterVisitsField) {
|
||
this.elements.iterVisitsField.style.display = isVisits ? 'block' : 'none';
|
||
}
|
||
}
|
||
|
||
updateTorModeUI() {
|
||
const isPages = this.elements.torModePages && this.elements.torModePages.checked;
|
||
if (this.elements.labelTorModeIteration) {
|
||
this.elements.labelTorModeIteration.classList.toggle('active', !isPages);
|
||
}
|
||
if (this.elements.labelTorModePages) {
|
||
this.elements.labelTorModePages.classList.toggle('active', isPages);
|
||
}
|
||
if (this.elements.torPagesField) {
|
||
this.elements.torPagesField.style.display = isPages ? 'block' : 'none';
|
||
}
|
||
if (this.elements.torStrategyVal) {
|
||
if (this.elements.useTor.checked) {
|
||
const pages = this.elements.torCircuitPages ? this.elements.torCircuitPages.value || 10 : 10;
|
||
this.elements.torStrategyVal.textContent = isPages ? `Every ${pages} Pages` : 'Per Iteration';
|
||
} else {
|
||
this.elements.torStrategyVal.textContent = 'Disabled';
|
||
}
|
||
}
|
||
}
|
||
|
||
async loadCountries() {
|
||
try {
|
||
const res = await fetch('/api/tor/countries');
|
||
const data = await res.json();
|
||
if (data.success && data.countries && data.countries.length > 0) {
|
||
this.availableCountries = data.countries;
|
||
} else {
|
||
this.availableCountries = this.getDefaultCountries();
|
||
}
|
||
} catch (e) {
|
||
this.availableCountries = this.getDefaultCountries();
|
||
}
|
||
this.renderCountryChips();
|
||
}
|
||
|
||
getDefaultCountries() {
|
||
return [
|
||
{ code: 'de', name: 'Germany', flag: '🇩🇪' },
|
||
{ code: 'nl', name: 'Netherlands', flag: '🇳🇱' },
|
||
{ code: 'fr', name: 'France', flag: '🇫🇷' },
|
||
{ code: 'ch', name: 'Switzerland', flag: '🇨🇭' },
|
||
{ code: 'at', name: 'Austria', flag: '🇦🇹' },
|
||
{ code: 'se', name: 'Sweden', flag: '🇸🇪' },
|
||
{ code: 'no', name: 'Norway', flag: '🇳🇴' },
|
||
{ code: 'fi', name: 'Finland', flag: '🇫🇮' },
|
||
{ code: 'dk', name: 'Denmark', flag: '🇩🇰' },
|
||
{ code: 'gb', name: 'United Kingdom', flag: '🇬🇧' },
|
||
{ code: 'pl', name: 'Poland', flag: '🇵🇱' },
|
||
{ code: 'it', name: 'Italy', flag: '🇮🇹' },
|
||
{ code: 'es', name: 'Spain', flag: '🇪🇸' },
|
||
{ code: 'be', name: 'Belgium', flag: '🇧🇪' },
|
||
{ code: 'cz', name: 'Czech Republic', flag: '🇨🇿' },
|
||
{ code: 'ro', name: 'Romania', flag: '🇷🇴' },
|
||
{ code: 'ie', name: 'Ireland', flag: '🇮🇪' },
|
||
{ code: 'pt', name: 'Portugal', flag: '🇵🇹' },
|
||
{ code: 'hu', name: 'Hungary', flag: '🇭🇺' },
|
||
{ code: 'is', name: 'Iceland', flag: '🇮🇸' },
|
||
{ code: 'lu', name: 'Luxembourg', flag: '🇱🇺' },
|
||
];
|
||
}
|
||
|
||
renderCountryChips() {
|
||
const grid = this.elements.countryChipsGrid;
|
||
if (!grid) return;
|
||
grid.innerHTML = '';
|
||
|
||
this.availableCountries.forEach(c => {
|
||
const isSelected = this.selectedExitNodes.has(c.code.toLowerCase());
|
||
const btn = document.createElement('button');
|
||
btn.type = 'button';
|
||
btn.className = `country-chip ${isSelected ? 'active' : ''}`;
|
||
btn.dataset.code = c.code.toLowerCase();
|
||
btn.innerHTML = `
|
||
<span class="chip-flag">${c.flag}</span>
|
||
<span class="chip-name">${c.name}</span>
|
||
<span class="chip-code">${c.code.toUpperCase()}</span>
|
||
`;
|
||
|
||
btn.addEventListener('click', () => {
|
||
const code = c.code.toLowerCase();
|
||
if (this.selectedExitNodes.has(code)) {
|
||
this.selectedExitNodes.delete(code);
|
||
} else {
|
||
this.selectedExitNodes.add(code);
|
||
}
|
||
this.updateExitNodesUI();
|
||
});
|
||
|
||
grid.appendChild(btn);
|
||
});
|
||
|
||
this.updateExitNodesUI();
|
||
}
|
||
|
||
updateExitNodesUI() {
|
||
const count = this.selectedExitNodes.size;
|
||
const chips = this.elements.countryChipsGrid ? this.elements.countryChipsGrid.querySelectorAll('.country-chip') : [];
|
||
|
||
chips.forEach(chip => {
|
||
const code = chip.dataset.code;
|
||
chip.classList.toggle('active', this.selectedExitNodes.has(code));
|
||
});
|
||
|
||
if (this.elements.exitNodesCountBadge) {
|
||
if (count === 0) {
|
||
this.elements.exitNodesCountBadge.textContent = 'Any / Global';
|
||
this.elements.exitNodesCountBadge.className = 'count-pill-sm';
|
||
} else {
|
||
this.elements.exitNodesCountBadge.textContent = `${count} Selected`;
|
||
this.elements.exitNodesCountBadge.className = 'count-pill-sm active';
|
||
}
|
||
}
|
||
|
||
if (this.elements.exitNodesSummaryText) {
|
||
if (count === 0) {
|
||
this.elements.exitNodesSummaryText.textContent = 'Active Exit Filter: Worldwide / Any Tor exit node (Global)';
|
||
} else {
|
||
const countryObjs = Array.from(this.selectedExitNodes).map(code => {
|
||
const found = this.availableCountries.find(c => c.code.toLowerCase() === code);
|
||
return found ? `${found.flag} ${found.name}` : code.toUpperCase();
|
||
});
|
||
if (countryObjs.length <= 3) {
|
||
this.elements.exitNodesSummaryText.textContent = `Active Exit Filter: ${countryObjs.join(', ')}`;
|
||
} else {
|
||
this.elements.exitNodesSummaryText.textContent = `Active Exit Filter: ${countryObjs.slice(0, 3).join(', ')} + ${countryObjs.length - 3} more`;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (this.elements.torFilterVal) {
|
||
if (!this.elements.useTor.checked) {
|
||
this.elements.torFilterVal.textContent = 'Disabled';
|
||
} else if (count === 0) {
|
||
this.elements.torFilterVal.textContent = 'Global (Any)';
|
||
} else {
|
||
const displayCodes = Array.from(this.selectedExitNodes).map(c => c.toUpperCase());
|
||
this.elements.torFilterVal.textContent = displayCodes.length <= 4 ? displayCodes.join(', ') : `${displayCodes.slice(0, 4).join(', ')} (+${displayCodes.length - 4})`;
|
||
}
|
||
}
|
||
}
|
||
|
||
updateTorIdentity(torData) {
|
||
const isTorEnabled = this.elements.useTor.checked;
|
||
this.updateTorModeUI();
|
||
this.updateExitNodesUI();
|
||
if (!torData || !torData.ip || torData.ip === 'Unknown') {
|
||
this.elements.torCard.classList.remove('active');
|
||
this.elements.torStatusBadge.className = isTorEnabled ? 'tor-pill tor-on' : 'tor-pill tor-off';
|
||
this.elements.torStatusBadge.textContent = isTorEnabled ? 'Tor Enabled (Awaiting Run)' : 'Tor Inactive';
|
||
this.elements.torIpVal.textContent = '--';
|
||
this.elements.torCountryVal.textContent = '--';
|
||
this.elements.torCityVal.textContent = '--';
|
||
this.elements.torOrgVal.textContent = '--';
|
||
if (this.elements.torRotationsVal) this.elements.torRotationsVal.textContent = '0';
|
||
if (this.elements.torFilterVal) this.elements.torFilterVal.textContent = isTorEnabled ? (this.selectedExitNodes.size > 0 ? Array.from(this.selectedExitNodes).map(c=>c.toUpperCase()).join(', ') : 'Global (Any)') : 'Disabled';
|
||
return;
|
||
}
|
||
|
||
this.elements.torCard.classList.add('active');
|
||
this.elements.torStatusBadge.className = 'tor-pill tor-on';
|
||
this.elements.torStatusBadge.textContent = 'Tor Routing Active';
|
||
|
||
this.elements.torIpVal.textContent = torData.ip || '--';
|
||
|
||
const countryStr = torData.country_code ? `${torData.country || ''} (${torData.country_code})` : (torData.country || '--');
|
||
this.elements.torCountryVal.textContent = countryStr;
|
||
|
||
const cityRegion = [torData.city, torData.region].filter(x => x && x !== 'Unknown').join(', ') || '--';
|
||
this.elements.torCityVal.textContent = cityRegion;
|
||
|
||
this.elements.torOrgVal.textContent = torData.org || '--';
|
||
|
||
if (this.elements.torRotationsVal) {
|
||
this.elements.torRotationsVal.textContent = torData.rotation_count !== undefined ? torData.rotation_count : '1';
|
||
}
|
||
|
||
if (this.elements.torFilterVal) {
|
||
if (torData.exit_nodes && torData.exit_nodes.length > 0) {
|
||
const codes = torData.exit_nodes.map(c => c.toUpperCase());
|
||
this.elements.torFilterVal.textContent = codes.length <= 4 ? codes.join(', ') : `${codes.slice(0, 4).join(', ')} (+${codes.length - 4})`;
|
||
} else if (this.selectedExitNodes.size > 0) {
|
||
const codes = Array.from(this.selectedExitNodes).map(c => c.toUpperCase());
|
||
this.elements.torFilterVal.textContent = codes.length <= 4 ? codes.join(', ') : `${codes.slice(0, 4).join(', ')} (+${codes.length - 4})`;
|
||
} else {
|
||
this.elements.torFilterVal.textContent = 'Global (Any)';
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
updateStatus(status, message, elapsed) {
|
||
this.status = status;
|
||
const badge = this.elements.crawlStatusBadge;
|
||
const label = badge.querySelector('.status-label');
|
||
|
||
badge.className = `status-badge ${status}`;
|
||
label.textContent = status.toUpperCase();
|
||
|
||
if (message) {
|
||
this.elements.progressMessage.textContent = message;
|
||
}
|
||
|
||
if (status === 'running') {
|
||
this.elements.btnStart.disabled = true;
|
||
this.elements.btnStop.disabled = false;
|
||
this.elements.progressBar.className = 'progress-bar-fill running';
|
||
this.startTimer(elapsed || 0);
|
||
} else {
|
||
this.elements.btnStart.disabled = false;
|
||
this.elements.btnStop.disabled = true;
|
||
this.elements.progressBar.className = 'progress-bar-fill';
|
||
this.stopTimer();
|
||
if (elapsed !== undefined) {
|
||
this.elements.elapsedTimer.textContent = this.formatDuration(elapsed);
|
||
}
|
||
}
|
||
}
|
||
|
||
updateStats(data) {
|
||
const stats = data.stats || {};
|
||
this.elements.statVisited.textContent = data.visited_count !== undefined ? data.visited_count : 0;
|
||
this.elements.statInternal.textContent = stats.internal || 0;
|
||
this.elements.statExternal.textContent = stats.external || 0;
|
||
this.elements.statCookies.textContent = stats.cookies_accepted || 0;
|
||
this.elements.statQueue.textContent = data.queue_size || 0;
|
||
|
||
const errorCount = data.error_count !== undefined ? data.error_count : (data.total_errors_all !== undefined ? data.total_errors_all : this.errors.length);
|
||
this.elements.statErrors.textContent = errorCount;
|
||
|
||
const totalVisitedAll = data.total_visited_all !== undefined ? data.total_visited_all : (data.visited_count || 0);
|
||
const totalErrorsAll = data.total_errors_all !== undefined ? data.total_errors_all : this.errors.length;
|
||
|
||
this.elements.errorCountBadge.textContent = totalErrorsAll;
|
||
|
||
if (this.elements.statTotalVisitedAll) {
|
||
this.elements.statTotalVisitedAll.textContent = totalVisitedAll;
|
||
}
|
||
if (this.elements.statTotalErrorsAll) {
|
||
this.elements.statTotalErrorsAll.textContent = totalErrorsAll;
|
||
}
|
||
}
|
||
|
||
// --- Timer Management ---
|
||
startTimer(initialElapsed = 0) {
|
||
this.stopTimer();
|
||
this.elapsedSeconds = Math.floor(initialElapsed);
|
||
this.elements.elapsedTimer.textContent = this.formatDuration(this.elapsedSeconds);
|
||
|
||
this.timerInterval = setInterval(() => {
|
||
this.elapsedSeconds++;
|
||
this.elements.elapsedTimer.textContent = this.formatDuration(this.elapsedSeconds);
|
||
}, 1000);
|
||
}
|
||
|
||
stopTimer() {
|
||
if (this.timerInterval) {
|
||
clearInterval(this.timerInterval);
|
||
this.timerInterval = null;
|
||
}
|
||
}
|
||
|
||
formatDuration(sec) {
|
||
const s = Math.floor(sec % 60);
|
||
const m = Math.floor((sec / 60) % 60);
|
||
const h = Math.floor(sec / 3600);
|
||
return `${h.toString().padStart(2, '0')}:${m.toString().padStart(2, '0')}:${s.toString().padStart(2, '0')}`;
|
||
}
|
||
|
||
formatTime(timestamp) {
|
||
const date = timestamp ? new Date(timestamp * 1000) : new Date();
|
||
return date.toTimeString().split(' ')[0];
|
||
}
|
||
|
||
// --- Dedicated Error Section ---
|
||
addError(errorEntry) {
|
||
this.errors.push(errorEntry);
|
||
this.renderErrorsTable();
|
||
const totalErr = errorEntry.total_errors_all !== undefined ? errorEntry.total_errors_all : this.errors.length;
|
||
this.elements.statErrors.textContent = this.elements.statErrors.textContent || totalErr;
|
||
this.elements.errorCountBadge.textContent = totalErr;
|
||
if (this.elements.statTotalErrorsAll) {
|
||
this.elements.statTotalErrorsAll.textContent = totalErr;
|
||
}
|
||
}
|
||
|
||
renderErrorsTable() {
|
||
const filterQuery = (this.elements.errorSearchInput.value || '').toLowerCase().trim();
|
||
const filtered = this.errors.filter(e => {
|
||
if (!filterQuery) return true;
|
||
return (
|
||
(e.url && e.url.toLowerCase().includes(filterQuery)) ||
|
||
(e.error && e.error.toLowerCase().includes(filterQuery)) ||
|
||
(e.source_url && e.source_url.toLowerCase().includes(filterQuery))
|
||
);
|
||
});
|
||
|
||
const tbody = this.elements.errorsTableBody;
|
||
if (filtered.length === 0) {
|
||
tbody.innerHTML = `
|
||
<tr class="empty-row">
|
||
<td colspan="6">
|
||
<div class="empty-state">
|
||
<span class="empty-icon">✨</span>
|
||
<p>${filterQuery ? 'No matching errors found for search filter.' : 'No website errors recorded yet. Happy crawling!'}</p>
|
||
</div>
|
||
</td>
|
||
</tr>
|
||
`;
|
||
return;
|
||
}
|
||
|
||
tbody.innerHTML = filtered.map((e, index) => `
|
||
<tr>
|
||
<td><strong>${index + 1}</strong></td>
|
||
<td class="url-cell">
|
||
<a href="${this.escapeHtml(e.url)}" target="_blank" rel="noopener noreferrer" style="color:inherit; text-decoration:none;">
|
||
${this.escapeHtml(e.url)}
|
||
</a>
|
||
</td>
|
||
<td>
|
||
<span class="error-badge">${this.escapeHtml(e.error)}</span>
|
||
</td>
|
||
<td class="source-cell">${this.escapeHtml(e.source_url || 'Direct Input')}</td>
|
||
<td>${this.formatTime(e.timestamp)}</td>
|
||
<td>
|
||
<button class="copy-btn" onclick="navigator.clipboard.writeText('${this.escapeHtml(e.url)}'); window.dashboard.showToast('Copied URL!', 'info');" title="Copy URL">
|
||
Copy
|
||
</button>
|
||
</td>
|
||
</tr>
|
||
`).join('');
|
||
}
|
||
|
||
// --- Live Log Console ---
|
||
addLog(logEntry) {
|
||
this.logs.push(logEntry);
|
||
if (this.logs.length > 2000) {
|
||
this.logs.shift();
|
||
}
|
||
this.renderLogLine(logEntry);
|
||
this.updateLogCount();
|
||
}
|
||
|
||
renderLogLine(logEntry) {
|
||
const activeLevel = this.elements.logLevelSelect.value;
|
||
const filterQuery = (this.elements.logSearchInput.value || '').toLowerCase();
|
||
|
||
if (activeLevel !== 'all' && logEntry.level !== activeLevel) {
|
||
return;
|
||
}
|
||
if (filterQuery && !logEntry.message.toLowerCase().includes(filterQuery)) {
|
||
return;
|
||
}
|
||
|
||
const lineDiv = document.createElement('div');
|
||
lineDiv.className = `log-line log-${logEntry.level}`;
|
||
|
||
let tagClass = 'tag-sys';
|
||
let tagText = '[INFO]';
|
||
if (logEntry.level === 'error') { tagClass = 'tag-err'; tagText = '[ERROR]'; }
|
||
else if (logEntry.level === 'warn') { tagClass = 'tag-warn'; tagText = '[WARN]'; }
|
||
else if (logEntry.level === 'success') { tagClass = 'tag-ok'; tagText = '[OK]'; }
|
||
|
||
lineDiv.innerHTML = `
|
||
<span class="log-time">[${this.formatTime(logEntry.timestamp)}]</span>
|
||
<span class="log-msg">${this.escapeHtml(logEntry.message)}</span>
|
||
`;
|
||
|
||
this.elements.logOutput.appendChild(lineDiv);
|
||
|
||
if (this.elements.autoScrollCheck.checked) {
|
||
this.elements.terminalContainer.scrollTop = this.elements.terminalContainer.scrollHeight;
|
||
}
|
||
}
|
||
|
||
filterLogs() {
|
||
this.elements.logOutput.innerHTML = '';
|
||
this.logs.forEach(log => this.renderLogLine(log));
|
||
}
|
||
|
||
updateLogCount() {
|
||
this.elements.logCountBadge.textContent = `${this.logs.length} lines`;
|
||
}
|
||
|
||
// --- API Calls ---
|
||
async loadConfig() {
|
||
try {
|
||
const res = await fetch('/api/config');
|
||
const data = await res.json();
|
||
if (data.success && data.config) {
|
||
const c = data.config;
|
||
this.elements.startUrl.value = c.start_url || '';
|
||
this.elements.numTabs.value = c.num_tabs || 5;
|
||
this.elements.numTabsVal.textContent = c.num_tabs || 5;
|
||
this.elements.numIterations.value = c.num_iterations || 1;
|
||
this.elements.headless.checked = !!c.headless;
|
||
this.elements.acceptCookies.checked = c.accept_cookies !== false;
|
||
if (this.elements.randomizeQueue) {
|
||
this.elements.randomizeQueue.checked = c.randomize_queue !== false;
|
||
}
|
||
this.elements.useTor.checked = !!c.use_tor;
|
||
|
||
const isVisitsMode = (c.iteration_mode === 'visits');
|
||
if (this.elements.iterModeVisits && this.elements.iterModeAll) {
|
||
this.elements.iterModeVisits.checked = isVisitsMode;
|
||
this.elements.iterModeAll.checked = !isVisitsMode;
|
||
}
|
||
if (this.elements.maxVisitsPerIteration) {
|
||
this.elements.maxVisitsPerIteration.value = c.max_visits_per_iteration || 10;
|
||
}
|
||
this.updateIterationModeUI();
|
||
|
||
const isPagesMode = (c.tor_circuit_mode === 'pages');
|
||
if (this.elements.torModePages && this.elements.torModeIteration) {
|
||
this.elements.torModePages.checked = isPagesMode;
|
||
this.elements.torModeIteration.checked = !isPagesMode;
|
||
}
|
||
if (this.elements.torCircuitPages) {
|
||
this.elements.torCircuitPages.value = c.tor_circuit_pages || 10;
|
||
}
|
||
if (this.elements.torMaxConsecutiveTimeouts) {
|
||
this.elements.torMaxConsecutiveTimeouts.value = c.tor_max_consecutive_timeouts || 5;
|
||
}
|
||
|
||
const exitNodes = Array.isArray(c.tor_exit_nodes) ? c.tor_exit_nodes : [];
|
||
this.selectedExitNodes = new Set(exitNodes.map(x => String(x).toLowerCase().trim()));
|
||
|
||
this.elements.timeoutMs.value = c.timeout_ms || 35000;
|
||
this.elements.slowMo.value = c.slow_mo || 100;
|
||
this.elements.torSocksPort.value = c.tor_socks_port || 9050;
|
||
this.elements.torControlPort.value = c.tor_control_port || 9051;
|
||
|
||
// User-Agent Population
|
||
const ua = (c.user_agent || 'random').trim();
|
||
if (this.elements.userAgentSelect) {
|
||
if (ua === 'random' || ua === 'default') {
|
||
this.elements.userAgentSelect.value = ua;
|
||
if (this.elements.userAgentCustom) {
|
||
this.elements.userAgentCustom.style.display = 'none';
|
||
this.elements.userAgentCustom.value = ua;
|
||
}
|
||
} else {
|
||
this.elements.userAgentSelect.value = 'custom';
|
||
if (this.elements.userAgentCustom) {
|
||
this.elements.userAgentCustom.style.display = 'block';
|
||
this.elements.userAgentCustom.value = ua;
|
||
}
|
||
}
|
||
}
|
||
|
||
this.updateTorVisibility();
|
||
}
|
||
} catch (err) {
|
||
console.error('Failed to load config:', err);
|
||
this.showToast('Could not load .env configuration', 'error');
|
||
}
|
||
}
|
||
|
||
getFormData() {
|
||
const isVisitsMode = this.elements.iterModeVisits ? this.elements.iterModeVisits.checked : false;
|
||
const maxVisits = this.elements.maxVisitsPerIteration ? parseInt(this.elements.maxVisitsPerIteration.value, 10) : 10;
|
||
const isPagesMode = this.elements.torModePages ? this.elements.torModePages.checked : false;
|
||
const circuitPages = this.elements.torCircuitPages ? parseInt(this.elements.torCircuitPages.value, 10) : 10;
|
||
const maxTimeouts = this.elements.torMaxConsecutiveTimeouts ? parseInt(this.elements.torMaxConsecutiveTimeouts.value, 10) : 5;
|
||
|
||
let userAgent = 'random';
|
||
if (this.elements.userAgentSelect) {
|
||
const sel = this.elements.userAgentSelect.value;
|
||
if (sel === 'custom') {
|
||
userAgent = (this.elements.userAgentCustom ? this.elements.userAgentCustom.value.trim() : '') || 'random';
|
||
} else {
|
||
userAgent = sel;
|
||
}
|
||
}
|
||
|
||
return {
|
||
start_url: this.elements.startUrl.value.trim(),
|
||
num_tabs: parseInt(this.elements.numTabs.value, 10),
|
||
num_iterations: parseInt(this.elements.numIterations.value, 10),
|
||
iteration_mode: isVisitsMode ? 'visits' : 'all',
|
||
max_visits_per_iteration: isNaN(maxVisits) || maxVisits < 1 ? 10 : maxVisits,
|
||
randomize_queue: this.elements.randomizeQueue ? this.elements.randomizeQueue.checked : true,
|
||
headless: this.elements.headless.checked,
|
||
accept_cookies: this.elements.acceptCookies.checked,
|
||
use_tor: this.elements.useTor.checked,
|
||
tor_circuit_mode: isPagesMode ? 'pages' : 'iteration',
|
||
tor_circuit_pages: isNaN(circuitPages) || circuitPages < 1 ? 10 : circuitPages,
|
||
tor_max_consecutive_timeouts: isNaN(maxTimeouts) || maxTimeouts < 1 ? 5 : maxTimeouts,
|
||
tor_exit_nodes: Array.from(this.selectedExitNodes),
|
||
timeout_ms: parseInt(this.elements.timeoutMs.value, 10),
|
||
slow_mo: parseInt(this.elements.slowMo.value, 10),
|
||
tor_socks_port: parseInt(this.elements.torSocksPort.value, 10),
|
||
tor_control_port: parseInt(this.elements.torControlPort.value, 10),
|
||
user_agent: userAgent,
|
||
};
|
||
}
|
||
|
||
async saveConfigToEnv() {
|
||
const payload = this.getFormData();
|
||
try {
|
||
const res = await fetch('/api/config', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(payload),
|
||
});
|
||
const data = await res.json();
|
||
if (data.success) {
|
||
this.showToast('Configuration saved to .env!', 'success');
|
||
} else {
|
||
this.showToast(data.detail || 'Failed to save .env', 'error');
|
||
}
|
||
} catch (err) {
|
||
this.showToast('Error saving .env file', 'error');
|
||
}
|
||
}
|
||
|
||
async startCrawl() {
|
||
const payload = this.getFormData();
|
||
if (!payload.start_url) {
|
||
this.showToast('Please enter a valid starting URL', 'error');
|
||
return;
|
||
}
|
||
|
||
try {
|
||
this.elements.btnStart.disabled = true;
|
||
const res = await fetch('/api/crawl/start', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(payload),
|
||
});
|
||
const data = await res.json();
|
||
if (!data.success) {
|
||
this.showToast(data.detail || 'Could not start crawler', 'error');
|
||
this.elements.btnStart.disabled = false;
|
||
} else {
|
||
this.showToast('Crawl started!', 'success');
|
||
}
|
||
} catch (err) {
|
||
this.showToast('Network error starting crawl', 'error');
|
||
this.elements.btnStart.disabled = false;
|
||
}
|
||
}
|
||
|
||
async stopCrawl() {
|
||
try {
|
||
this.elements.btnStop.disabled = true;
|
||
const res = await fetch('/api/crawl/stop', { method: 'POST' });
|
||
const data = await res.json();
|
||
if (data.success) {
|
||
this.showToast('Stopping crawler workers...', 'info');
|
||
} else {
|
||
this.showToast(data.detail || 'Could not stop crawler', 'error');
|
||
}
|
||
} catch (err) {
|
||
this.showToast('Error sending stop signal', 'error');
|
||
}
|
||
}
|
||
|
||
showToast(message, type = 'info') {
|
||
const container = this.elements.toastContainer;
|
||
const toast = document.createElement('div');
|
||
toast.className = `toast toast-${type}`;
|
||
|
||
let icon = 'ℹ️';
|
||
if (type === 'success') icon = '✅';
|
||
if (type === 'error') icon = '❌';
|
||
|
||
toast.innerHTML = `<span>${icon}</span><span>${this.escapeHtml(message)}</span>`;
|
||
container.appendChild(toast);
|
||
|
||
setTimeout(() => {
|
||
toast.style.opacity = '0';
|
||
toast.style.transform = 'translateX(100%)';
|
||
toast.style.transition = 'all 0.3s ease';
|
||
setTimeout(() => toast.remove(), 300);
|
||
}, 3500);
|
||
}
|
||
|
||
escapeHtml(str) {
|
||
if (!str) return '';
|
||
return String(str)
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, ''');
|
||
}
|
||
}
|
||
|
||
// Global initialization
|
||
document.addEventListener('DOMContentLoaded', () => {
|
||
window.dashboard = new WebsurferDashboard();
|
||
});
|