Merge pull request #43 from JRS1986/feat/status-page-and-bypass-fallbacks
Add loading status page and multi-source bypass fallbacks
This commit is contained in:
+529
-54
@@ -2,14 +2,22 @@ import os
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import flask
|
||||
import re
|
||||
import requests
|
||||
import uuid
|
||||
import threading
|
||||
import time
|
||||
from flask import request, Response
|
||||
from bs4 import BeautifulSoup
|
||||
from flask import request
|
||||
from urllib.parse import urlparse, urljoin, quote
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
googlebot_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
}
|
||||
|
||||
jobs = {}
|
||||
|
||||
html = """
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
@@ -19,8 +27,9 @@ html = """
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<title>13ft Ladder</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet" async>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap" rel="stylesheet" async>
|
||||
<style>
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
font-family: 'Open Sans', sans-serif;
|
||||
background-color: #FFF;
|
||||
@@ -53,36 +62,33 @@ html = """
|
||||
border-radius: 5px;
|
||||
width: 100%;
|
||||
font-size: 1rem;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
input[type="submit"] {
|
||||
padding: 10px;
|
||||
background-color: #6a0dad;
|
||||
padding: 16px;
|
||||
background-color: #a327f0;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
width: 100%;
|
||||
text-transform: uppercase;
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.3s ease;
|
||||
}
|
||||
|
||||
input[type="submit"]:hover {
|
||||
background-color: #4e0875;
|
||||
background-color: #821fc0;
|
||||
}
|
||||
|
||||
/* Toggle switch styles */
|
||||
.dark-mode-toggle {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
}
|
||||
|
||||
.dark-mode-toggle input {
|
||||
display: none;
|
||||
}
|
||||
.dark-mode-toggle input { display: none; }
|
||||
|
||||
.dark-mode-toggle label {
|
||||
cursor: pointer;
|
||||
@@ -107,49 +113,137 @@ html = """
|
||||
transition: 0.3s;
|
||||
}
|
||||
|
||||
.dark-mode-toggle input:checked+label {
|
||||
background: #6a0dad;
|
||||
}
|
||||
|
||||
.dark-mode-toggle input:checked+label { background: #7359f8; }
|
||||
.dark-mode-toggle input:checked+label:after {
|
||||
left: calc(100% - 2px);
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
|
||||
/* Responsive adjustments */
|
||||
@media only screen and (max-width: 600px) {
|
||||
form {
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.2rem;
|
||||
}
|
||||
form, .status-container { padding: 10px; }
|
||||
h1 { font-size: 1.2rem; }
|
||||
}
|
||||
|
||||
/* Dark mode styles */
|
||||
body.dark-mode {
|
||||
background-color: #333;
|
||||
color: #FFF;
|
||||
body.dark-mode { background-color: #191a21; color: #FFF; }
|
||||
body.dark-mode h1 { color: #FFF; }
|
||||
body.dark-mode input[type=text] { background-color: #555; border: 1px solid #777; color: #FFF; }
|
||||
body.dark-mode input[type="submit"] { background-color: #7359f8; }
|
||||
body.dark-mode input[type="submit"]:hover { background-color: #5c47c6; }
|
||||
body.dark-mode .status-container { background: #23243a; }
|
||||
body.dark-mode .step { color: #aaa; }
|
||||
body.dark-mode .step.active { color: #fff; }
|
||||
body.dark-mode .step.done { color: #7359f8; }
|
||||
body.dark-mode .error-box { background: #3a1c1c; border-color: #ff6b6b; color: #ffaaaa; }
|
||||
|
||||
/* Status page styles */
|
||||
.status-container {
|
||||
max-width: 480px;
|
||||
width: 100%;
|
||||
padding: 30px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
body.dark-mode h1 {
|
||||
color: #FFF;
|
||||
.status-container h1 { margin-bottom: 30px; }
|
||||
|
||||
.url-display {
|
||||
font-size: 0.85rem;
|
||||
color: #888;
|
||||
margin-bottom: 24px;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
body.dark-mode input[type=text] {
|
||||
background-color: #555;
|
||||
border: 1px solid #777;
|
||||
color: #FFF;
|
||||
.progress-bar-track {
|
||||
width: 100%;
|
||||
height: 6px;
|
||||
background: #e0e0e0;
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
margin-bottom: 28px;
|
||||
}
|
||||
|
||||
body.dark-mode input[type="submit"] {
|
||||
background-color: #9b30ff;
|
||||
body.dark-mode .progress-bar-track { background: #444; }
|
||||
|
||||
.progress-bar-fill {
|
||||
height: 100%;
|
||||
width: 0%;
|
||||
background: linear-gradient(90deg, #a327f0, #7359f8);
|
||||
border-radius: 3px;
|
||||
transition: width 0.5s ease;
|
||||
}
|
||||
|
||||
body.dark-mode input[type="submit"]:hover {
|
||||
background-color: #7a1bb5;
|
||||
.steps { text-align: left; margin-bottom: 24px; }
|
||||
|
||||
.step {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 8px 0;
|
||||
color: #bbb;
|
||||
font-size: 0.95rem;
|
||||
transition: color 0.3s;
|
||||
}
|
||||
|
||||
.step.active { color: #333; font-weight: 600; }
|
||||
.step.done { color: #a327f0; }
|
||||
|
||||
.step-icon {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border: 2px solid #e0e0e0;
|
||||
border-top-color: #a327f0;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
body.dark-mode .spinner { border-color: #444; border-top-color: #7359f8; }
|
||||
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.checkmark { color: #a327f0; font-weight: bold; font-size: 1.1rem; }
|
||||
|
||||
.elapsed {
|
||||
font-size: 0.85rem;
|
||||
color: #999;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.error-box {
|
||||
background: #fff0f0;
|
||||
border: 1px solid #ffcccc;
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
margin-top: 16px;
|
||||
color: #cc3333;
|
||||
font-size: 0.9rem;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.error-box strong { display: block; margin-bottom: 6px; }
|
||||
|
||||
.retry-btn {
|
||||
display: inline-block;
|
||||
margin-top: 16px;
|
||||
padding: 10px 24px;
|
||||
background: #a327f0;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
font-size: 0.95rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.retry-btn:hover { background: #821fc0; }
|
||||
</style>
|
||||
</head>
|
||||
|
||||
@@ -158,12 +252,28 @@ html = """
|
||||
<input type="checkbox" id="dark-mode-toggle">
|
||||
<label for="dark-mode-toggle" title="Toggle Dark Mode"></label>
|
||||
</div>
|
||||
<form action="/article" method="post">
|
||||
<h1>Enter Website Link</h1>
|
||||
<label for="link">Link of the website you want to remove paywall for:</label>
|
||||
<input type="text" id="link" name="link" required autofocus>
|
||||
<input type="submit" value="Submit">
|
||||
</form>
|
||||
|
||||
<div id="form-view">
|
||||
<form id="url-form">
|
||||
<h1>Enter Website Link</h1>
|
||||
<label for="link">Link of the website you want to remove paywall for:</label>
|
||||
<input type="text" id="link" name="link" required autofocus>
|
||||
<input type="submit" value="Submit">
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="status-view" style="display:none;">
|
||||
<div class="status-container">
|
||||
<h1>Fetching Article</h1>
|
||||
<div class="url-display" id="status-url"></div>
|
||||
<div class="progress-bar-track">
|
||||
<div class="progress-bar-fill" id="progress-fill"></div>
|
||||
</div>
|
||||
<div class="steps" id="steps-list"></div>
|
||||
<div class="elapsed" id="elapsed-time"></div>
|
||||
<div id="error-area"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const toggleSwitch = document.getElementById('dark-mode-toggle');
|
||||
@@ -183,6 +293,116 @@ html = """
|
||||
localStorage.setItem('theme', 'light');
|
||||
}
|
||||
});
|
||||
|
||||
const STEPS = [
|
||||
{ id: 'connect', label: 'Connecting to website...' },
|
||||
{ id: 'fetch', label: 'Downloading page content...' },
|
||||
{ id: 'detect', label: 'Checking for anti-bot challenges...' },
|
||||
{ id: 'fallback_freedium', label: 'Trying Freedium (Medium bypass)...' },
|
||||
{ id: 'fallback_org', label: 'Trying archive.org snapshot...' },
|
||||
{ id: 'fallback_ph', label: 'Trying archive.today / archive.ph snapshot...' },
|
||||
{ id: 'process', label: 'Processing article...' },
|
||||
{ id: 'cleanup', label: 'Cleaning up & preparing view...' },
|
||||
{ id: 'done', label: 'Done!' }
|
||||
];
|
||||
|
||||
function renderSteps(activeIdx) {
|
||||
const container = document.getElementById('steps-list');
|
||||
container.innerHTML = '';
|
||||
STEPS.forEach((step, i) => {
|
||||
const div = document.createElement('div');
|
||||
div.className = 'step' + (i < activeIdx ? ' done' : i === activeIdx ? ' active' : '');
|
||||
const icon = document.createElement('div');
|
||||
icon.className = 'step-icon';
|
||||
if (i < activeIdx) {
|
||||
icon.innerHTML = '<span class="checkmark">✓</span>';
|
||||
} else if (i === activeIdx) {
|
||||
icon.innerHTML = '<div class="spinner"></div>';
|
||||
} else {
|
||||
icon.innerHTML = '<span style="color:#ddd;">○</span>';
|
||||
}
|
||||
const text = document.createElement('span');
|
||||
text.textContent = step.label;
|
||||
div.appendChild(icon);
|
||||
div.appendChild(text);
|
||||
container.appendChild(div);
|
||||
});
|
||||
}
|
||||
|
||||
function showStatus(url) {
|
||||
document.getElementById('form-view').style.display = 'none';
|
||||
document.getElementById('status-view').style.display = 'flex';
|
||||
document.getElementById('status-url').textContent = url;
|
||||
document.getElementById('error-area').innerHTML = '';
|
||||
renderSteps(0);
|
||||
document.getElementById('progress-fill').style.width = '0%';
|
||||
}
|
||||
|
||||
function showError(message) {
|
||||
const area = document.getElementById('error-area');
|
||||
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
|
||||
message + '</div>' +
|
||||
'<a href="/" class="retry-btn">Try Another URL</a>';
|
||||
}
|
||||
|
||||
document.getElementById('url-form').addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
const link = document.getElementById('link').value.trim();
|
||||
if (!link) return;
|
||||
showStatus(link);
|
||||
|
||||
const startTime = Date.now();
|
||||
const elapsedEl = document.getElementById('elapsed-time');
|
||||
const progressEl = document.getElementById('progress-fill');
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const sec = ((Date.now() - startTime) / 1000).toFixed(0);
|
||||
elapsedEl.textContent = sec + 's elapsed';
|
||||
}, 500);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('link', link);
|
||||
|
||||
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
|
||||
let currentStep = 0;
|
||||
|
||||
evtSource.addEventListener('step', function(e) {
|
||||
const data = JSON.parse(e.data);
|
||||
const idx = STEPS.findIndex(s => s.id === data.step);
|
||||
if (idx >= 0) {
|
||||
currentStep = idx;
|
||||
renderSteps(idx);
|
||||
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
|
||||
progressEl.style.width = pct + '%';
|
||||
}
|
||||
});
|
||||
|
||||
evtSource.addEventListener('done', function(e) {
|
||||
evtSource.close();
|
||||
clearInterval(timer);
|
||||
renderSteps(STEPS.length);
|
||||
progressEl.style.width = '100%';
|
||||
const data = JSON.parse(e.data);
|
||||
setTimeout(() => {
|
||||
document.open();
|
||||
document.write(data.html);
|
||||
document.close();
|
||||
}, 400);
|
||||
});
|
||||
|
||||
evtSource.addEventListener('error_msg', function(e) {
|
||||
evtSource.close();
|
||||
clearInterval(timer);
|
||||
const data = JSON.parse(e.data);
|
||||
showError(data.message);
|
||||
});
|
||||
|
||||
evtSource.onerror = function() {
|
||||
evtSource.close();
|
||||
clearInterval(timer);
|
||||
showError('Connection to server lost. The page may be taking too long to load.');
|
||||
};
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
@@ -195,12 +415,11 @@ def add_base_tag(html_content, original_url):
|
||||
parsed_url = urlparse(original_url)
|
||||
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
|
||||
|
||||
|
||||
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
|
||||
if parsed_url.path and not parsed_url.path.endswith("/"):
|
||||
base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
|
||||
base_tag = soup.find("base")
|
||||
|
||||
print(base_url)
|
||||
if not base_tag:
|
||||
new_base_tag = soup.new_tag("base", href=base_url)
|
||||
if soup.head:
|
||||
@@ -213,19 +432,236 @@ def add_base_tag(html_content, original_url):
|
||||
return str(soup)
|
||||
|
||||
|
||||
def bypass_paywall(url):
|
||||
"""
|
||||
Bypass paywall for a given url
|
||||
"""
|
||||
if url.startswith("http"):
|
||||
response = requests.get(url, headers=googlebot_headers)
|
||||
response.encoding = response.apparent_encoding
|
||||
return add_base_tag(response.text, response.url)
|
||||
CHALLENGE_SIGNATURES = [
|
||||
"just a moment",
|
||||
"checking your browser",
|
||||
"cloudflare",
|
||||
"cf-challenge",
|
||||
"ddos protection",
|
||||
"attention required",
|
||||
"enable javascript and cookies to continue",
|
||||
"please verify you are a human",
|
||||
]
|
||||
|
||||
|
||||
def is_challenge_page(html_text):
|
||||
if not html_text:
|
||||
return False
|
||||
lower = html_text[:8000].lower()
|
||||
hits = sum(1 for sig in CHALLENGE_SIGNATURES if sig in lower)
|
||||
if hits >= 1 and len(html_text) < 20000:
|
||||
return True
|
||||
return hits >= 2
|
||||
|
||||
|
||||
def set_step(job_id, step):
|
||||
if job_id and job_id in jobs:
|
||||
jobs[job_id]['step'] = step
|
||||
|
||||
|
||||
REAL_BROWSER_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
ARCHIVE_PH_MIRRORS = ["archive.ph", "archive.today", "archive.is", "archive.li"]
|
||||
|
||||
FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"]
|
||||
|
||||
|
||||
def is_medium_url(url):
|
||||
try:
|
||||
return bypass_paywall("https://" + url)
|
||||
host = (urlparse(url).hostname or "").lower()
|
||||
return host == "medium.com" or host.endswith(".medium.com")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def fetch_via_freedium(url, job_id=None):
|
||||
"""Freedium is a Medium-specific paywall bypass service."""
|
||||
set_step(job_id, 'fallback_freedium')
|
||||
clean_url = url.split('?')[0]
|
||||
for mirror in FREEDIUM_MIRRORS:
|
||||
try:
|
||||
freedium_url = f"https://{mirror}/{clean_url}"
|
||||
resp = requests.get(
|
||||
freedium_url,
|
||||
headers=REAL_BROWSER_HEADERS,
|
||||
timeout=30,
|
||||
allow_redirects=True,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
continue
|
||||
resp.encoding = resp.apparent_encoding
|
||||
text = resp.text
|
||||
if is_challenge_page(text):
|
||||
continue
|
||||
if 'main-content' not in text or len(text) < 2000:
|
||||
continue
|
||||
return text, resp.url
|
||||
except Exception:
|
||||
continue
|
||||
return None, None
|
||||
|
||||
|
||||
def fetch_via_archive_org(url, job_id=None):
|
||||
set_step(job_id, 'fallback_org')
|
||||
wayback_api = f"https://archive.org/wayback/available?url={url}"
|
||||
try:
|
||||
meta = requests.get(wayback_api, timeout=15).json()
|
||||
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
|
||||
if not snapshot.get("available"):
|
||||
return None, None
|
||||
archived_url = snapshot["url"]
|
||||
if archived_url.startswith("http://"):
|
||||
archived_url = "https://" + archived_url[len("http://"):]
|
||||
if "/web/" in archived_url and "id_/" not in archived_url:
|
||||
archived_url = archived_url.replace("/web/", "/web/", 1)
|
||||
archived_url = archived_url.replace(
|
||||
archived_url.split("/web/")[1].split("/")[0],
|
||||
archived_url.split("/web/")[1].split("/")[0] + "id_",
|
||||
1,
|
||||
)
|
||||
resp = requests.get(archived_url, headers=googlebot_headers, timeout=30)
|
||||
resp.encoding = resp.apparent_encoding
|
||||
return resp.text, resp.url
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def fetch_via_archive_ph(url, job_id=None):
|
||||
"""Try archive.today / archive.ph / archive.is — all mirrors of the same service."""
|
||||
set_step(job_id, 'fallback_ph')
|
||||
for mirror in ARCHIVE_PH_MIRRORS:
|
||||
try:
|
||||
newest_url = f"https://{mirror}/newest/{url}"
|
||||
resp = requests.get(
|
||||
newest_url,
|
||||
headers=REAL_BROWSER_HEADERS,
|
||||
timeout=20,
|
||||
allow_redirects=True,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
continue
|
||||
|
||||
final = resp.url
|
||||
if not re.search(r"https?://archive\.(ph|today|is|li)/[A-Za-z0-9]{4,}", final):
|
||||
continue
|
||||
if re.search(r"archive\.(ph|today|is|li)/newest/", final):
|
||||
continue
|
||||
|
||||
text = resp.text
|
||||
if is_challenge_page(text):
|
||||
continue
|
||||
|
||||
lower_head = text[:5000].lower()
|
||||
if "no results" in lower_head or "0 captures" in lower_head:
|
||||
continue
|
||||
|
||||
return text, final
|
||||
except Exception:
|
||||
continue
|
||||
return None, None
|
||||
|
||||
|
||||
def bypass_paywall(url, job_id=None):
|
||||
set_step(job_id, 'connect')
|
||||
|
||||
if not url.startswith("http"):
|
||||
url = "https://" + url
|
||||
|
||||
medium = is_medium_url(url)
|
||||
html_text = ""
|
||||
final_url = url
|
||||
|
||||
if medium:
|
||||
freedium_html, freedium_final = fetch_via_freedium(url, job_id)
|
||||
if freedium_html:
|
||||
set_step(job_id, 'process')
|
||||
result = add_base_tag(freedium_html, freedium_final)
|
||||
set_step(job_id, 'cleanup')
|
||||
return result
|
||||
else:
|
||||
set_step(job_id, 'fetch')
|
||||
response = requests.get(url, headers=googlebot_headers, timeout=30)
|
||||
response.encoding = response.apparent_encoding
|
||||
html_text = response.text
|
||||
final_url = response.url
|
||||
|
||||
set_step(job_id, 'detect')
|
||||
if not html_text or is_challenge_page(html_text):
|
||||
recovered = False
|
||||
|
||||
if not medium:
|
||||
archived_html, archived_url = fetch_via_archive_org(url, job_id)
|
||||
if archived_html and not is_challenge_page(archived_html):
|
||||
html_text = archived_html
|
||||
final_url = archived_url
|
||||
recovered = True
|
||||
|
||||
if not recovered:
|
||||
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
|
||||
if archived_html and not is_challenge_page(archived_html):
|
||||
html_text = archived_html
|
||||
final_url = archived_url
|
||||
recovered = True
|
||||
else:
|
||||
archived_html, archived_url = fetch_via_archive_org(url, job_id)
|
||||
if archived_html and not is_challenge_page(archived_html):
|
||||
html_text = archived_html
|
||||
final_url = archived_url
|
||||
recovered = True
|
||||
|
||||
if not recovered:
|
||||
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
|
||||
if archived_html and not is_challenge_page(archived_html):
|
||||
html_text = archived_html
|
||||
final_url = archived_url
|
||||
recovered = True
|
||||
|
||||
if not recovered:
|
||||
if medium:
|
||||
msg = (
|
||||
"Medium served an anti-bot challenge and Freedium / archive.org / "
|
||||
"archive.today all came back empty. The article may be too new to "
|
||||
"have been archived yet."
|
||||
)
|
||||
else:
|
||||
msg = (
|
||||
"The site served an anti-bot challenge (Cloudflare or similar) "
|
||||
"and no usable snapshot was found on archive.org or archive.today. "
|
||||
"This site is actively blocking automated requests."
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
|
||||
set_step(job_id, 'process')
|
||||
result = add_base_tag(html_text, final_url)
|
||||
|
||||
set_step(job_id, 'cleanup')
|
||||
return result
|
||||
|
||||
|
||||
def fetch_worker(job_id, url):
|
||||
try:
|
||||
result = bypass_paywall(url, job_id)
|
||||
jobs[job_id]['result'] = result
|
||||
jobs[job_id]['step'] = 'done'
|
||||
except requests.exceptions.Timeout:
|
||||
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
|
||||
jobs[job_id]['step'] = 'error'
|
||||
except requests.exceptions.ConnectionError:
|
||||
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
|
||||
jobs[job_id]['step'] = 'error'
|
||||
except requests.exceptions.RequestException as e:
|
||||
return bypass_paywall("http://" + url)
|
||||
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
|
||||
jobs[job_id]['step'] = 'error'
|
||||
except RuntimeError as e:
|
||||
jobs[job_id]['error'] = str(e)
|
||||
jobs[job_id]['step'] = 'error'
|
||||
except Exception as e:
|
||||
jobs[job_id]['error'] = f'Unexpected error: {e}'
|
||||
jobs[job_id]['step'] = 'error'
|
||||
|
||||
|
||||
@app.route("/")
|
||||
@@ -233,6 +669,45 @@ def main_page():
|
||||
return html
|
||||
|
||||
|
||||
@app.route("/status/<path:url>")
|
||||
def status_stream(url):
|
||||
import json
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
|
||||
|
||||
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
def generate():
|
||||
last_step = None
|
||||
while True:
|
||||
job = jobs.get(job_id)
|
||||
if not job:
|
||||
break
|
||||
|
||||
current = job['step']
|
||||
if current != last_step:
|
||||
last_step = current
|
||||
if current == 'done':
|
||||
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
|
||||
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
|
||||
break
|
||||
elif current == 'error':
|
||||
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
|
||||
break
|
||||
else:
|
||||
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
jobs.pop(job_id, None)
|
||||
|
||||
return Response(generate(), mimetype='text/event-stream',
|
||||
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'})
|
||||
|
||||
|
||||
@app.route("/article", methods=["POST"])
|
||||
def show_article():
|
||||
link = flask.request.form["link"]
|
||||
|
||||
Reference in New Issue
Block a user