Merge pull request #43 from JRS1986/feat/status-page-and-bypass-fallbacks

Add loading status page and multi-source bypass fallbacks
This commit is contained in:
Wasi Master
2026-05-12 00:24:57 +06:00
committed by GitHub
+529 -54
View File
@@ -2,14 +2,22 @@ import os
from urllib.parse import urljoin, urlparse from urllib.parse import urljoin, urlparse
import flask import flask
import re
import requests import requests
import uuid
import threading
import time
from flask import request, Response
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from flask import request from urllib.parse import urlparse, urljoin, quote
app = flask.Flask(__name__) app = flask.Flask(__name__)
googlebot_headers = { googlebot_headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
} }
jobs = {}
html = """ html = """
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
@@ -19,8 +27,9 @@ html = """
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>13ft Ladder</title> <title>13ft Ladder</title>
<link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet" async> <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap" rel="stylesheet" async>
<style> <style>
* { box-sizing: border-box; }
body { body {
font-family: 'Open Sans', sans-serif; font-family: 'Open Sans', sans-serif;
background-color: #FFF; background-color: #FFF;
@@ -53,36 +62,33 @@ html = """
border-radius: 5px; border-radius: 5px;
width: 100%; width: 100%;
font-size: 1rem; font-size: 1rem;
box-sizing: border-box;
} }
input[type="submit"] { input[type="submit"] {
padding: 10px; padding: 16px;
background-color: #6a0dad; background-color: #a327f0;
color: #fff; color: #fff;
border: none; border: none;
border-radius: 5px; border-radius: 5px;
width: 100%; width: 100%;
text-transform: uppercase; text-transform: uppercase;
font-size: 1rem;
font-weight: 600; font-weight: 600;
cursor: pointer; cursor: pointer;
transition: background-color 0.3s ease; transition: background-color 0.3s ease;
} }
input[type="submit"]:hover { input[type="submit"]:hover {
background-color: #4e0875; background-color: #821fc0;
} }
/* Toggle switch styles */
.dark-mode-toggle { .dark-mode-toggle {
position: absolute; position: absolute;
top: 10px; top: 10px;
right: 10px; right: 10px;
} }
.dark-mode-toggle input { .dark-mode-toggle input { display: none; }
display: none;
}
.dark-mode-toggle label { .dark-mode-toggle label {
cursor: pointer; cursor: pointer;
@@ -107,49 +113,137 @@ html = """
transition: 0.3s; transition: 0.3s;
} }
.dark-mode-toggle input:checked+label { .dark-mode-toggle input:checked+label { background: #7359f8; }
background: #6a0dad;
}
.dark-mode-toggle input:checked+label:after { .dark-mode-toggle input:checked+label:after {
left: calc(100% - 2px); left: calc(100% - 2px);
transform: translateX(-100%); transform: translateX(-100%);
} }
/* Responsive adjustments */
@media only screen and (max-width: 600px) { @media only screen and (max-width: 600px) {
form { form, .status-container { padding: 10px; }
padding: 10px; h1 { font-size: 1.2rem; }
}
h1 {
font-size: 1.2rem;
}
} }
/* Dark mode styles */ body.dark-mode { background-color: #191a21; color: #FFF; }
body.dark-mode { body.dark-mode h1 { color: #FFF; }
background-color: #333; body.dark-mode input[type=text] { background-color: #555; border: 1px solid #777; color: #FFF; }
color: #FFF; body.dark-mode input[type="submit"] { background-color: #7359f8; }
body.dark-mode input[type="submit"]:hover { background-color: #5c47c6; }
body.dark-mode .status-container { background: #23243a; }
body.dark-mode .step { color: #aaa; }
body.dark-mode .step.active { color: #fff; }
body.dark-mode .step.done { color: #7359f8; }
body.dark-mode .error-box { background: #3a1c1c; border-color: #ff6b6b; color: #ffaaaa; }
/* Status page styles */
.status-container {
max-width: 480px;
width: 100%;
padding: 30px;
text-align: center;
} }
body.dark-mode h1 { .status-container h1 { margin-bottom: 30px; }
color: #FFF;
.url-display {
font-size: 0.85rem;
color: #888;
margin-bottom: 24px;
word-break: break-all;
} }
body.dark-mode input[type=text] { .progress-bar-track {
background-color: #555; width: 100%;
border: 1px solid #777; height: 6px;
color: #FFF; background: #e0e0e0;
border-radius: 3px;
overflow: hidden;
margin-bottom: 28px;
} }
body.dark-mode input[type="submit"] { body.dark-mode .progress-bar-track { background: #444; }
background-color: #9b30ff;
.progress-bar-fill {
height: 100%;
width: 0%;
background: linear-gradient(90deg, #a327f0, #7359f8);
border-radius: 3px;
transition: width 0.5s ease;
} }
body.dark-mode input[type="submit"]:hover { .steps { text-align: left; margin-bottom: 24px; }
background-color: #7a1bb5;
.step {
display: flex;
align-items: center;
gap: 12px;
padding: 8px 0;
color: #bbb;
font-size: 0.95rem;
transition: color 0.3s;
} }
.step.active { color: #333; font-weight: 600; }
.step.done { color: #a327f0; }
.step-icon {
width: 24px;
height: 24px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
}
.spinner {
width: 18px;
height: 18px;
border: 2px solid #e0e0e0;
border-top-color: #a327f0;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
body.dark-mode .spinner { border-color: #444; border-top-color: #7359f8; }
@keyframes spin { to { transform: rotate(360deg); } }
.checkmark { color: #a327f0; font-weight: bold; font-size: 1.1rem; }
.elapsed {
font-size: 0.85rem;
color: #999;
margin-top: 4px;
}
.error-box {
background: #fff0f0;
border: 1px solid #ffcccc;
border-radius: 8px;
padding: 16px;
margin-top: 16px;
color: #cc3333;
font-size: 0.9rem;
text-align: left;
}
.error-box strong { display: block; margin-bottom: 6px; }
.retry-btn {
display: inline-block;
margin-top: 16px;
padding: 10px 24px;
background: #a327f0;
color: #fff;
border: none;
border-radius: 5px;
font-size: 0.95rem;
font-weight: 600;
cursor: pointer;
text-decoration: none;
}
.retry-btn:hover { background: #821fc0; }
</style> </style>
</head> </head>
@@ -158,12 +252,28 @@ html = """
<input type="checkbox" id="dark-mode-toggle"> <input type="checkbox" id="dark-mode-toggle">
<label for="dark-mode-toggle" title="Toggle Dark Mode"></label> <label for="dark-mode-toggle" title="Toggle Dark Mode"></label>
</div> </div>
<form action="/article" method="post">
<h1>Enter Website Link</h1> <div id="form-view">
<label for="link">Link of the website you want to remove paywall for:</label> <form id="url-form">
<input type="text" id="link" name="link" required autofocus> <h1>Enter Website Link</h1>
<input type="submit" value="Submit"> <label for="link">Link of the website you want to remove paywall for:</label>
</form> <input type="text" id="link" name="link" required autofocus>
<input type="submit" value="Submit">
</form>
</div>
<div id="status-view" style="display:none;">
<div class="status-container">
<h1>Fetching Article</h1>
<div class="url-display" id="status-url"></div>
<div class="progress-bar-track">
<div class="progress-bar-fill" id="progress-fill"></div>
</div>
<div class="steps" id="steps-list"></div>
<div class="elapsed" id="elapsed-time"></div>
<div id="error-area"></div>
</div>
</div>
<script> <script>
const toggleSwitch = document.getElementById('dark-mode-toggle'); const toggleSwitch = document.getElementById('dark-mode-toggle');
@@ -183,6 +293,116 @@ html = """
localStorage.setItem('theme', 'light'); localStorage.setItem('theme', 'light');
} }
}); });
const STEPS = [
{ id: 'connect', label: 'Connecting to website...' },
{ id: 'fetch', label: 'Downloading page content...' },
{ id: 'detect', label: 'Checking for anti-bot challenges...' },
{ id: 'fallback_freedium', label: 'Trying Freedium (Medium bypass)...' },
{ id: 'fallback_org', label: 'Trying archive.org snapshot...' },
{ id: 'fallback_ph', label: 'Trying archive.today / archive.ph snapshot...' },
{ id: 'process', label: 'Processing article...' },
{ id: 'cleanup', label: 'Cleaning up & preparing view...' },
{ id: 'done', label: 'Done!' }
];
function renderSteps(activeIdx) {
const container = document.getElementById('steps-list');
container.innerHTML = '';
STEPS.forEach((step, i) => {
const div = document.createElement('div');
div.className = 'step' + (i < activeIdx ? ' done' : i === activeIdx ? ' active' : '');
const icon = document.createElement('div');
icon.className = 'step-icon';
if (i < activeIdx) {
icon.innerHTML = '<span class="checkmark">&#10003;</span>';
} else if (i === activeIdx) {
icon.innerHTML = '<div class="spinner"></div>';
} else {
icon.innerHTML = '<span style="color:#ddd;">&#9675;</span>';
}
const text = document.createElement('span');
text.textContent = step.label;
div.appendChild(icon);
div.appendChild(text);
container.appendChild(div);
});
}
function showStatus(url) {
document.getElementById('form-view').style.display = 'none';
document.getElementById('status-view').style.display = 'flex';
document.getElementById('status-url').textContent = url;
document.getElementById('error-area').innerHTML = '';
renderSteps(0);
document.getElementById('progress-fill').style.width = '0%';
}
function showError(message) {
const area = document.getElementById('error-area');
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
message + '</div>' +
'<a href="/" class="retry-btn">Try Another URL</a>';
}
document.getElementById('url-form').addEventListener('submit', function(e) {
e.preventDefault();
const link = document.getElementById('link').value.trim();
if (!link) return;
showStatus(link);
const startTime = Date.now();
const elapsedEl = document.getElementById('elapsed-time');
const progressEl = document.getElementById('progress-fill');
const timer = setInterval(() => {
const sec = ((Date.now() - startTime) / 1000).toFixed(0);
elapsedEl.textContent = sec + 's elapsed';
}, 500);
const formData = new FormData();
formData.append('link', link);
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
let currentStep = 0;
evtSource.addEventListener('step', function(e) {
const data = JSON.parse(e.data);
const idx = STEPS.findIndex(s => s.id === data.step);
if (idx >= 0) {
currentStep = idx;
renderSteps(idx);
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
progressEl.style.width = pct + '%';
}
});
evtSource.addEventListener('done', function(e) {
evtSource.close();
clearInterval(timer);
renderSteps(STEPS.length);
progressEl.style.width = '100%';
const data = JSON.parse(e.data);
setTimeout(() => {
document.open();
document.write(data.html);
document.close();
}, 400);
});
evtSource.addEventListener('error_msg', function(e) {
evtSource.close();
clearInterval(timer);
const data = JSON.parse(e.data);
showError(data.message);
});
evtSource.onerror = function() {
evtSource.close();
clearInterval(timer);
showError('Connection to server lost. The page may be taking too long to load.');
};
});
</script> </script>
</body> </body>
@@ -195,12 +415,11 @@ def add_base_tag(html_content, original_url):
parsed_url = urlparse(original_url) parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/" base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html" # Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith("/"): if parsed_url.path and not parsed_url.path.endswith("/"):
base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/") base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
base_tag = soup.find("base") base_tag = soup.find("base")
print(base_url)
if not base_tag: if not base_tag:
new_base_tag = soup.new_tag("base", href=base_url) new_base_tag = soup.new_tag("base", href=base_url)
if soup.head: if soup.head:
@@ -213,19 +432,236 @@ def add_base_tag(html_content, original_url):
return str(soup) return str(soup)
def bypass_paywall(url): CHALLENGE_SIGNATURES = [
""" "just a moment",
Bypass paywall for a given url "checking your browser",
""" "cloudflare",
if url.startswith("http"): "cf-challenge",
response = requests.get(url, headers=googlebot_headers) "ddos protection",
response.encoding = response.apparent_encoding "attention required",
return add_base_tag(response.text, response.url) "enable javascript and cookies to continue",
"please verify you are a human",
]
def is_challenge_page(html_text):
if not html_text:
return False
lower = html_text[:8000].lower()
hits = sum(1 for sig in CHALLENGE_SIGNATURES if sig in lower)
if hits >= 1 and len(html_text) < 20000:
return True
return hits >= 2
def set_step(job_id, step):
if job_id and job_id in jobs:
jobs[job_id]['step'] = step
REAL_BROWSER_HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
}
ARCHIVE_PH_MIRRORS = ["archive.ph", "archive.today", "archive.is", "archive.li"]
FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"]
def is_medium_url(url):
try: try:
return bypass_paywall("https://" + url) host = (urlparse(url).hostname or "").lower()
return host == "medium.com" or host.endswith(".medium.com")
except Exception:
return False
def fetch_via_freedium(url, job_id=None):
"""Freedium is a Medium-specific paywall bypass service."""
set_step(job_id, 'fallback_freedium')
clean_url = url.split('?')[0]
for mirror in FREEDIUM_MIRRORS:
try:
freedium_url = f"https://{mirror}/{clean_url}"
resp = requests.get(
freedium_url,
headers=REAL_BROWSER_HEADERS,
timeout=30,
allow_redirects=True,
)
if resp.status_code != 200:
continue
resp.encoding = resp.apparent_encoding
text = resp.text
if is_challenge_page(text):
continue
if 'main-content' not in text or len(text) < 2000:
continue
return text, resp.url
except Exception:
continue
return None, None
def fetch_via_archive_org(url, job_id=None):
set_step(job_id, 'fallback_org')
wayback_api = f"https://archive.org/wayback/available?url={url}"
try:
meta = requests.get(wayback_api, timeout=15).json()
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
if not snapshot.get("available"):
return None, None
archived_url = snapshot["url"]
if archived_url.startswith("http://"):
archived_url = "https://" + archived_url[len("http://"):]
if "/web/" in archived_url and "id_/" not in archived_url:
archived_url = archived_url.replace("/web/", "/web/", 1)
archived_url = archived_url.replace(
archived_url.split("/web/")[1].split("/")[0],
archived_url.split("/web/")[1].split("/")[0] + "id_",
1,
)
resp = requests.get(archived_url, headers=googlebot_headers, timeout=30)
resp.encoding = resp.apparent_encoding
return resp.text, resp.url
except Exception:
return None, None
def fetch_via_archive_ph(url, job_id=None):
"""Try archive.today / archive.ph / archive.is — all mirrors of the same service."""
set_step(job_id, 'fallback_ph')
for mirror in ARCHIVE_PH_MIRRORS:
try:
newest_url = f"https://{mirror}/newest/{url}"
resp = requests.get(
newest_url,
headers=REAL_BROWSER_HEADERS,
timeout=20,
allow_redirects=True,
)
if resp.status_code != 200:
continue
final = resp.url
if not re.search(r"https?://archive\.(ph|today|is|li)/[A-Za-z0-9]{4,}", final):
continue
if re.search(r"archive\.(ph|today|is|li)/newest/", final):
continue
text = resp.text
if is_challenge_page(text):
continue
lower_head = text[:5000].lower()
if "no results" in lower_head or "0 captures" in lower_head:
continue
return text, final
except Exception:
continue
return None, None
def bypass_paywall(url, job_id=None):
set_step(job_id, 'connect')
if not url.startswith("http"):
url = "https://" + url
medium = is_medium_url(url)
html_text = ""
final_url = url
if medium:
freedium_html, freedium_final = fetch_via_freedium(url, job_id)
if freedium_html:
set_step(job_id, 'process')
result = add_base_tag(freedium_html, freedium_final)
set_step(job_id, 'cleanup')
return result
else:
set_step(job_id, 'fetch')
response = requests.get(url, headers=googlebot_headers, timeout=30)
response.encoding = response.apparent_encoding
html_text = response.text
final_url = response.url
set_step(job_id, 'detect')
if not html_text or is_challenge_page(html_text):
recovered = False
if not medium:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
else:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
if medium:
msg = (
"Medium served an anti-bot challenge and Freedium / archive.org / "
"archive.today all came back empty. The article may be too new to "
"have been archived yet."
)
else:
msg = (
"The site served an anti-bot challenge (Cloudflare or similar) "
"and no usable snapshot was found on archive.org or archive.today. "
"This site is actively blocking automated requests."
)
raise RuntimeError(msg)
set_step(job_id, 'process')
result = add_base_tag(html_text, final_url)
set_step(job_id, 'cleanup')
return result
def fetch_worker(job_id, url):
try:
result = bypass_paywall(url, job_id)
jobs[job_id]['result'] = result
jobs[job_id]['step'] = 'done'
except requests.exceptions.Timeout:
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.ConnectionError:
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
return bypass_paywall("http://" + url) jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
jobs[job_id]['step'] = 'error'
except RuntimeError as e:
jobs[job_id]['error'] = str(e)
jobs[job_id]['step'] = 'error'
except Exception as e:
jobs[job_id]['error'] = f'Unexpected error: {e}'
jobs[job_id]['step'] = 'error'
@app.route("/") @app.route("/")
@@ -233,6 +669,45 @@ def main_page():
return html return html
@app.route("/status/<path:url>")
def status_stream(url):
import json
job_id = str(uuid.uuid4())
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
thread.daemon = True
thread.start()
def generate():
last_step = None
while True:
job = jobs.get(job_id)
if not job:
break
current = job['step']
if current != last_step:
last_step = current
if current == 'done':
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
break
elif current == 'error':
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
break
else:
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
time.sleep(0.2)
jobs.pop(job_id, None)
return Response(generate(), mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'})
@app.route("/article", methods=["POST"]) @app.route("/article", methods=["POST"])
def show_article(): def show_article():
link = flask.request.form["link"] link = flask.request.form["link"]