Add loading status page and multi-source bypass fallbacks

Replaces the previous synchronous form-POST flow (which left users
staring at a white screen for the duration of the upstream fetch)
with an SSE-driven status page that shows real-time progress through
each stage of the bypass attempt.

Bypass pipeline:
- Direct fetch with Googlebot UA (unchanged for non-Medium URLs)
- Detect Cloudflare-style anti-bot challenge pages in the response
- Fallback 1: Freedium (freedium-mirror.cfd, then freedium.cfd) for
  medium.com hosts — short-circuited before direct fetch since
  Cloudflare consistently blocks the Googlebot UA from non-Google IPs
- Fallback 2: archive.org Wayback Machine snapshot
- Fallback 3: archive.today / archive.ph / archive.is / archive.li
  mirrors (all four tried in sequence)
- Surface a clear error if every source fails

UI:
- Animated step list with spinner / checkmark per stage
- Progress bar and elapsed-time counter
- Honest error box with retry button instead of a blank page

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Jan R. Seyler
2026-05-11 15:00:00 +02:00
co-authored by Claude Opus 4.7
parent 641ec49000
commit f0a12864ec
+526 -51
View File
@@ -1,13 +1,20 @@
import flask
import re
import requests
from flask import request
import uuid
import threading
import time
from flask import request, Response
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
from urllib.parse import urlparse, urljoin, quote
app = flask.Flask(__name__)
googlebot_headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
}
jobs = {}
html = """
<!DOCTYPE html>
<html lang="en">
@@ -17,8 +24,9 @@ html = """
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>13ft Ladder</title>
<link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet" async>
<link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap" rel="stylesheet" async>
<style>
* { box-sizing: border-box; }
body {
font-family: 'Open Sans', sans-serif;
background-color: #FFF;
@@ -51,36 +59,33 @@ html = """
border-radius: 5px;
width: 100%;
font-size: 1rem;
box-sizing: border-box;
}
input[type="submit"] {
padding: 10px;
background-color: #6a0dad;
padding: 16px;
background-color: #a327f0;
color: #fff;
border: none;
border-radius: 5px;
width: 100%;
text-transform: uppercase;
font-size: 1rem;
font-weight: 600;
cursor: pointer;
transition: background-color 0.3s ease;
}
input[type="submit"]:hover {
background-color: #4e0875;
background-color: #821fc0;
}
/* Toggle switch styles */
.dark-mode-toggle {
position: absolute;
top: 10px;
right: 10px;
}
.dark-mode-toggle input {
display: none;
}
.dark-mode-toggle input { display: none; }
.dark-mode-toggle label {
cursor: pointer;
@@ -105,49 +110,137 @@ html = """
transition: 0.3s;
}
.dark-mode-toggle input:checked+label {
background: #6a0dad;
}
.dark-mode-toggle input:checked+label { background: #7359f8; }
.dark-mode-toggle input:checked+label:after {
left: calc(100% - 2px);
transform: translateX(-100%);
}
/* Responsive adjustments */
@media only screen and (max-width: 600px) {
form {
padding: 10px;
form, .status-container { padding: 10px; }
h1 { font-size: 1.2rem; }
}
h1 {
font-size: 1.2rem;
}
body.dark-mode { background-color: #191a21; color: #FFF; }
body.dark-mode h1 { color: #FFF; }
body.dark-mode input[type=text] { background-color: #555; border: 1px solid #777; color: #FFF; }
body.dark-mode input[type="submit"] { background-color: #7359f8; }
body.dark-mode input[type="submit"]:hover { background-color: #5c47c6; }
body.dark-mode .status-container { background: #23243a; }
body.dark-mode .step { color: #aaa; }
body.dark-mode .step.active { color: #fff; }
body.dark-mode .step.done { color: #7359f8; }
body.dark-mode .error-box { background: #3a1c1c; border-color: #ff6b6b; color: #ffaaaa; }
/* Status page styles */
.status-container {
max-width: 480px;
width: 100%;
padding: 30px;
text-align: center;
}
/* Dark mode styles */
body.dark-mode {
background-color: #333;
color: #FFF;
.status-container h1 { margin-bottom: 30px; }
.url-display {
font-size: 0.85rem;
color: #888;
margin-bottom: 24px;
word-break: break-all;
}
body.dark-mode h1 {
color: #FFF;
.progress-bar-track {
width: 100%;
height: 6px;
background: #e0e0e0;
border-radius: 3px;
overflow: hidden;
margin-bottom: 28px;
}
body.dark-mode input[type=text] {
background-color: #555;
border: 1px solid #777;
color: #FFF;
body.dark-mode .progress-bar-track { background: #444; }
.progress-bar-fill {
height: 100%;
width: 0%;
background: linear-gradient(90deg, #a327f0, #7359f8);
border-radius: 3px;
transition: width 0.5s ease;
}
body.dark-mode input[type="submit"] {
background-color: #9b30ff;
.steps { text-align: left; margin-bottom: 24px; }
.step {
display: flex;
align-items: center;
gap: 12px;
padding: 8px 0;
color: #bbb;
font-size: 0.95rem;
transition: color 0.3s;
}
body.dark-mode input[type="submit"]:hover {
background-color: #7a1bb5;
.step.active { color: #333; font-weight: 600; }
.step.done { color: #a327f0; }
.step-icon {
width: 24px;
height: 24px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
}
.spinner {
width: 18px;
height: 18px;
border: 2px solid #e0e0e0;
border-top-color: #a327f0;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
body.dark-mode .spinner { border-color: #444; border-top-color: #7359f8; }
@keyframes spin { to { transform: rotate(360deg); } }
.checkmark { color: #a327f0; font-weight: bold; font-size: 1.1rem; }
.elapsed {
font-size: 0.85rem;
color: #999;
margin-top: 4px;
}
.error-box {
background: #fff0f0;
border: 1px solid #ffcccc;
border-radius: 8px;
padding: 16px;
margin-top: 16px;
color: #cc3333;
font-size: 0.9rem;
text-align: left;
}
.error-box strong { display: block; margin-bottom: 6px; }
.retry-btn {
display: inline-block;
margin-top: 16px;
padding: 10px 24px;
background: #a327f0;
color: #fff;
border: none;
border-radius: 5px;
font-size: 0.95rem;
font-weight: 600;
cursor: pointer;
text-decoration: none;
}
.retry-btn:hover { background: #821fc0; }
</style>
</head>
@@ -156,12 +249,28 @@ html = """
<input type="checkbox" id="dark-mode-toggle">
<label for="dark-mode-toggle" title="Toggle Dark Mode"></label>
</div>
<form action="/article" method="post">
<div id="form-view">
<form id="url-form">
<h1>Enter Website Link</h1>
<label for="link">Link of the website you want to remove paywall for:</label>
<input type="text" id="link" name="link" required autofocus>
<input type="submit" value="Submit">
</form>
</div>
<div id="status-view" style="display:none;">
<div class="status-container">
<h1>Fetching Article</h1>
<div class="url-display" id="status-url"></div>
<div class="progress-bar-track">
<div class="progress-bar-fill" id="progress-fill"></div>
</div>
<div class="steps" id="steps-list"></div>
<div class="elapsed" id="elapsed-time"></div>
<div id="error-area"></div>
</div>
</div>
<script>
const toggleSwitch = document.getElementById('dark-mode-toggle');
@@ -181,23 +290,132 @@ html = """
localStorage.setItem('theme', 'light');
}
});
const STEPS = [
{ id: 'connect', label: 'Connecting to website...' },
{ id: 'fetch', label: 'Downloading page content...' },
{ id: 'detect', label: 'Checking for anti-bot challenges...' },
{ id: 'fallback_freedium', label: 'Trying Freedium (Medium bypass)...' },
{ id: 'fallback_org', label: 'Trying archive.org snapshot...' },
{ id: 'fallback_ph', label: 'Trying archive.today / archive.ph snapshot...' },
{ id: 'process', label: 'Processing article...' },
{ id: 'cleanup', label: 'Cleaning up & preparing view...' },
{ id: 'done', label: 'Done!' }
];
function renderSteps(activeIdx) {
const container = document.getElementById('steps-list');
container.innerHTML = '';
STEPS.forEach((step, i) => {
const div = document.createElement('div');
div.className = 'step' + (i < activeIdx ? ' done' : i === activeIdx ? ' active' : '');
const icon = document.createElement('div');
icon.className = 'step-icon';
if (i < activeIdx) {
icon.innerHTML = '<span class="checkmark">&#10003;</span>';
} else if (i === activeIdx) {
icon.innerHTML = '<div class="spinner"></div>';
} else {
icon.innerHTML = '<span style="color:#ddd;">&#9675;</span>';
}
const text = document.createElement('span');
text.textContent = step.label;
div.appendChild(icon);
div.appendChild(text);
container.appendChild(div);
});
}
function showStatus(url) {
document.getElementById('form-view').style.display = 'none';
document.getElementById('status-view').style.display = 'flex';
document.getElementById('status-url').textContent = url;
document.getElementById('error-area').innerHTML = '';
renderSteps(0);
document.getElementById('progress-fill').style.width = '0%';
}
function showError(message) {
const area = document.getElementById('error-area');
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
message + '</div>' +
'<a href="/" class="retry-btn">Try Another URL</a>';
}
document.getElementById('url-form').addEventListener('submit', function(e) {
e.preventDefault();
const link = document.getElementById('link').value.trim();
if (!link) return;
showStatus(link);
const startTime = Date.now();
const elapsedEl = document.getElementById('elapsed-time');
const progressEl = document.getElementById('progress-fill');
const timer = setInterval(() => {
const sec = ((Date.now() - startTime) / 1000).toFixed(0);
elapsedEl.textContent = sec + 's elapsed';
}, 500);
const formData = new FormData();
formData.append('link', link);
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
let currentStep = 0;
evtSource.addEventListener('step', function(e) {
const data = JSON.parse(e.data);
const idx = STEPS.findIndex(s => s.id === data.step);
if (idx >= 0) {
currentStep = idx;
renderSteps(idx);
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
progressEl.style.width = pct + '%';
}
});
evtSource.addEventListener('done', function(e) {
evtSource.close();
clearInterval(timer);
renderSteps(STEPS.length);
progressEl.style.width = '100%';
const data = JSON.parse(e.data);
setTimeout(() => {
document.open();
document.write(data.html);
document.close();
}, 400);
});
evtSource.addEventListener('error_msg', function(e) {
evtSource.close();
clearInterval(timer);
const data = JSON.parse(e.data);
showError(data.message);
});
evtSource.onerror = function() {
evtSource.close();
clearInterval(timer);
showError('Connection to server lost. The page may be taking too long to load.');
};
});
</script>
</body>
</html>
"""
def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser')
parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
base_tag = soup.find('base')
print(base_url)
if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url)
if soup.head:
@@ -209,19 +427,237 @@ def add_base_tag(html_content, original_url):
return str(soup)
def bypass_paywall(url):
"""
Bypass paywall for a given url
"""
if url.startswith("http"):
response = requests.get(url, headers=googlebot_headers)
response.encoding = response.apparent_encoding
return add_base_tag(response.text, response.url)
CHALLENGE_SIGNATURES = [
"just a moment",
"checking your browser",
"cloudflare",
"cf-challenge",
"ddos protection",
"attention required",
"enable javascript and cookies to continue",
"please verify you are a human",
]
def is_challenge_page(html_text):
if not html_text:
return False
lower = html_text[:8000].lower()
hits = sum(1 for sig in CHALLENGE_SIGNATURES if sig in lower)
if hits >= 1 and len(html_text) < 20000:
return True
return hits >= 2
def set_step(job_id, step):
if job_id and job_id in jobs:
jobs[job_id]['step'] = step
REAL_BROWSER_HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
}
ARCHIVE_PH_MIRRORS = ["archive.ph", "archive.today", "archive.is", "archive.li"]
FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"]
def is_medium_url(url):
try:
return bypass_paywall("https://" + url)
host = (urlparse(url).hostname or "").lower()
return host == "medium.com" or host.endswith(".medium.com")
except Exception:
return False
def fetch_via_freedium(url, job_id=None):
"""Freedium is a Medium-specific paywall bypass service."""
set_step(job_id, 'fallback_freedium')
clean_url = url.split('?')[0]
for mirror in FREEDIUM_MIRRORS:
try:
freedium_url = f"https://{mirror}/{clean_url}"
resp = requests.get(
freedium_url,
headers=REAL_BROWSER_HEADERS,
timeout=30,
allow_redirects=True,
)
if resp.status_code != 200:
continue
resp.encoding = resp.apparent_encoding
text = resp.text
if is_challenge_page(text):
continue
if 'main-content' not in text or len(text) < 2000:
continue
return text, resp.url
except Exception:
continue
return None, None
def fetch_via_archive_org(url, job_id=None):
set_step(job_id, 'fallback_org')
wayback_api = f"https://archive.org/wayback/available?url={url}"
try:
meta = requests.get(wayback_api, timeout=15).json()
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
if not snapshot.get("available"):
return None, None
archived_url = snapshot["url"]
if archived_url.startswith("http://"):
archived_url = "https://" + archived_url[len("http://"):]
if "/web/" in archived_url and "id_/" not in archived_url:
archived_url = archived_url.replace("/web/", "/web/", 1)
archived_url = archived_url.replace(
archived_url.split("/web/")[1].split("/")[0],
archived_url.split("/web/")[1].split("/")[0] + "id_",
1,
)
resp = requests.get(archived_url, headers=googlebot_headers, timeout=30)
resp.encoding = resp.apparent_encoding
return resp.text, resp.url
except Exception:
return None, None
def fetch_via_archive_ph(url, job_id=None):
"""Try archive.today / archive.ph / archive.is — all mirrors of the same service."""
set_step(job_id, 'fallback_ph')
for mirror in ARCHIVE_PH_MIRRORS:
try:
newest_url = f"https://{mirror}/newest/{url}"
resp = requests.get(
newest_url,
headers=REAL_BROWSER_HEADERS,
timeout=20,
allow_redirects=True,
)
if resp.status_code != 200:
continue
final = resp.url
if not re.search(r"https?://archive\.(ph|today|is|li)/[A-Za-z0-9]{4,}", final):
continue
if re.search(r"archive\.(ph|today|is|li)/newest/", final):
continue
text = resp.text
if is_challenge_page(text):
continue
lower_head = text[:5000].lower()
if "no results" in lower_head or "0 captures" in lower_head:
continue
return text, final
except Exception:
continue
return None, None
def bypass_paywall(url, job_id=None):
set_step(job_id, 'connect')
if not url.startswith("http"):
url = "https://" + url
medium = is_medium_url(url)
html_text = ""
final_url = url
if medium:
freedium_html, freedium_final = fetch_via_freedium(url, job_id)
if freedium_html:
set_step(job_id, 'process')
result = add_base_tag(freedium_html, freedium_final)
set_step(job_id, 'cleanup')
return result
else:
set_step(job_id, 'fetch')
response = requests.get(url, headers=googlebot_headers, timeout=30)
response.encoding = response.apparent_encoding
html_text = response.text
final_url = response.url
set_step(job_id, 'detect')
if not html_text or is_challenge_page(html_text):
recovered = False
if not medium:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
else:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
if medium:
msg = (
"Medium served an anti-bot challenge and Freedium / archive.org / "
"archive.today all came back empty. The article may be too new to "
"have been archived yet."
)
else:
msg = (
"The site served an anti-bot challenge (Cloudflare or similar) "
"and no usable snapshot was found on archive.org or archive.today. "
"This site is actively blocking automated requests."
)
raise RuntimeError(msg)
set_step(job_id, 'process')
result = add_base_tag(html_text, final_url)
set_step(job_id, 'cleanup')
return result
def fetch_worker(job_id, url):
try:
result = bypass_paywall(url, job_id)
jobs[job_id]['result'] = result
jobs[job_id]['step'] = 'done'
except requests.exceptions.Timeout:
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.ConnectionError:
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.RequestException as e:
return bypass_paywall("http://" + url)
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
jobs[job_id]['step'] = 'error'
except RuntimeError as e:
jobs[job_id]['error'] = str(e)
jobs[job_id]['step'] = 'error'
except Exception as e:
jobs[job_id]['error'] = f'Unexpected error: {e}'
jobs[job_id]['step'] = 'error'
@app.route("/")
@@ -229,6 +665,45 @@ def main_page():
return html
@app.route("/status/<path:url>")
def status_stream(url):
import json
job_id = str(uuid.uuid4())
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
thread.daemon = True
thread.start()
def generate():
last_step = None
while True:
job = jobs.get(job_id)
if not job:
break
current = job['step']
if current != last_step:
last_step = current
if current == 'done':
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
break
elif current == 'error':
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
break
else:
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
time.sleep(0.2)
jobs.pop(job_id, None)
return Response(generate(), mimetype='text/event-stream',
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'})
@app.route("/article", methods=["POST"])
def show_article():
link = flask.request.form["link"]
@@ -236,8 +711,8 @@ def show_article():
return bypass_paywall(link)
except requests.exceptions.RequestException as e:
return str(e), 400
except e:
raise e
except Exception as exc:
raise exc
@app.route("/", defaults={"path": ""})
@@ -251,7 +726,7 @@ def get_article(path):
return bypass_paywall(actual_url)
except requests.exceptions.RequestException as e:
return str(e), 400
except e:
except Exception as e:
raise e
else:
return "Invalid URL", 400