Add loading status page and multi-source bypass fallbacks
Replaces the previous synchronous form-POST flow (which left users staring at a white screen for the duration of the upstream fetch) with an SSE-driven status page that shows real-time progress through each stage of the bypass attempt. Bypass pipeline: - Direct fetch with Googlebot UA (unchanged for non-Medium URLs) - Detect Cloudflare-style anti-bot challenge pages in the response - Fallback 1: Freedium (freedium-mirror.cfd, then freedium.cfd) for medium.com hosts — short-circuited before direct fetch since Cloudflare consistently blocks the Googlebot UA from non-Google IPs - Fallback 2: archive.org Wayback Machine snapshot - Fallback 3: archive.today / archive.ph / archive.is / archive.li mirrors (all four tried in sequence) - Surface a clear error if every source fails UI: - Animated step list with spinner / checkmark per stage - Progress bar and elapsed-time counter - Honest error box with retry button instead of a blank page Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
641ec49000
commit
f0a12864ec
+533
-58
@@ -1,13 +1,20 @@
|
|||||||
import flask
|
import flask
|
||||||
|
import re
|
||||||
import requests
|
import requests
|
||||||
from flask import request
|
import uuid
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from flask import request, Response
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from urllib.parse import urlparse, urljoin
|
from urllib.parse import urlparse, urljoin, quote
|
||||||
|
|
||||||
app = flask.Flask(__name__)
|
app = flask.Flask(__name__)
|
||||||
googlebot_headers = {
|
googlebot_headers = {
|
||||||
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
jobs = {}
|
||||||
|
|
||||||
html = """
|
html = """
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
@@ -17,8 +24,9 @@ html = """
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||||
<title>13ft Ladder</title>
|
<title>13ft Ladder</title>
|
||||||
<link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet" async>
|
<link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap" rel="stylesheet" async>
|
||||||
<style>
|
<style>
|
||||||
|
* { box-sizing: border-box; }
|
||||||
body {
|
body {
|
||||||
font-family: 'Open Sans', sans-serif;
|
font-family: 'Open Sans', sans-serif;
|
||||||
background-color: #FFF;
|
background-color: #FFF;
|
||||||
@@ -51,36 +59,33 @@ html = """
|
|||||||
border-radius: 5px;
|
border-radius: 5px;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
font-size: 1rem;
|
font-size: 1rem;
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
input[type="submit"] {
|
input[type="submit"] {
|
||||||
padding: 10px;
|
padding: 16px;
|
||||||
background-color: #6a0dad;
|
background-color: #a327f0;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
border: none;
|
border: none;
|
||||||
border-radius: 5px;
|
border-radius: 5px;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
|
font-size: 1rem;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
transition: background-color 0.3s ease;
|
transition: background-color 0.3s ease;
|
||||||
}
|
}
|
||||||
|
|
||||||
input[type="submit"]:hover {
|
input[type="submit"]:hover {
|
||||||
background-color: #4e0875;
|
background-color: #821fc0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Toggle switch styles */
|
|
||||||
.dark-mode-toggle {
|
.dark-mode-toggle {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
top: 10px;
|
top: 10px;
|
||||||
right: 10px;
|
right: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark-mode-toggle input {
|
.dark-mode-toggle input { display: none; }
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dark-mode-toggle label {
|
.dark-mode-toggle label {
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
@@ -105,49 +110,137 @@ html = """
|
|||||||
transition: 0.3s;
|
transition: 0.3s;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark-mode-toggle input:checked+label {
|
.dark-mode-toggle input:checked+label { background: #7359f8; }
|
||||||
background: #6a0dad;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dark-mode-toggle input:checked+label:after {
|
.dark-mode-toggle input:checked+label:after {
|
||||||
left: calc(100% - 2px);
|
left: calc(100% - 2px);
|
||||||
transform: translateX(-100%);
|
transform: translateX(-100%);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Responsive adjustments */
|
|
||||||
@media only screen and (max-width: 600px) {
|
@media only screen and (max-width: 600px) {
|
||||||
form {
|
form, .status-container { padding: 10px; }
|
||||||
padding: 10px;
|
h1 { font-size: 1.2rem; }
|
||||||
}
|
|
||||||
|
|
||||||
h1 {
|
|
||||||
font-size: 1.2rem;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Dark mode styles */
|
body.dark-mode { background-color: #191a21; color: #FFF; }
|
||||||
body.dark-mode {
|
body.dark-mode h1 { color: #FFF; }
|
||||||
background-color: #333;
|
body.dark-mode input[type=text] { background-color: #555; border: 1px solid #777; color: #FFF; }
|
||||||
color: #FFF;
|
body.dark-mode input[type="submit"] { background-color: #7359f8; }
|
||||||
|
body.dark-mode input[type="submit"]:hover { background-color: #5c47c6; }
|
||||||
|
body.dark-mode .status-container { background: #23243a; }
|
||||||
|
body.dark-mode .step { color: #aaa; }
|
||||||
|
body.dark-mode .step.active { color: #fff; }
|
||||||
|
body.dark-mode .step.done { color: #7359f8; }
|
||||||
|
body.dark-mode .error-box { background: #3a1c1c; border-color: #ff6b6b; color: #ffaaaa; }
|
||||||
|
|
||||||
|
/* Status page styles */
|
||||||
|
.status-container {
|
||||||
|
max-width: 480px;
|
||||||
|
width: 100%;
|
||||||
|
padding: 30px;
|
||||||
|
text-align: center;
|
||||||
}
|
}
|
||||||
|
|
||||||
body.dark-mode h1 {
|
.status-container h1 { margin-bottom: 30px; }
|
||||||
color: #FFF;
|
|
||||||
|
.url-display {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: #888;
|
||||||
|
margin-bottom: 24px;
|
||||||
|
word-break: break-all;
|
||||||
}
|
}
|
||||||
|
|
||||||
body.dark-mode input[type=text] {
|
.progress-bar-track {
|
||||||
background-color: #555;
|
width: 100%;
|
||||||
border: 1px solid #777;
|
height: 6px;
|
||||||
color: #FFF;
|
background: #e0e0e0;
|
||||||
|
border-radius: 3px;
|
||||||
|
overflow: hidden;
|
||||||
|
margin-bottom: 28px;
|
||||||
}
|
}
|
||||||
|
|
||||||
body.dark-mode input[type="submit"] {
|
body.dark-mode .progress-bar-track { background: #444; }
|
||||||
background-color: #9b30ff;
|
|
||||||
|
.progress-bar-fill {
|
||||||
|
height: 100%;
|
||||||
|
width: 0%;
|
||||||
|
background: linear-gradient(90deg, #a327f0, #7359f8);
|
||||||
|
border-radius: 3px;
|
||||||
|
transition: width 0.5s ease;
|
||||||
}
|
}
|
||||||
|
|
||||||
body.dark-mode input[type="submit"]:hover {
|
.steps { text-align: left; margin-bottom: 24px; }
|
||||||
background-color: #7a1bb5;
|
|
||||||
|
.step {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 8px 0;
|
||||||
|
color: #bbb;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
transition: color 0.3s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.step.active { color: #333; font-weight: 600; }
|
||||||
|
.step.done { color: #a327f0; }
|
||||||
|
|
||||||
|
.step-icon {
|
||||||
|
width: 24px;
|
||||||
|
height: 24px;
|
||||||
|
flex-shrink: 0;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.spinner {
|
||||||
|
width: 18px;
|
||||||
|
height: 18px;
|
||||||
|
border: 2px solid #e0e0e0;
|
||||||
|
border-top-color: #a327f0;
|
||||||
|
border-radius: 50%;
|
||||||
|
animation: spin 0.8s linear infinite;
|
||||||
|
}
|
||||||
|
|
||||||
|
body.dark-mode .spinner { border-color: #444; border-top-color: #7359f8; }
|
||||||
|
|
||||||
|
@keyframes spin { to { transform: rotate(360deg); } }
|
||||||
|
|
||||||
|
.checkmark { color: #a327f0; font-weight: bold; font-size: 1.1rem; }
|
||||||
|
|
||||||
|
.elapsed {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: #999;
|
||||||
|
margin-top: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-box {
|
||||||
|
background: #fff0f0;
|
||||||
|
border: 1px solid #ffcccc;
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 16px;
|
||||||
|
margin-top: 16px;
|
||||||
|
color: #cc3333;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-box strong { display: block; margin-bottom: 6px; }
|
||||||
|
|
||||||
|
.retry-btn {
|
||||||
|
display: inline-block;
|
||||||
|
margin-top: 16px;
|
||||||
|
padding: 10px 24px;
|
||||||
|
background: #a327f0;
|
||||||
|
color: #fff;
|
||||||
|
border: none;
|
||||||
|
border-radius: 5px;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
font-weight: 600;
|
||||||
|
cursor: pointer;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.retry-btn:hover { background: #821fc0; }
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
@@ -156,12 +249,28 @@ html = """
|
|||||||
<input type="checkbox" id="dark-mode-toggle">
|
<input type="checkbox" id="dark-mode-toggle">
|
||||||
<label for="dark-mode-toggle" title="Toggle Dark Mode"></label>
|
<label for="dark-mode-toggle" title="Toggle Dark Mode"></label>
|
||||||
</div>
|
</div>
|
||||||
<form action="/article" method="post">
|
|
||||||
<h1>Enter Website Link</h1>
|
<div id="form-view">
|
||||||
<label for="link">Link of the website you want to remove paywall for:</label>
|
<form id="url-form">
|
||||||
<input type="text" id="link" name="link" required autofocus>
|
<h1>Enter Website Link</h1>
|
||||||
<input type="submit" value="Submit">
|
<label for="link">Link of the website you want to remove paywall for:</label>
|
||||||
</form>
|
<input type="text" id="link" name="link" required autofocus>
|
||||||
|
<input type="submit" value="Submit">
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="status-view" style="display:none;">
|
||||||
|
<div class="status-container">
|
||||||
|
<h1>Fetching Article</h1>
|
||||||
|
<div class="url-display" id="status-url"></div>
|
||||||
|
<div class="progress-bar-track">
|
||||||
|
<div class="progress-bar-fill" id="progress-fill"></div>
|
||||||
|
</div>
|
||||||
|
<div class="steps" id="steps-list"></div>
|
||||||
|
<div class="elapsed" id="elapsed-time"></div>
|
||||||
|
<div id="error-area"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
const toggleSwitch = document.getElementById('dark-mode-toggle');
|
const toggleSwitch = document.getElementById('dark-mode-toggle');
|
||||||
@@ -181,23 +290,132 @@ html = """
|
|||||||
localStorage.setItem('theme', 'light');
|
localStorage.setItem('theme', 'light');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const STEPS = [
|
||||||
|
{ id: 'connect', label: 'Connecting to website...' },
|
||||||
|
{ id: 'fetch', label: 'Downloading page content...' },
|
||||||
|
{ id: 'detect', label: 'Checking for anti-bot challenges...' },
|
||||||
|
{ id: 'fallback_freedium', label: 'Trying Freedium (Medium bypass)...' },
|
||||||
|
{ id: 'fallback_org', label: 'Trying archive.org snapshot...' },
|
||||||
|
{ id: 'fallback_ph', label: 'Trying archive.today / archive.ph snapshot...' },
|
||||||
|
{ id: 'process', label: 'Processing article...' },
|
||||||
|
{ id: 'cleanup', label: 'Cleaning up & preparing view...' },
|
||||||
|
{ id: 'done', label: 'Done!' }
|
||||||
|
];
|
||||||
|
|
||||||
|
function renderSteps(activeIdx) {
|
||||||
|
const container = document.getElementById('steps-list');
|
||||||
|
container.innerHTML = '';
|
||||||
|
STEPS.forEach((step, i) => {
|
||||||
|
const div = document.createElement('div');
|
||||||
|
div.className = 'step' + (i < activeIdx ? ' done' : i === activeIdx ? ' active' : '');
|
||||||
|
const icon = document.createElement('div');
|
||||||
|
icon.className = 'step-icon';
|
||||||
|
if (i < activeIdx) {
|
||||||
|
icon.innerHTML = '<span class="checkmark">✓</span>';
|
||||||
|
} else if (i === activeIdx) {
|
||||||
|
icon.innerHTML = '<div class="spinner"></div>';
|
||||||
|
} else {
|
||||||
|
icon.innerHTML = '<span style="color:#ddd;">○</span>';
|
||||||
|
}
|
||||||
|
const text = document.createElement('span');
|
||||||
|
text.textContent = step.label;
|
||||||
|
div.appendChild(icon);
|
||||||
|
div.appendChild(text);
|
||||||
|
container.appendChild(div);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function showStatus(url) {
|
||||||
|
document.getElementById('form-view').style.display = 'none';
|
||||||
|
document.getElementById('status-view').style.display = 'flex';
|
||||||
|
document.getElementById('status-url').textContent = url;
|
||||||
|
document.getElementById('error-area').innerHTML = '';
|
||||||
|
renderSteps(0);
|
||||||
|
document.getElementById('progress-fill').style.width = '0%';
|
||||||
|
}
|
||||||
|
|
||||||
|
function showError(message) {
|
||||||
|
const area = document.getElementById('error-area');
|
||||||
|
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
|
||||||
|
message + '</div>' +
|
||||||
|
'<a href="/" class="retry-btn">Try Another URL</a>';
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById('url-form').addEventListener('submit', function(e) {
|
||||||
|
e.preventDefault();
|
||||||
|
const link = document.getElementById('link').value.trim();
|
||||||
|
if (!link) return;
|
||||||
|
showStatus(link);
|
||||||
|
|
||||||
|
const startTime = Date.now();
|
||||||
|
const elapsedEl = document.getElementById('elapsed-time');
|
||||||
|
const progressEl = document.getElementById('progress-fill');
|
||||||
|
|
||||||
|
const timer = setInterval(() => {
|
||||||
|
const sec = ((Date.now() - startTime) / 1000).toFixed(0);
|
||||||
|
elapsedEl.textContent = sec + 's elapsed';
|
||||||
|
}, 500);
|
||||||
|
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('link', link);
|
||||||
|
|
||||||
|
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
|
||||||
|
let currentStep = 0;
|
||||||
|
|
||||||
|
evtSource.addEventListener('step', function(e) {
|
||||||
|
const data = JSON.parse(e.data);
|
||||||
|
const idx = STEPS.findIndex(s => s.id === data.step);
|
||||||
|
if (idx >= 0) {
|
||||||
|
currentStep = idx;
|
||||||
|
renderSteps(idx);
|
||||||
|
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
|
||||||
|
progressEl.style.width = pct + '%';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
evtSource.addEventListener('done', function(e) {
|
||||||
|
evtSource.close();
|
||||||
|
clearInterval(timer);
|
||||||
|
renderSteps(STEPS.length);
|
||||||
|
progressEl.style.width = '100%';
|
||||||
|
const data = JSON.parse(e.data);
|
||||||
|
setTimeout(() => {
|
||||||
|
document.open();
|
||||||
|
document.write(data.html);
|
||||||
|
document.close();
|
||||||
|
}, 400);
|
||||||
|
});
|
||||||
|
|
||||||
|
evtSource.addEventListener('error_msg', function(e) {
|
||||||
|
evtSource.close();
|
||||||
|
clearInterval(timer);
|
||||||
|
const data = JSON.parse(e.data);
|
||||||
|
showError(data.message);
|
||||||
|
});
|
||||||
|
|
||||||
|
evtSource.onerror = function() {
|
||||||
|
evtSource.close();
|
||||||
|
clearInterval(timer);
|
||||||
|
showError('Connection to server lost. The page may be taking too long to load.');
|
||||||
|
};
|
||||||
|
});
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|
||||||
</html>
|
</html>
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def add_base_tag(html_content, original_url):
|
def add_base_tag(html_content, original_url):
|
||||||
soup = BeautifulSoup(html_content, 'html.parser')
|
soup = BeautifulSoup(html_content, 'html.parser')
|
||||||
parsed_url = urlparse(original_url)
|
parsed_url = urlparse(original_url)
|
||||||
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
|
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
|
||||||
|
|
||||||
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
|
|
||||||
if parsed_url.path and not parsed_url.path.endswith('/'):
|
if parsed_url.path and not parsed_url.path.endswith('/'):
|
||||||
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
|
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
|
||||||
base_tag = soup.find('base')
|
base_tag = soup.find('base')
|
||||||
|
|
||||||
print(base_url)
|
|
||||||
if not base_tag:
|
if not base_tag:
|
||||||
new_base_tag = soup.new_tag('base', href=base_url)
|
new_base_tag = soup.new_tag('base', href=base_url)
|
||||||
if soup.head:
|
if soup.head:
|
||||||
@@ -209,19 +427,237 @@ def add_base_tag(html_content, original_url):
|
|||||||
|
|
||||||
return str(soup)
|
return str(soup)
|
||||||
|
|
||||||
def bypass_paywall(url):
|
|
||||||
"""
|
|
||||||
Bypass paywall for a given url
|
|
||||||
"""
|
|
||||||
if url.startswith("http"):
|
|
||||||
response = requests.get(url, headers=googlebot_headers)
|
|
||||||
response.encoding = response.apparent_encoding
|
|
||||||
return add_base_tag(response.text, response.url)
|
|
||||||
|
|
||||||
|
CHALLENGE_SIGNATURES = [
|
||||||
|
"just a moment",
|
||||||
|
"checking your browser",
|
||||||
|
"cloudflare",
|
||||||
|
"cf-challenge",
|
||||||
|
"ddos protection",
|
||||||
|
"attention required",
|
||||||
|
"enable javascript and cookies to continue",
|
||||||
|
"please verify you are a human",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def is_challenge_page(html_text):
|
||||||
|
if not html_text:
|
||||||
|
return False
|
||||||
|
lower = html_text[:8000].lower()
|
||||||
|
hits = sum(1 for sig in CHALLENGE_SIGNATURES if sig in lower)
|
||||||
|
if hits >= 1 and len(html_text) < 20000:
|
||||||
|
return True
|
||||||
|
return hits >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def set_step(job_id, step):
|
||||||
|
if job_id and job_id in jobs:
|
||||||
|
jobs[job_id]['step'] = step
|
||||||
|
|
||||||
|
|
||||||
|
REAL_BROWSER_HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
ARCHIVE_PH_MIRRORS = ["archive.ph", "archive.today", "archive.is", "archive.li"]
|
||||||
|
|
||||||
|
FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"]
|
||||||
|
|
||||||
|
|
||||||
|
def is_medium_url(url):
|
||||||
try:
|
try:
|
||||||
return bypass_paywall("https://" + url)
|
host = (urlparse(url).hostname or "").lower()
|
||||||
|
return host == "medium.com" or host.endswith(".medium.com")
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_via_freedium(url, job_id=None):
|
||||||
|
"""Freedium is a Medium-specific paywall bypass service."""
|
||||||
|
set_step(job_id, 'fallback_freedium')
|
||||||
|
clean_url = url.split('?')[0]
|
||||||
|
for mirror in FREEDIUM_MIRRORS:
|
||||||
|
try:
|
||||||
|
freedium_url = f"https://{mirror}/{clean_url}"
|
||||||
|
resp = requests.get(
|
||||||
|
freedium_url,
|
||||||
|
headers=REAL_BROWSER_HEADERS,
|
||||||
|
timeout=30,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
continue
|
||||||
|
resp.encoding = resp.apparent_encoding
|
||||||
|
text = resp.text
|
||||||
|
if is_challenge_page(text):
|
||||||
|
continue
|
||||||
|
if 'main-content' not in text or len(text) < 2000:
|
||||||
|
continue
|
||||||
|
return text, resp.url
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_via_archive_org(url, job_id=None):
|
||||||
|
set_step(job_id, 'fallback_org')
|
||||||
|
wayback_api = f"https://archive.org/wayback/available?url={url}"
|
||||||
|
try:
|
||||||
|
meta = requests.get(wayback_api, timeout=15).json()
|
||||||
|
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
|
||||||
|
if not snapshot.get("available"):
|
||||||
|
return None, None
|
||||||
|
archived_url = snapshot["url"]
|
||||||
|
if archived_url.startswith("http://"):
|
||||||
|
archived_url = "https://" + archived_url[len("http://"):]
|
||||||
|
if "/web/" in archived_url and "id_/" not in archived_url:
|
||||||
|
archived_url = archived_url.replace("/web/", "/web/", 1)
|
||||||
|
archived_url = archived_url.replace(
|
||||||
|
archived_url.split("/web/")[1].split("/")[0],
|
||||||
|
archived_url.split("/web/")[1].split("/")[0] + "id_",
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
resp = requests.get(archived_url, headers=googlebot_headers, timeout=30)
|
||||||
|
resp.encoding = resp.apparent_encoding
|
||||||
|
return resp.text, resp.url
|
||||||
|
except Exception:
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_via_archive_ph(url, job_id=None):
|
||||||
|
"""Try archive.today / archive.ph / archive.is — all mirrors of the same service."""
|
||||||
|
set_step(job_id, 'fallback_ph')
|
||||||
|
for mirror in ARCHIVE_PH_MIRRORS:
|
||||||
|
try:
|
||||||
|
newest_url = f"https://{mirror}/newest/{url}"
|
||||||
|
resp = requests.get(
|
||||||
|
newest_url,
|
||||||
|
headers=REAL_BROWSER_HEADERS,
|
||||||
|
timeout=20,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
continue
|
||||||
|
|
||||||
|
final = resp.url
|
||||||
|
if not re.search(r"https?://archive\.(ph|today|is|li)/[A-Za-z0-9]{4,}", final):
|
||||||
|
continue
|
||||||
|
if re.search(r"archive\.(ph|today|is|li)/newest/", final):
|
||||||
|
continue
|
||||||
|
|
||||||
|
text = resp.text
|
||||||
|
if is_challenge_page(text):
|
||||||
|
continue
|
||||||
|
|
||||||
|
lower_head = text[:5000].lower()
|
||||||
|
if "no results" in lower_head or "0 captures" in lower_head:
|
||||||
|
continue
|
||||||
|
|
||||||
|
return text, final
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def bypass_paywall(url, job_id=None):
|
||||||
|
set_step(job_id, 'connect')
|
||||||
|
|
||||||
|
if not url.startswith("http"):
|
||||||
|
url = "https://" + url
|
||||||
|
|
||||||
|
medium = is_medium_url(url)
|
||||||
|
html_text = ""
|
||||||
|
final_url = url
|
||||||
|
|
||||||
|
if medium:
|
||||||
|
freedium_html, freedium_final = fetch_via_freedium(url, job_id)
|
||||||
|
if freedium_html:
|
||||||
|
set_step(job_id, 'process')
|
||||||
|
result = add_base_tag(freedium_html, freedium_final)
|
||||||
|
set_step(job_id, 'cleanup')
|
||||||
|
return result
|
||||||
|
else:
|
||||||
|
set_step(job_id, 'fetch')
|
||||||
|
response = requests.get(url, headers=googlebot_headers, timeout=30)
|
||||||
|
response.encoding = response.apparent_encoding
|
||||||
|
html_text = response.text
|
||||||
|
final_url = response.url
|
||||||
|
|
||||||
|
set_step(job_id, 'detect')
|
||||||
|
if not html_text or is_challenge_page(html_text):
|
||||||
|
recovered = False
|
||||||
|
|
||||||
|
if not medium:
|
||||||
|
archived_html, archived_url = fetch_via_archive_org(url, job_id)
|
||||||
|
if archived_html and not is_challenge_page(archived_html):
|
||||||
|
html_text = archived_html
|
||||||
|
final_url = archived_url
|
||||||
|
recovered = True
|
||||||
|
|
||||||
|
if not recovered:
|
||||||
|
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
|
||||||
|
if archived_html and not is_challenge_page(archived_html):
|
||||||
|
html_text = archived_html
|
||||||
|
final_url = archived_url
|
||||||
|
recovered = True
|
||||||
|
else:
|
||||||
|
archived_html, archived_url = fetch_via_archive_org(url, job_id)
|
||||||
|
if archived_html and not is_challenge_page(archived_html):
|
||||||
|
html_text = archived_html
|
||||||
|
final_url = archived_url
|
||||||
|
recovered = True
|
||||||
|
|
||||||
|
if not recovered:
|
||||||
|
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
|
||||||
|
if archived_html and not is_challenge_page(archived_html):
|
||||||
|
html_text = archived_html
|
||||||
|
final_url = archived_url
|
||||||
|
recovered = True
|
||||||
|
|
||||||
|
if not recovered:
|
||||||
|
if medium:
|
||||||
|
msg = (
|
||||||
|
"Medium served an anti-bot challenge and Freedium / archive.org / "
|
||||||
|
"archive.today all came back empty. The article may be too new to "
|
||||||
|
"have been archived yet."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
msg = (
|
||||||
|
"The site served an anti-bot challenge (Cloudflare or similar) "
|
||||||
|
"and no usable snapshot was found on archive.org or archive.today. "
|
||||||
|
"This site is actively blocking automated requests."
|
||||||
|
)
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
set_step(job_id, 'process')
|
||||||
|
result = add_base_tag(html_text, final_url)
|
||||||
|
|
||||||
|
set_step(job_id, 'cleanup')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_worker(job_id, url):
|
||||||
|
try:
|
||||||
|
result = bypass_paywall(url, job_id)
|
||||||
|
jobs[job_id]['result'] = result
|
||||||
|
jobs[job_id]['step'] = 'done'
|
||||||
|
except requests.exceptions.Timeout:
|
||||||
|
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
|
||||||
|
jobs[job_id]['step'] = 'error'
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
|
||||||
|
jobs[job_id]['step'] = 'error'
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
return bypass_paywall("http://" + url)
|
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
|
||||||
|
jobs[job_id]['step'] = 'error'
|
||||||
|
except RuntimeError as e:
|
||||||
|
jobs[job_id]['error'] = str(e)
|
||||||
|
jobs[job_id]['step'] = 'error'
|
||||||
|
except Exception as e:
|
||||||
|
jobs[job_id]['error'] = f'Unexpected error: {e}'
|
||||||
|
jobs[job_id]['step'] = 'error'
|
||||||
|
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
@@ -229,6 +665,45 @@ def main_page():
|
|||||||
return html
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/status/<path:url>")
|
||||||
|
def status_stream(url):
|
||||||
|
import json
|
||||||
|
|
||||||
|
job_id = str(uuid.uuid4())
|
||||||
|
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
|
||||||
|
|
||||||
|
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
|
||||||
|
thread.daemon = True
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
def generate():
|
||||||
|
last_step = None
|
||||||
|
while True:
|
||||||
|
job = jobs.get(job_id)
|
||||||
|
if not job:
|
||||||
|
break
|
||||||
|
|
||||||
|
current = job['step']
|
||||||
|
if current != last_step:
|
||||||
|
last_step = current
|
||||||
|
if current == 'done':
|
||||||
|
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
|
||||||
|
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
|
||||||
|
break
|
||||||
|
elif current == 'error':
|
||||||
|
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
|
||||||
|
|
||||||
|
time.sleep(0.2)
|
||||||
|
|
||||||
|
jobs.pop(job_id, None)
|
||||||
|
|
||||||
|
return Response(generate(), mimetype='text/event-stream',
|
||||||
|
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'})
|
||||||
|
|
||||||
|
|
||||||
@app.route("/article", methods=["POST"])
|
@app.route("/article", methods=["POST"])
|
||||||
def show_article():
|
def show_article():
|
||||||
link = flask.request.form["link"]
|
link = flask.request.form["link"]
|
||||||
@@ -236,8 +711,8 @@ def show_article():
|
|||||||
return bypass_paywall(link)
|
return bypass_paywall(link)
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
return str(e), 400
|
return str(e), 400
|
||||||
except e:
|
except Exception as exc:
|
||||||
raise e
|
raise exc
|
||||||
|
|
||||||
|
|
||||||
@app.route("/", defaults={"path": ""})
|
@app.route("/", defaults={"path": ""})
|
||||||
@@ -251,7 +726,7 @@ def get_article(path):
|
|||||||
return bypass_paywall(actual_url)
|
return bypass_paywall(actual_url)
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
return str(e), 400
|
return str(e), 400
|
||||||
except e:
|
except Exception as e:
|
||||||
raise e
|
raise e
|
||||||
else:
|
else:
|
||||||
return "Invalid URL", 400
|
return "Invalid URL", 400
|
||||||
|
|||||||
Reference in New Issue
Block a user