Fix Freedium fallback for the current mirror layout
Build / Build and release (push) Failing after 5m53s

freedium.cfd no longer resolves, and freedium-mirror.cfd has been
rewritten as a SvelteKit app that renders the post inside <main> rather
than the old <div id="main-content">. The content check therefore
rejected every mirror response, so Medium articles always fell through
to the archive fallbacks.

Accept either marker, and reject the mirror's error pages explicitly:
they return HTTP 200 with a streamed error payload and ship an empty
<article> skeleton, so <article> alone does not prove a post rendered.

Also fix an unrelated crash in process_html_document that blocked this
path (and every page with an inline script): get_text() has no "default"
keyword, so it raised TypeError.

Closes #49
This commit is contained in:
Wasi Master
2026-09-18 03:29:28 +06:00
parent d7505d639a
commit d03b120c41
+21 -2
View File
@@ -637,7 +637,7 @@ def process_html_document(html_content, original_url):
# Strip anti-hotlinking, anti-embedding, and frame-busting scripts. # Strip anti-hotlinking, anti-embedding, and frame-busting scripts.
for script in soup.find_all("script", src=False): for script in soup.find_all("script", src=False):
content = script.get_text(default="") content = script.get_text()
# Target the specific hostname check # Target the specific hostname check
if "location.hostname" in content and ("location.href" in content or "location.replace" in content): if "location.hostname" in content and ("location.href" in content or "location.replace" in content):
@@ -726,6 +726,25 @@ ARCHIVE_PH_MIRRORS = ["archive.ph", "archive.today", "archive.is", "archive.li"]
FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"] FREEDIUM_MIRRORS = ["freedium-mirror.cfd", "freedium.cfd"]
# The original Freedium layout wrapped the post in <div id="main-content">, but
# the current mirror renders it inside a <main> element instead. Its error pages
# (e.g. "Article not found") still ship an empty <article> skeleton, so <article>
# on its own is not proof that the post was rendered.
FREEDIUM_CONTENT_MARKERS = ("main-content", "<main")
# SvelteKit streams the failure into the page body instead of using a non-200
# status, so the machine-readable error code is what marks a miss.
FREEDIUM_ERROR_MARKERS = ("ARTICLE_NOT_FOUND",)
def freedium_has_article(html_text):
"""True when a Freedium response actually contains the rendered post."""
if not html_text or len(html_text) < 2000:
return False
if any(marker in html_text for marker in FREEDIUM_ERROR_MARKERS):
return False
return any(marker in html_text for marker in FREEDIUM_CONTENT_MARKERS)
def is_medium_url(url): def is_medium_url(url):
try: try:
@@ -754,7 +773,7 @@ def fetch_via_freedium(url, job_id=None):
text = resp.text text = resp.text
if is_challenge_page(text): if is_challenge_page(text):
continue continue
if 'main-content' not in text or len(text) < 2000: if not freedium_has_article(text):
continue continue
return text, resp.url return text, resp.url
except Exception: except Exception: