Remove 3dfilamentprofiles integration (Issue #7)

3dfilamentprofiles.com has Vercel bot protection blocking automated requests.
Their GitHub repo only has sample data (~3 filaments).
SpoolmanDB already has 6,957+ filaments, making this unnecessary.

Removed:
- spoolman/externaldb.py: ExternalSource enum, Filament3DFP model, 3dfp sync
- client/src/utils/queryExternalDB.ts: ExternalSource enum, source field
- client/src/components/filamentImportModal.tsx: Source filter tabs

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

CLAUDE.md

@@ -132,7 +132,7 @@ This is a fork with UX improvements. Issues are tracked on the Gitea instance ab
 | 4 | Layout Max-Width | ✅ DONE |
 | 5 | Temperature Ranges | ✅ DONE |
 | 6 | Spool Adjustment History | ✅ DONE |
-| 7 | 3dfilamentprofiles Integration | ⚠️ PARTIAL (sample only) |
+| 7 | 3dfilamentprofiles Integration | ❌ REMOVED (bot protection) |
 | 8 | (see Gitea #8) | ✅ DONE |
 | 9 | Hierarchical Locations (Room/Bin) | Open |
 | 10 | Database Import/Export | ✅ DONE |
@@ -223,30 +223,10 @@ Frontend:
 - `client/src/pages/spools/show.tsx` - Collapsible adjustment history table
 - `client/public/locales/en/common.json` - Translation keys for history table
 
-### Issue #7 Complete - 3dfilamentprofiles Integration
+### Issue #7 REMOVED - 3dfilamentprofiles Integration
 
-Added 3dfilamentprofiles.com as a second external database source alongside SpoolmanDB.
+**Removed** - 3dfilamentprofiles.com has Vercel bot protection blocking automated requests.
-
+Their GitHub repo only has sample data (~3 filaments). SpoolmanDB has 6,957+ filaments, so this was unnecessary.
-Backend (`spoolman/externaldb.py`):
-- Added `ExternalSource` enum (SPOOLMANDB, FILAMENT_PROFILES_3D)
-- Added `Filament3DFP` model for raw 3dfp data
-- Added `_transform_3dfp_to_external()` to convert 3dfp format to ExternalFilament
-- Updated `_sync()` to fetch from both sources and merge results
-- Environment variables: `EXTERNAL_DB_3DFP_URL`, `EXTERNAL_DB_3DFP_ENABLED`
-
-Frontend:
-- `client/src/utils/queryExternalDB.ts` - Added `ExternalSource` enum, `source` field, temp range fields
-- `client/src/components/filamentImportModal.tsx` - Added tabs to filter by source (All/SpoolmanDB/3DFP)
-- `client/src/pages/filaments/create.tsx` - Updated import handler for temp range fields
-
-Data mapping (3dfp → Spoolman):
-- `brand_name` → `manufacturer`
-- `color` + `material_type` → `name`
-- `rgb` → `color_hex` (strip # prefix)
-- `properties.temp_min/max` → `extruder_temp/extruder_temp_max`
-- `properties.bed_temp_min/max` → `bed_temp/bed_temp_max`
-- Diameter defaults to 1.75mm (3dfp doesn't provide)
-- Density from material lookup table
 
 ### Issue #10 Complete - Database Import/Export
 
@@ -373,11 +353,8 @@ Some filaments have temp info in comment field (e.g., "extrude 190-230 / bed 50-
 ### Filament Select Not Refreshing After Inline Creation
 When creating a filament via the inline modal on spool create page, the select dropdown doesn't immediately show the new filament with its name (shows ID instead).
 
-### 3dfilamentprofiles Integration Incomplete (Issue #7)
+### Issue #7 - 3dfilamentprofiles Integration REMOVED
-The backend code and frontend UI exist, but we're only fetching `sample-filaments.json` (~3 test filaments). The jklewa/filament-profiles-data GitHub repo only provides sample data. Full dataset requires either:
+The 3dfilamentprofiles.com website has Vercel bot protection that blocks automated requests. Their GitHub repo (jklewa/filament-profiles-data) only provides sample data (~3 filaments). Since SpoolmanDB already has 6,957+ filaments from major brands, the 3dfp integration was removed rather than maintaining dead code.
-1. Running their parser.py script against 3dfilamentprofiles.com
-2. Directly querying the website API
-3. Hosting our own generated full dataset
 
 ### Future: Issue #9 - Hierarchical Locations
 Add room/bin/shelf support for locations. See Gitea issue #9 for details.

client/src/components/filamentImportModal.tsx

@@ -1,10 +1,10 @@
 import { useTranslate } from "@refinedev/core";
-import { Form, Modal, Select, Tabs } from "antd";
+import { Form, Modal, Select } from "antd";
-import { useMemo, useState } from "react";
+import { useMemo } from "react";
 import { Trans } from "react-i18next";
 import { formatFilamentLabel } from "../pages/spools/functions";
 import { searchMatches } from "../utils/filtering";
-import { ExternalFilament, ExternalSource, useGetExternalDBFilaments } from "../utils/queryExternalDB";
+import { ExternalFilament, useGetExternalDBFilaments } from "../utils/queryExternalDB";
 
 export function FilamentImportModal(props: {
   isOpen: boolean;
@@ -13,25 +13,14 @@ export function FilamentImportModal(props: {
 }) {
   const [form] = Form.useForm();
   const t = useTranslate();
-  const [selectedSource, setSelectedSource] = useState<ExternalSource | "all">("all");
 
   const externalFilaments = useGetExternalDBFilaments();
 
-  // Filter and format filaments based on selected source
+  // Format filaments for the select dropdown
   const filamentOptions = useMemo(() => {
-    const filtered = externalFilaments.data?.filter((item) => {
+    const options = (externalFilaments.data ?? []).map((item) => {
-      if (selectedSource === "all") return true;
-      return item.source === selectedSource;
-    }) ?? [];
-
-    const options = filtered.map((item) => {
-      // Add source indicator to label if showing all
-      const sourceIndicator = selectedSource === "all" && item.source === ExternalSource.FILAMENT_PROFILES_3D
-        ? "[3DFP] "
-        : "";
-
       return {
-        label: sourceIndicator + formatFilamentLabel(
+        label: formatFilamentLabel(
           item.name,
           item.diameter,
           item.manufacturer,
@@ -46,20 +35,8 @@ export function FilamentImportModal(props: {
 
     options.sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: "base" }));
     return options;
-  }, [externalFilaments.data, selectedSource]);
-
-  // Count filaments by source
-  const counts = useMemo(() => {
-    const spoolmandb = externalFilaments.data?.filter(f => f.source === ExternalSource.SPOOLMANDB || !f.source).length ?? 0;
-    const threeDFP = externalFilaments.data?.filter(f => f.source === ExternalSource.FILAMENT_PROFILES_3D).length ?? 0;
-    return { spoolmandb, threeDFP, total: spoolmandb + threeDFP };
   }, [externalFilaments.data]);
 
-  const handleSourceChange = (key: string) => {
-    setSelectedSource(key as ExternalSource | "all");
-    form.resetFields(["filament"]);
-  };
-
   return (
     <Modal
       title={t("filament.form.import_external")}
@@ -77,25 +54,6 @@ export function FilamentImportModal(props: {
         />
       </p>
 
-      <Tabs
-        activeKey={selectedSource}
-        onChange={handleSourceChange}
-        items={[
-          {
-            key: "all",
-            label: `${t("filament.form.all_sources")} (${counts.total})`,
-          },
-          {
-            key: ExternalSource.SPOOLMANDB,
-            label: `SpoolmanDB (${counts.spoolmandb})`,
-          },
-          {
-            key: ExternalSource.FILAMENT_PROFILES_3D,
-            label: `3D Filament Profiles (${counts.threeDFP})`,
-          },
-        ]}
-      />
-
       <Form
         layout="vertical"
         form={form}
@@ -107,7 +65,6 @@ export function FilamentImportModal(props: {
           props.onImport(filament);
           props.onClose();
           form.resetFields();
-          setSelectedSource("all");
         }}
       >
         <Form.Item name="filament" rules={[{ required: true }]}>

client/src/utils/queryExternalDB.ts

@@ -22,11 +22,6 @@ export enum Pattern {
   SPARKLE = "sparkle",
 }
 
-export enum ExternalSource {
-  SPOOLMANDB = "spoolmandb",
-  FILAMENT_PROFILES_3D = "3dfilamentprofiles",
-}
-
 export interface ExternalFilament {
   id: string;
   manufacturer: string;
@@ -48,7 +43,6 @@ export interface ExternalFilament {
   pattern?: Pattern;
   translucent: boolean;
   glow: boolean;
-  source?: ExternalSource;
 }
 
 export interface ExternalMaterial {

spoolman/externaldb.py

@@ -6,7 +6,7 @@ import os
 from collections.abc import Iterator
 from enum import Enum
 from pathlib import Path
-from typing import Any, Optional
+from typing import Optional
 from urllib.parse import urljoin
 
 import hishel
@@ -20,16 +20,8 @@ logger = logging.getLogger(__name__)
 
 
 DEFAULT_EXTERNAL_DB_URL = "https://donkie.github.io/SpoolmanDB/"
-DEFAULT_3DFP_URL = "https://raw.githubusercontent.com/jklewa/filament-profiles-data/main/"
 DEFAULT_SYNC_INTERVAL = 3600
 
-
-class ExternalSource(Enum):
-    """External database sources."""
-
-    SPOOLMANDB = "spoolmandb"
-    FILAMENT_PROFILES_3D = "3dfilamentprofiles"
-
 controller = hishel.Controller(allow_stale=True)
 try:
     cache_path = get_cache_dir() / "hishel"
@@ -97,10 +89,6 @@ class ExternalFilament(BaseModel):
     pattern: Optional[Pattern] = Field(default=None, description="Pattern of the filament.", examples=[Pattern.MARBLE])
     translucent: bool = Field(default=False, description="Whether the filament is translucent.")
     glow: bool = Field(default=False, description="Whether the filament is glow-in-the-dark.")
-    source: ExternalSource = Field(
-        default=ExternalSource.SPOOLMANDB,
-        description="External database source this filament came from.",
-    )
 
 
 class ExternalFilamentsFile(RootModel):
@@ -134,146 +122,11 @@ class ExternalMaterialsFile(RootModel):
         return self.root[index]
 
 
-# 3dfilamentprofiles.com data models
-class FilamentProperties3DFP(BaseModel):
-    """Properties object from 3dfilamentprofiles."""
-
-    temp_min: Optional[int] = None
-    temp_max: Optional[int] = None
-    bed_temp_min: Optional[int] = None
-    bed_temp_max: Optional[int] = None
-    spool_weight: Optional[float] = None
-    flow_ratio: Optional[float] = None
-
-
-class Filament3DFP(BaseModel):
-    """Raw filament data from 3dfilamentprofiles.com."""
-
-    id: int
-    brand_name: str
-    material: str
-    material_type: Optional[str] = None
-    color: str
-    rgb: Optional[str] = None
-    properties: Optional[dict[str, Any]] = None
-
-
-class FilamentsFile3DFP(BaseModel):
-    """The filaments.json file from 3dfilamentprofiles."""
-
-    filaments: list[Filament3DFP]
-
-
-# Material density defaults (used when 3dfilamentprofiles doesn't provide density)
-MATERIAL_DENSITIES: dict[str, float] = {
-    "PLA": 1.24,
-    "ABS": 1.04,
-    "PETG": 1.27,
-    "NYLON": 1.52,
-    "TPU": 1.21,
-    "PC": 1.3,
-    "WOOD": 1.28,
-    "CF": 1.3,
-    "PC/ABS": 1.19,
-    "HIPS": 1.03,
-    "PVA": 1.23,
-    "ASA": 1.05,
-    "PP": 0.9,
-    "POM": 1.4,
-    "PMMA": 1.18,
-    "FPE": 2.16,
-}
-
-
-def _get_density_for_material(material: str) -> float:
-    """Get density for a material, defaulting to PLA's density if unknown."""
-    material_upper = material.upper()
-    for key, density in MATERIAL_DENSITIES.items():
-        if key in material_upper:
-            return density
-    return 1.24  # Default to PLA density
-
-
-def _transform_3dfp_to_external(filament: Filament3DFP) -> ExternalFilament:
-    """Transform a 3dfilamentprofiles filament to ExternalFilament format."""
-    # Build name from material_type and color
-    name_parts = []
-    if filament.material_type:
-        name_parts.append(filament.material_type)
-    name_parts.append(filament.color)
-    name = " ".join(name_parts)
-
-    # Handle color hex (3dfp includes # prefix, SpoolmanDB doesn't)
-    color_hex = None
-    if filament.rgb:
-        color_hex = filament.rgb.lstrip("#").lower()
-
-    # Extract properties
-    props = filament.properties or {}
-    extruder_temp = props.get("temp_min")
-    extruder_temp_max = props.get("temp_max")
-    bed_temp = props.get("bed_temp_min")
-    bed_temp_max = props.get("bed_temp_max")
-    spool_weight = props.get("spool_weight")
-
-    # Determine finish from material_type
-    finish = None
-    if filament.material_type:
-        material_type_lower = filament.material_type.lower()
-        if "matte" in material_type_lower:
-            finish = Finish.MATTE
-        elif "glossy" in material_type_lower or "silk" in material_type_lower:
-            finish = Finish.GLOSSY
-
-    # Determine pattern from material_type
-    pattern = None
-    if filament.material_type:
-        material_type_lower = filament.material_type.lower()
-        if "marble" in material_type_lower:
-            pattern = Pattern.MARBLE
-        elif "sparkle" in material_type_lower or "glitter" in material_type_lower:
-            pattern = Pattern.SPARKLE
-
-    return ExternalFilament(
-        id=f"3dfp_{filament.id}",
-        manufacturer=filament.brand_name,
-        name=name,
-        material=filament.material,
-        density=_get_density_for_material(filament.material),
-        weight=None,  # 3dfp doesn't provide filament weight
-        spool_weight=spool_weight,
-        spool_type=None,
-        diameter=1.75,  # Default, 3dfp doesn't provide diameter
-        color_hex=color_hex,
-        color_hexes=None,
-        extruder_temp=extruder_temp,
-        extruder_temp_max=extruder_temp_max,
-        bed_temp=bed_temp,
-        bed_temp_max=bed_temp_max,
-        finish=finish,
-        multi_color_direction=None,
-        pattern=pattern,
-        translucent=False,
-        glow=False,
-        source=ExternalSource.FILAMENT_PROFILES_3D,
-    )
-
-
 def get_external_db_url() -> str:
     """Get the external database URL from environment variables. Defaults to DEFAULT_EXTERNAL_DB_URL."""
     return os.getenv("EXTERNAL_DB_URL", DEFAULT_EXTERNAL_DB_URL)
 
 
-def get_3dfp_url() -> str:
-    """Get the 3dfilamentprofiles URL from environment variables."""
-    return os.getenv("EXTERNAL_DB_3DFP_URL", DEFAULT_3DFP_URL)
-
-
-def is_3dfp_enabled() -> bool:
-    """Check if 3dfilamentprofiles sync is enabled."""
-    return os.getenv("EXTERNAL_DB_3DFP_ENABLED", "true").lower() in ("true", "1", "yes")
-
-
 def get_external_db_sync_interval() -> int:
     """Get the external database sync interval from environment variables. Defaults to DEFAULT_SYNC_INTERVAL."""
     return int(os.getenv("EXTERNAL_DB_SYNC_INTERVAL", DEFAULT_SYNC_INTERVAL))
@@ -315,70 +168,36 @@ def get_materials_file() -> Path:
     return filecache.get_file("materials.json")
 
 
-def _parse_3dfp_filaments(data: bytes) -> list[Filament3DFP]:
-    """Parse 3dfilamentprofiles JSON data."""
-    try:
-        parsed = FilamentsFile3DFP.model_validate_json(data)
-        return parsed.filaments
-    except Exception:
-        # Fallback: try parsing as raw list
-        import json
-
-        raw = json.loads(data)
-        if isinstance(raw, dict) and "filaments" in raw:
-            return [Filament3DFP.model_validate(f) for f in raw["filaments"]]
-        return []
-
-
 async def _sync() -> None:
-    logger.info("Syncing external DB.")
+    """Sync filaments and materials from SpoolmanDB."""
+    logger.info("Syncing external DB (SpoolmanDB).")
 
-    all_filaments: list[ExternalFilament] = []
-
-    # Sync SpoolmanDB
     spoolmandb_url = get_external_db_url()
-    if spoolmandb_url.strip():
+    if not spoolmandb_url.strip():
+        logger.warning("External DB URL is empty. Skipping sync.")
+        return
+
+    # Sync filaments
     try:
-            spoolmandb_filaments = _parse_filaments_from_bytes(
+        filaments = _parse_filaments_from_bytes(
             await _download_file(urljoin(spoolmandb_url, "filaments.json"))
         )
-            # Mark all SpoolmanDB filaments with source
+        _write_to_local_cache("filaments.json", filaments.model_dump_json().encode())
-            for filament in spoolmandb_filaments.root:
+        logger.info("SpoolmanDB filaments synced: %d filaments", len(filaments.root))
-                filament.source = ExternalSource.SPOOLMANDB
-            all_filaments.extend(spoolmandb_filaments.root)
-            logger.info("SpoolmanDB synced: %d filaments", len(spoolmandb_filaments.root))
     except Exception as e:
-            logger.warning("Failed to sync SpoolmanDB: %s", e)
+        logger.warning("Failed to sync SpoolmanDB filaments: %s", e)
 
-    # Sync 3dfilamentprofiles
+    # Sync materials
-    if is_3dfp_enabled():
-        url_3dfp = get_3dfp_url()
-        if url_3dfp.strip():
-            try:
-                raw_data = await _download_file(urljoin(url_3dfp, "sample-filaments.json"))
-                filaments_3dfp = _parse_3dfp_filaments(raw_data)
-                transformed = [_transform_3dfp_to_external(f) for f in filaments_3dfp]
-                all_filaments.extend(transformed)
-                logger.info("3dfilamentprofiles synced: %d filaments", len(transformed))
-            except Exception as e:
-                logger.warning("Failed to sync 3dfilamentprofiles: %s", e)
-
-    # Write combined filaments
-    combined = ExternalFilamentsFile(root=all_filaments)
-    _write_to_local_cache("filaments.json", combined.model_dump_json().encode())
-
-    # Materials only from SpoolmanDB
-    if spoolmandb_url.strip():
     try:
         materials = _parse_materials_from_bytes(
             await _download_file(urljoin(spoolmandb_url, "materials.json"))
         )
         _write_to_local_cache("materials.json", materials.model_dump_json().encode())
-            logger.info("Materials synced: %d", len(materials.root))
+        logger.info("SpoolmanDB materials synced: %d materials", len(materials.root))
     except Exception as e:
-            logger.warning("Failed to sync materials: %s", e)
+        logger.warning("Failed to sync SpoolmanDB materials: %s", e)
 
-    logger.info("External DB sync complete. Total filaments: %d", len(all_filaments))
+    logger.info("External DB sync complete.")
 
 
 def schedule_tasks(scheduler: Scheduler) -> None: