diff --git a/CLAUDE.md b/CLAUDE.md index b398cc1..45cf9ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,7 +132,7 @@ This is a fork with UX improvements. Issues are tracked on the Gitea instance ab | 4 | Layout Max-Width | ✅ DONE | | 5 | Temperature Ranges | ✅ DONE | | 6 | Spool Adjustment History | ✅ DONE | -| 7 | 3dfilamentprofiles Integration | ⚠️ PARTIAL (sample only) | +| 7 | 3dfilamentprofiles Integration | ❌ REMOVED (bot protection) | | 8 | (see Gitea #8) | ✅ DONE | | 9 | Hierarchical Locations (Room/Bin) | Open | | 10 | Database Import/Export | ✅ DONE | @@ -223,30 +223,10 @@ Frontend: - `client/src/pages/spools/show.tsx` - Collapsible adjustment history table - `client/public/locales/en/common.json` - Translation keys for history table -### Issue #7 Complete - 3dfilamentprofiles Integration +### Issue #7 REMOVED - 3dfilamentprofiles Integration -Added 3dfilamentprofiles.com as a second external database source alongside SpoolmanDB. - -Backend (`spoolman/externaldb.py`): -- Added `ExternalSource` enum (SPOOLMANDB, FILAMENT_PROFILES_3D) -- Added `Filament3DFP` model for raw 3dfp data -- Added `_transform_3dfp_to_external()` to convert 3dfp format to ExternalFilament -- Updated `_sync()` to fetch from both sources and merge results -- Environment variables: `EXTERNAL_DB_3DFP_URL`, `EXTERNAL_DB_3DFP_ENABLED` - -Frontend: -- `client/src/utils/queryExternalDB.ts` - Added `ExternalSource` enum, `source` field, temp range fields -- `client/src/components/filamentImportModal.tsx` - Added tabs to filter by source (All/SpoolmanDB/3DFP) -- `client/src/pages/filaments/create.tsx` - Updated import handler for temp range fields - -Data mapping (3dfp → Spoolman): -- `brand_name` → `manufacturer` -- `color` + `material_type` → `name` -- `rgb` → `color_hex` (strip # prefix) -- `properties.temp_min/max` → `extruder_temp/extruder_temp_max` -- `properties.bed_temp_min/max` → `bed_temp/bed_temp_max` -- Diameter defaults to 1.75mm (3dfp doesn't provide) -- Density from material lookup table +**Removed** - 3dfilamentprofiles.com has Vercel bot protection blocking automated requests. +Their GitHub repo only has sample data (~3 filaments). SpoolmanDB has 6,957+ filaments, so this was unnecessary. ### Issue #10 Complete - Database Import/Export @@ -373,11 +353,8 @@ Some filaments have temp info in comment field (e.g., "extrude 190-230 / bed 50- ### Filament Select Not Refreshing After Inline Creation When creating a filament via the inline modal on spool create page, the select dropdown doesn't immediately show the new filament with its name (shows ID instead). -### 3dfilamentprofiles Integration Incomplete (Issue #7) -The backend code and frontend UI exist, but we're only fetching `sample-filaments.json` (~3 test filaments). The jklewa/filament-profiles-data GitHub repo only provides sample data. Full dataset requires either: -1. Running their parser.py script against 3dfilamentprofiles.com -2. Directly querying the website API -3. Hosting our own generated full dataset +### Issue #7 - 3dfilamentprofiles Integration REMOVED +The 3dfilamentprofiles.com website has Vercel bot protection that blocks automated requests. Their GitHub repo (jklewa/filament-profiles-data) only provides sample data (~3 filaments). Since SpoolmanDB already has 6,957+ filaments from major brands, the 3dfp integration was removed rather than maintaining dead code. ### Future: Issue #9 - Hierarchical Locations Add room/bin/shelf support for locations. See Gitea issue #9 for details. diff --git a/client/src/components/filamentImportModal.tsx b/client/src/components/filamentImportModal.tsx index ff9bf6b..4fdf27d 100644 --- a/client/src/components/filamentImportModal.tsx +++ b/client/src/components/filamentImportModal.tsx @@ -1,10 +1,10 @@ import { useTranslate } from "@refinedev/core"; -import { Form, Modal, Select, Tabs } from "antd"; -import { useMemo, useState } from "react"; +import { Form, Modal, Select } from "antd"; +import { useMemo } from "react"; import { Trans } from "react-i18next"; import { formatFilamentLabel } from "../pages/spools/functions"; import { searchMatches } from "../utils/filtering"; -import { ExternalFilament, ExternalSource, useGetExternalDBFilaments } from "../utils/queryExternalDB"; +import { ExternalFilament, useGetExternalDBFilaments } from "../utils/queryExternalDB"; export function FilamentImportModal(props: { isOpen: boolean; @@ -13,25 +13,14 @@ export function FilamentImportModal(props: { }) { const [form] = Form.useForm(); const t = useTranslate(); - const [selectedSource, setSelectedSource] = useState("all"); const externalFilaments = useGetExternalDBFilaments(); - // Filter and format filaments based on selected source + // Format filaments for the select dropdown const filamentOptions = useMemo(() => { - const filtered = externalFilaments.data?.filter((item) => { - if (selectedSource === "all") return true; - return item.source === selectedSource; - }) ?? []; - - const options = filtered.map((item) => { - // Add source indicator to label if showing all - const sourceIndicator = selectedSource === "all" && item.source === ExternalSource.FILAMENT_PROFILES_3D - ? "[3DFP] " - : ""; - + const options = (externalFilaments.data ?? []).map((item) => { return { - label: sourceIndicator + formatFilamentLabel( + label: formatFilamentLabel( item.name, item.diameter, item.manufacturer, @@ -46,20 +35,8 @@ export function FilamentImportModal(props: { options.sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: "base" })); return options; - }, [externalFilaments.data, selectedSource]); - - // Count filaments by source - const counts = useMemo(() => { - const spoolmandb = externalFilaments.data?.filter(f => f.source === ExternalSource.SPOOLMANDB || !f.source).length ?? 0; - const threeDFP = externalFilaments.data?.filter(f => f.source === ExternalSource.FILAMENT_PROFILES_3D).length ?? 0; - return { spoolmandb, threeDFP, total: spoolmandb + threeDFP }; }, [externalFilaments.data]); - const handleSourceChange = (key: string) => { - setSelectedSource(key as ExternalSource | "all"); - form.resetFields(["filament"]); - }; - return (

- -
diff --git a/client/src/utils/queryExternalDB.ts b/client/src/utils/queryExternalDB.ts index 1401d42..ed87075 100644 --- a/client/src/utils/queryExternalDB.ts +++ b/client/src/utils/queryExternalDB.ts @@ -22,11 +22,6 @@ export enum Pattern { SPARKLE = "sparkle", } -export enum ExternalSource { - SPOOLMANDB = "spoolmandb", - FILAMENT_PROFILES_3D = "3dfilamentprofiles", -} - export interface ExternalFilament { id: string; manufacturer: string; @@ -48,7 +43,6 @@ export interface ExternalFilament { pattern?: Pattern; translucent: boolean; glow: boolean; - source?: ExternalSource; } export interface ExternalMaterial { diff --git a/spoolman/externaldb.py b/spoolman/externaldb.py index 6d04629..9217218 100644 --- a/spoolman/externaldb.py +++ b/spoolman/externaldb.py @@ -6,7 +6,7 @@ import os from collections.abc import Iterator from enum import Enum from pathlib import Path -from typing import Any, Optional +from typing import Optional from urllib.parse import urljoin import hishel @@ -20,16 +20,8 @@ logger = logging.getLogger(__name__) DEFAULT_EXTERNAL_DB_URL = "https://donkie.github.io/SpoolmanDB/" -DEFAULT_3DFP_URL = "https://raw.githubusercontent.com/jklewa/filament-profiles-data/main/" DEFAULT_SYNC_INTERVAL = 3600 - -class ExternalSource(Enum): - """External database sources.""" - - SPOOLMANDB = "spoolmandb" - FILAMENT_PROFILES_3D = "3dfilamentprofiles" - controller = hishel.Controller(allow_stale=True) try: cache_path = get_cache_dir() / "hishel" @@ -97,10 +89,6 @@ class ExternalFilament(BaseModel): pattern: Optional[Pattern] = Field(default=None, description="Pattern of the filament.", examples=[Pattern.MARBLE]) translucent: bool = Field(default=False, description="Whether the filament is translucent.") glow: bool = Field(default=False, description="Whether the filament is glow-in-the-dark.") - source: ExternalSource = Field( - default=ExternalSource.SPOOLMANDB, - description="External database source this filament came from.", - ) class ExternalFilamentsFile(RootModel): @@ -134,146 +122,11 @@ class ExternalMaterialsFile(RootModel): return self.root[index] -# 3dfilamentprofiles.com data models -class FilamentProperties3DFP(BaseModel): - """Properties object from 3dfilamentprofiles.""" - - temp_min: Optional[int] = None - temp_max: Optional[int] = None - bed_temp_min: Optional[int] = None - bed_temp_max: Optional[int] = None - spool_weight: Optional[float] = None - flow_ratio: Optional[float] = None - - -class Filament3DFP(BaseModel): - """Raw filament data from 3dfilamentprofiles.com.""" - - id: int - brand_name: str - material: str - material_type: Optional[str] = None - color: str - rgb: Optional[str] = None - properties: Optional[dict[str, Any]] = None - - -class FilamentsFile3DFP(BaseModel): - """The filaments.json file from 3dfilamentprofiles.""" - - filaments: list[Filament3DFP] - - -# Material density defaults (used when 3dfilamentprofiles doesn't provide density) -MATERIAL_DENSITIES: dict[str, float] = { - "PLA": 1.24, - "ABS": 1.04, - "PETG": 1.27, - "NYLON": 1.52, - "TPU": 1.21, - "PC": 1.3, - "WOOD": 1.28, - "CF": 1.3, - "PC/ABS": 1.19, - "HIPS": 1.03, - "PVA": 1.23, - "ASA": 1.05, - "PP": 0.9, - "POM": 1.4, - "PMMA": 1.18, - "FPE": 2.16, -} - - -def _get_density_for_material(material: str) -> float: - """Get density for a material, defaulting to PLA's density if unknown.""" - material_upper = material.upper() - for key, density in MATERIAL_DENSITIES.items(): - if key in material_upper: - return density - return 1.24 # Default to PLA density - - -def _transform_3dfp_to_external(filament: Filament3DFP) -> ExternalFilament: - """Transform a 3dfilamentprofiles filament to ExternalFilament format.""" - # Build name from material_type and color - name_parts = [] - if filament.material_type: - name_parts.append(filament.material_type) - name_parts.append(filament.color) - name = " ".join(name_parts) - - # Handle color hex (3dfp includes # prefix, SpoolmanDB doesn't) - color_hex = None - if filament.rgb: - color_hex = filament.rgb.lstrip("#").lower() - - # Extract properties - props = filament.properties or {} - extruder_temp = props.get("temp_min") - extruder_temp_max = props.get("temp_max") - bed_temp = props.get("bed_temp_min") - bed_temp_max = props.get("bed_temp_max") - spool_weight = props.get("spool_weight") - - # Determine finish from material_type - finish = None - if filament.material_type: - material_type_lower = filament.material_type.lower() - if "matte" in material_type_lower: - finish = Finish.MATTE - elif "glossy" in material_type_lower or "silk" in material_type_lower: - finish = Finish.GLOSSY - - # Determine pattern from material_type - pattern = None - if filament.material_type: - material_type_lower = filament.material_type.lower() - if "marble" in material_type_lower: - pattern = Pattern.MARBLE - elif "sparkle" in material_type_lower or "glitter" in material_type_lower: - pattern = Pattern.SPARKLE - - return ExternalFilament( - id=f"3dfp_{filament.id}", - manufacturer=filament.brand_name, - name=name, - material=filament.material, - density=_get_density_for_material(filament.material), - weight=None, # 3dfp doesn't provide filament weight - spool_weight=spool_weight, - spool_type=None, - diameter=1.75, # Default, 3dfp doesn't provide diameter - color_hex=color_hex, - color_hexes=None, - extruder_temp=extruder_temp, - extruder_temp_max=extruder_temp_max, - bed_temp=bed_temp, - bed_temp_max=bed_temp_max, - finish=finish, - multi_color_direction=None, - pattern=pattern, - translucent=False, - glow=False, - source=ExternalSource.FILAMENT_PROFILES_3D, - ) - - def get_external_db_url() -> str: """Get the external database URL from environment variables. Defaults to DEFAULT_EXTERNAL_DB_URL.""" return os.getenv("EXTERNAL_DB_URL", DEFAULT_EXTERNAL_DB_URL) -def get_3dfp_url() -> str: - """Get the 3dfilamentprofiles URL from environment variables.""" - return os.getenv("EXTERNAL_DB_3DFP_URL", DEFAULT_3DFP_URL) - - -def is_3dfp_enabled() -> bool: - """Check if 3dfilamentprofiles sync is enabled.""" - return os.getenv("EXTERNAL_DB_3DFP_ENABLED", "true").lower() in ("true", "1", "yes") - - def get_external_db_sync_interval() -> int: """Get the external database sync interval from environment variables. Defaults to DEFAULT_SYNC_INTERVAL.""" return int(os.getenv("EXTERNAL_DB_SYNC_INTERVAL", DEFAULT_SYNC_INTERVAL)) @@ -315,70 +168,36 @@ def get_materials_file() -> Path: return filecache.get_file("materials.json") -def _parse_3dfp_filaments(data: bytes) -> list[Filament3DFP]: - """Parse 3dfilamentprofiles JSON data.""" - try: - parsed = FilamentsFile3DFP.model_validate_json(data) - return parsed.filaments - except Exception: - # Fallback: try parsing as raw list - import json - - raw = json.loads(data) - if isinstance(raw, dict) and "filaments" in raw: - return [Filament3DFP.model_validate(f) for f in raw["filaments"]] - return [] - - async def _sync() -> None: - logger.info("Syncing external DB.") + """Sync filaments and materials from SpoolmanDB.""" + logger.info("Syncing external DB (SpoolmanDB).") - all_filaments: list[ExternalFilament] = [] - - # Sync SpoolmanDB spoolmandb_url = get_external_db_url() - if spoolmandb_url.strip(): - try: - spoolmandb_filaments = _parse_filaments_from_bytes( - await _download_file(urljoin(spoolmandb_url, "filaments.json")) - ) - # Mark all SpoolmanDB filaments with source - for filament in spoolmandb_filaments.root: - filament.source = ExternalSource.SPOOLMANDB - all_filaments.extend(spoolmandb_filaments.root) - logger.info("SpoolmanDB synced: %d filaments", len(spoolmandb_filaments.root)) - except Exception as e: - logger.warning("Failed to sync SpoolmanDB: %s", e) + if not spoolmandb_url.strip(): + logger.warning("External DB URL is empty. Skipping sync.") + return - # Sync 3dfilamentprofiles - if is_3dfp_enabled(): - url_3dfp = get_3dfp_url() - if url_3dfp.strip(): - try: - raw_data = await _download_file(urljoin(url_3dfp, "sample-filaments.json")) - filaments_3dfp = _parse_3dfp_filaments(raw_data) - transformed = [_transform_3dfp_to_external(f) for f in filaments_3dfp] - all_filaments.extend(transformed) - logger.info("3dfilamentprofiles synced: %d filaments", len(transformed)) - except Exception as e: - logger.warning("Failed to sync 3dfilamentprofiles: %s", e) + # Sync filaments + try: + filaments = _parse_filaments_from_bytes( + await _download_file(urljoin(spoolmandb_url, "filaments.json")) + ) + _write_to_local_cache("filaments.json", filaments.model_dump_json().encode()) + logger.info("SpoolmanDB filaments synced: %d filaments", len(filaments.root)) + except Exception as e: + logger.warning("Failed to sync SpoolmanDB filaments: %s", e) - # Write combined filaments - combined = ExternalFilamentsFile(root=all_filaments) - _write_to_local_cache("filaments.json", combined.model_dump_json().encode()) + # Sync materials + try: + materials = _parse_materials_from_bytes( + await _download_file(urljoin(spoolmandb_url, "materials.json")) + ) + _write_to_local_cache("materials.json", materials.model_dump_json().encode()) + logger.info("SpoolmanDB materials synced: %d materials", len(materials.root)) + except Exception as e: + logger.warning("Failed to sync SpoolmanDB materials: %s", e) - # Materials only from SpoolmanDB - if spoolmandb_url.strip(): - try: - materials = _parse_materials_from_bytes( - await _download_file(urljoin(spoolmandb_url, "materials.json")) - ) - _write_to_local_cache("materials.json", materials.model_dump_json().encode()) - logger.info("Materials synced: %d", len(materials.root)) - except Exception as e: - logger.warning("Failed to sync materials: %s", e) - - logger.info("External DB sync complete. Total filaments: %d", len(all_filaments)) + logger.info("External DB sync complete.") def schedule_tasks(scheduler: Scheduler) -> None: