Merge branch 'master' into feature/spool_revisions

.env.example

@@ -34,3 +34,8 @@
 # Host and port to listen on
 SPOOLMAN_HOST=0.0.0.0
 SPOOLMAN_PORT=7912
+
+
+# Enable Collect Prometheus metrics at database
+# Default: FALSE
+#SPOOLMAN_METRICS_ENABLED=TRUE

README.md

@@ -115,7 +115,7 @@ These are either set in the .env file if you use the standalone installation, or
 If you want to connect with an external database instead, specify the `SPOOLMAN_DB_*` environment variables from the table below.
 
 | Variable                  | Description                                                                                                                  |
-| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+|---------------------------|------------------------------------------------------------------------------------------------------------------------------|
 | SPOOLMAN_DB_TYPE          | Type of database, any of: `postgres`, `mysql`, `sqlite`, `cockroachdb`                                                       |
 | SPOOLMAN_DB_HOST          | Database hostname                                                                                                            |
 | SPOOLMAN_DB_PORT          | Database port                                                                                                                |
@@ -130,6 +130,7 @@ If you want to connect with an external database instead, specify the `SPOOLMAN_
 | PGID                      | (*docker only*) Set the GID of the user in the docker container. Default is 1000.                                            |
 | SPOOLMAN_PORT             | The port Spoolman should run on (default: 8000)                                                                              |
 | SPOOLMAN_HOST             | The hostname/ip Spoolman should bind to (default: 0.0.0.0)                                                                   |
+| SPOOLMAN_METRICS_ENABLED  | Enable collect Spoolman prometheus metrics at database. Default `False`                                                      |
 
 ## Frequently Asked Questions (FAQs)
 ### QR Code Does not work on HTTP / The page is not served over HTTPS
@@ -148,6 +149,7 @@ To setup yourself for Python development, do the following:
 1. Clone this repo
 2. CD into the repo
 3. Install PDM: `pip install --user pdm`
+> At pre-commit hook used pdm==2.7.4
 4. Install Spoolman dependencies: `pdm sync`
 
 And you should be all setup. Read the Style and Integration Testing sections below as well.

client/src/pages/spools/create.tsx

@@ -50,7 +50,9 @@ export const SpoolCreate: React.FC<IResourceComponentsProps & CreateOrCloneProps
     formProps.initialValues.used_weight = 0;
 
     // Fix the filament_id
-    formProps.initialValues.filament_id = formProps.initialValues.filament.id;
+    if (formProps.initialValues.filament) {
+      formProps.initialValues.filament_id = formProps.initialValues.filament.id;
+    }
   }
 
   // If the query variable filament_id is set, set the filament_id field to that value

pdm.lock

@@ -250,6 +250,12 @@ dependencies = [
     "virtualenv>=20.10.0",
 ]
 
+[[package]]
+name = "prometheus-client"
+version = "0.20.0"
+requires_python = ">=3.8"
+summary = "Python client for the Prometheus monitoring system."
+
 [[package]]
 name = "psycopg2-binary"
 version = "2.9.7"
@@ -438,7 +444,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files"
 lock_version = "4.2"
 cross_platform = true
 groups = ["default", "dev"]
-content_hash = "sha256:4fde87958090f465a7284770a9804602c79d57403c2cbb39223c5e31ef7621b0"
+content_hash = "sha256:d07792f8a4d33b46c1f6b8397b33505830958a636c49a0234a1541e669457d99"
 
 [metadata.files]
 "aiomysql 0.2.0" = [
@@ -780,6 +786,10 @@ content_hash = "sha256:4fde87958090f465a7284770a9804602c79d57403c2cbb39223c5e31e
     {url = "https://files.pythonhosted.org/packages/35/0e/564c71fe3cdf59a4acaaccaea354d066e5d9044eba564dac070bb2075432/pre_commit-3.3.3.tar.gz", hash = "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"},
     {url = "https://files.pythonhosted.org/packages/e3/b7/1d145c985d8be9729672a45b8b8113030ad60dff45dec592efc4e5f5897a/pre_commit-3.3.3-py2.py3-none-any.whl", hash = "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb"},
 ]
+"prometheus-client 0.20.0" = [
+    {url = "https://files.pythonhosted.org/packages/3d/39/3be07741a33356127c4fe633768ee450422c1231c6d34b951fee1458308d/prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"},
+    {url = "https://files.pythonhosted.org/packages/c7/98/745b810d822103adca2df8decd4c0bbe839ba7ad3511af3f0d09692fc0f0/prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"},
+]
 "psycopg2-binary 2.9.7" = [
     {url = "https://files.pythonhosted.org/packages/01/40/660b4fae9d3c6f8281bc514f13c2915cbadfb1903ac57d0d67a3b599637c/psycopg2_binary-2.9.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ea5f8ee87f1eddc818fc04649d952c526db4426d26bab16efbe5a0c52b27d6ab"},
     {url = "https://files.pythonhosted.org/packages/01/a3/50875e4480556726ad7f9092ae5bccdc47fad92425373cbec1f6b302a0e4/psycopg2_binary-2.9.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc10da7e7df3380426521e8c1ed975d22df678639da2ed0ec3244c3dc2ab54c8"},
@@ -932,6 +942,7 @@ content_hash = "sha256:4fde87958090f465a7284770a9804602c79d57403c2cbb39223c5e31e
     {url = "https://files.pythonhosted.org/packages/ba/91/090818dfa62e85181f3ae23dd1e8b7ea7f09684864a900cab72d29c57346/PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
     {url = "https://files.pythonhosted.org/packages/bc/06/1b305bf6aa704343be85444c9d011f626c763abb40c0edc1cad13bfd7f86/PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {url = "https://files.pythonhosted.org/packages/c1/39/47ed4d65beec9ce07267b014be85ed9c204fa373515355d3efa62d19d892/PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
+    {url = "https://files.pythonhosted.org/packages/c7/4c/4a2908632fc980da6d918b9de9c1d9d7d7e70b2672b1ad5166ed27841ef7/PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {url = "https://files.pythonhosted.org/packages/c7/d1/02baa09d39b1bb1ebaf0d850d106d1bdcb47c91958557f471153c49dc03b/PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
     {url = "https://files.pythonhosted.org/packages/c8/6b/6600ac24725c7388255b2f5add93f91e58a5d7efaf4af244fdbcc11a541b/PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
     {url = "https://files.pythonhosted.org/packages/cc/5c/fcabd17918348c7db2eeeb0575705aaf3f7ab1657f6ce29b2e31737dd5d1/PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},

pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     "psycopg2-binary~=2.9",
     "setuptools~=68.0",
     "WebSockets>=11.0.3",
+    "prometheus-client>=0.20.0",
 ]
 requires-python = ">=3.9"
 

spoolman/database/database.py

@@ -1,5 +1,4 @@
 """SQLAlchemy database setup."""
-
 import datetime
 import logging
 import shutil
@@ -14,6 +13,7 @@ from sqlalchemy import URL
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
 
 from spoolman import env
+from spoolman.prometheus.metrics import filament_metrics, spool_metrics
 
 logger = logging.getLogger(__name__)
 
@@ -183,6 +183,15 @@ async def _backup_task() -> Optional[Path]:
     return __db.backup_and_rotate(env.get_backups_dir(), num_backups=5)
 
 
+async def _metrics() -> None:
+    """Create some useful prometheus metrics."""
+    logger.debug("Start metrics collection")
+    async for session in get_db_session():
+        await filament_metrics(session)
+        await spool_metrics(session)
+    logger.debug("End metrics collection")
+
+
 def schedule_tasks(scheduler: Scheduler) -> None:
     """Schedule tasks to be executed by the provided scheduler.
 
@@ -191,6 +200,10 @@ def schedule_tasks(scheduler: Scheduler) -> None:
     """
     if __db is None:
         raise RuntimeError("DB is not setup.")
+    if env.is_metrics_enabled():
+        logger.info("Scheduling automatic metric collection.")
+        # Run every minute, may be needs specify timer
+        scheduler.minutely(datetime.time(second=0), _metrics)
     if not env.is_automatic_backup_enabled():
         return
     if "sqlite" in __db.connection_url.drivername:

spoolman/env.py

@@ -380,3 +380,21 @@ def is_data_dir_mounted() -> bool:
     mounts = subprocess.run("mount", check=True, stdout=subprocess.PIPE, text=True)  # noqa: S603, S607
     data_dir = str(get_data_dir().resolve())
     return any(data_dir in line for line in mounts.stdout.splitlines())
+
+
+def is_metrics_enabled() -> bool:
+    """Get whether collect prometheus metrics at database is enabled.
+
+        Returns False if no environment variable was set for collect metrics.
+
+    Returns:
+        bool: Whether collect metrics is enabled.
+    """
+    metrics_enabled = os.getenv("SPOOLMAN_METRICS_ENABLED", "FALSE").upper()
+    if metrics_enabled in {"FALSE", "0"}:
+        return False
+    if metrics_enabled in {"TRUE", "1"}:
+        return True
+    raise ValueError(
+        f"Failed to parse SPOOLMAN_METRICS_ENABLED variable: Unknown metrics enabled '{metrics_enabled}'.",
+    )

spoolman/main.py

@@ -1,5 +1,4 @@
 """Main entrypoint to the server."""
-
 import logging
 import subprocess
 from logging.handlers import TimedRotatingFileHandler
@@ -9,12 +8,15 @@ import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import PlainTextResponse
+from prometheus_client import generate_latest
 from scheduler.asyncio.scheduler import Scheduler
 
 from spoolman import env
 from spoolman.api.v1.router import app as v1_app
 from spoolman.client import SinglePageApplication
 from spoolman.database import database
+from spoolman.prometheus.metrics import registry
 
 # Define a console logger
 console_handler = logging.StreamHandler()
@@ -37,7 +39,23 @@ app = FastAPI(
 )
 app.add_middleware(GZipMiddleware)
 app.mount("/api/v1", v1_app)
-app.mount("/", app=SinglePageApplication(directory="client/dist"), name="client")
+
+
+# WA for prometheus /metrics bind with SinglePageApp at root
+@app.get(
+    "/metrics",
+    response_class=PlainTextResponse,
+    name="Get metrics for prometheus",
+    description=(
+        "Get app metrics for prometheusIf enabled SPOOLMAN_METRICS_ENABLED returned metrics by Spools and Filaments"
+    ),
+)
+def get_metrics() -> bytes:
+    """Return prometheus metrics."""
+    return generate_latest(registry)
+
+
+app.mount("", app=SinglePageApplication(directory="client/dist"))
 
 
 # Allow all origins if in debug mode

spoolman/prometheus/metrics.py

@@ -0,0 +1,86 @@
+"""Prometheus metrics collectors."""
+import logging
+from typing import Callable
+
+import sqlalchemy
+from prometheus_client import REGISTRY, Gauge, make_asgi_app
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import contains_eager
+
+from spoolman.database import models
+
+registry = REGISTRY
+
+PREFIX = "spoolman"
+
+SPOOL_PRICE = Gauge("%s_spool_price" % PREFIX, "Total Spool price", ["spool_id", "filament_id"])
+SPOOL_USED_WEIGHT = Gauge("%s_spool_weight_used" % PREFIX, "Spool Used Weight", ["spool_id", "filament_id"])
+FILAMENT_INFO = Gauge(
+    "%s_filament_info" % PREFIX,
+    "Filament information",
+    ["filament_id", "vendor", "name", "material", "color"],
+)
+FILAMENT_DENSITY = Gauge("%s_filament_density" % PREFIX, "Density of filament", ["filament_id"])
+FILAMENT_DIAMETER = Gauge("%s_filament_diameter" % PREFIX, "Diameter of filament", ["filament_id"])
+FILAMENT_WEIGHT = Gauge("%s_filament_weight" % PREFIX, "Net weight of filament", ["filament_id"])
+
+logger = logging.getLogger(__name__)
+
+
+def make_metrics_app() -> Callable:
+    """Start ASGI prometheus app with global registry."""
+    logger.info("Start metrics app")
+    return make_asgi_app(registry=registry)
+
+
+metrics_app = make_asgi_app()
+
+
+async def spool_metrics(db: AsyncSession) -> None:
+    """Get metrics by Spools from DB and write to prometheus.
+
+    Args:
+        db: async db session
+    """
+    stmt = sqlalchemy.select(models.Spool).where(
+        sqlalchemy.or_(
+            models.Spool.archived.is_(False),
+            models.Spool.archived.is_(None),
+        ),
+    )
+    rows = await db.execute(stmt)
+    result = list(rows.unique().scalars().all())
+    for row in result:
+        if row.price is not None:
+            SPOOL_PRICE.labels(str(row.id), str(row.filament_id)).set(row.price)
+        SPOOL_USED_WEIGHT.labels(str(row.id), str(row.filament_id)).set(row.used_weight)
+
+
+async def filament_metrics(db: AsyncSession) -> None:
+    """Get metrics and info by Filaments from DB and write to prometheus.
+
+    Args:
+        db: async db session
+    """
+    stmt = (
+        sqlalchemy.select(models.Filament)
+        .options(contains_eager(models.Filament.vendor))
+        .join(models.Filament.vendor, isouter=True)
+    )
+    rows = await db.execute(stmt)
+    result = list(rows.unique().scalars().all())
+    for row in result:
+        vendor_name = "-"
+        if row.vendor is not None:
+            vendor_name = row.vendor.name
+        FILAMENT_INFO.labels(
+            str(row.id),
+            vendor_name,
+            row.name,
+            row.material,
+            row.color_hex,
+        ).set(1)
+        FILAMENT_DENSITY.labels(str(row.id)).set(row.density)
+        FILAMENT_DIAMETER.labels(str(row.id)).set(row.diameter)
+        if row.weight is not None:
+            FILAMENT_WEIGHT.labels(str(row.id)).set(row.weight)