Added a new backend settings system

This can be used for saving runtime-configurable settings in the database.
2024-01-03 18:44:09 +01:00
parent cad6d4a7f7
commit 44da991d47
11 changed files with 573 additions and 3 deletions
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -1,7 +1,18 @@
 # Migrations

-Creating a new version:
+Migrations are used to create and update the database schema. They are run automatically every time Spoolman starts.
+
+To create a new migration, edit the tables as desired in `spoolman/database/models.py`, then start the Spoolman server to update your local sqlite database.
+
 ```bash
 pdm run python -m spoolman.main
+```
+
+Stop the server once it's up.
+
+Then, let Alembic automatically create a new migration file:
+```bash
 pdm run alembic revision -m "some title" --autogenerate
 ```
+
+Go into the created migration and make sure it looks good, that the column changes etc are as desired. Format it with Black and Ruff. Commit.
--- a/migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py
+++ b/migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py
@@ -0,0 +1,32 @@
+"""Added Settings table.
+
+Revision ID: ccbb17aeda7c
+Revises: 92793c8a937c
+Create Date: 2024-01-03 13:46:41.362341
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ccbb17aeda7c"
+down_revision = "92793c8a937c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Perform the upgrade."""
+    op.create_table(
+        "setting",
+        sa.Column("key", sa.String(length=64), nullable=False),
+        sa.Column("value", sa.Text(), nullable=False),
+        sa.Column("last_updated", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("key"),
+    )
+    op.create_index(op.f("ix_setting_key"), "setting", ["key"], unique=False)
+
+
+def downgrade() -> None:
+    """Perform the downgrade."""
+    op.drop_index(op.f("ix_setting_key"), table_name="setting")
+    op.drop_table("setting")
--- a/spoolman/api/v1/models.py
+++ b/spoolman/api/v1/models.py
@@ -9,6 +9,7 @@ from pydantic import Field

 from spoolman.database import models
 from spoolman.math import length_from_weight
+from spoolman.settings import SettingDefinition, SettingType


 def datetime_to_str(dt: datetime) -> str:
@@ -31,6 +32,29 @@ class Message(BaseModel):
    message: str = Field()


+class SettingResponse(BaseModel):
+    value: str = Field(description="Setting value.")
+    is_set: bool = Field(description="Whether the setting has been set. If false, 'value' contains the default value.")
+    type: SettingType = Field(description="Setting type. This corresponds with JSON types.")
+
+
+class SettingKV(BaseModel):
+    key: str = Field(description="Setting key.")
+    setting: SettingResponse = Field(description="Setting value.")
+
+    @staticmethod
+    def from_db(definition: SettingDefinition, set_value: Optional[str]) -> "SettingKV":
+        """Create a new Pydantic vendor object from a database vendor object."""
+        return SettingKV(
+            key=definition.key,
+            setting=SettingResponse(
+                value=set_value if set_value is not None else definition.default,
+                is_set=set_value is not None,
+                type=definition.type,
+            ),
+        )
+
+
 class Vendor(BaseModel):
    id: int = Field(description="Unique internal ID of this vendor.")
    registered: datetime = Field(description="When the vendor was registered in the database. UTC Timezone.")
@@ -274,3 +298,10 @@ class VendorEvent(Event):

    payload: Vendor = Field(description="Updated vendor.")
    resource: Literal["vendor"] = Field(description="Resource type.")
+
+
+class SettingEvent(Event):
+    """Event."""
+
+    payload: SettingKV = Field(description="Updated setting.")
+    resource: Literal["setting"] = Field(description="Resource type.")
--- a/spoolman/api/v1/router.py
+++ b/spoolman/api/v1/router.py
@@ -15,7 +15,7 @@ from spoolman.database.database import backup_global_db
 from spoolman.exceptions import ItemNotFoundError
 from spoolman.ws import websocket_manager

-from . import filament, models, other, spool, vendor
+from . import filament, models, other, setting, spool, vendor

 logger = logging.getLogger(__name__)

@@ -107,4 +107,5 @@ async def notify(
 app.include_router(filament.router)
 app.include_router(spool.router)
 app.include_router(vendor.router)
+app.include_router(setting.router)
 app.include_router(other.router)
--- a/spoolman/api/v1/setting.py
+++ b/spoolman/api/v1/setting.py
@@ -0,0 +1,194 @@
+"""Vendor related endpoints."""
+
+import asyncio
+import logging
+from typing import Annotated, Union
+
+from fastapi import APIRouter, Body, Depends, WebSocket, WebSocketDisconnect
+from fastapi.responses import JSONResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from spoolman.api.v1.models import Message, SettingEvent, SettingResponse
+from spoolman.database import setting
+from spoolman.database.database import get_db_session
+from spoolman.exceptions import ItemNotFoundError
+from spoolman.settings import SETTINGS, parse_setting
+from spoolman.ws import websocket_manager
+
+router = APIRouter(
+    prefix="/setting",
+    tags=["setting"],
+)
+
+# ruff: noqa: D103,B008
+
+logger = logging.getLogger(__name__)
+
+
+@router.websocket(
+    "",
+    name="Listen to setting changes",
+)
+async def notify_any(
+    websocket: WebSocket,
+) -> None:
+    await websocket.accept()
+    websocket_manager.connect(("setting",), websocket)
+    try:
+        while True:
+            await asyncio.sleep(0.5)
+            if await websocket.receive_text():
+                await websocket.send_json({"status": "healthy"})
+    except WebSocketDisconnect:
+        websocket_manager.disconnect(("setting",), websocket)
+
+
+@router.get(
+    "/{key}",
+    name="Get setting",
+    description=(
+        "Get a specific setting. If the setting has not been set, the default value will be returned."
+        "A websocket is served on the same path to listen for changes to the setting. "
+        "See the HTTP Response code 299 for the content of the websocket messages."
+    ),
+    response_model_exclude_none=True,
+    response_model=SettingResponse,
+    responses={404: {"model": Message}, 299: {"model": SettingEvent, "description": "Websocket message"}},
+)
+async def get(
+    db: Annotated[AsyncSession, Depends(get_db_session)],
+    key: str,
+) -> Union[SettingResponse, JSONResponse]:
+    try:
+        definition = parse_setting(key)
+    except ValueError as e:
+        return JSONResponse(status_code=404, content=Message(message=str(e)).dict())
+
+    try:
+        db_item = await setting.get(db, definition)
+        value = db_item.value
+        is_set = True
+    except ItemNotFoundError:
+        value = definition.default
+        is_set = False
+
+    return SettingResponse(
+        value=value,
+        is_set=is_set,
+        type=definition.type,
+    )
+
+
+@router.get(
+    "/",
+    name="Get all settings",
+    description=("Get all settings, set or not. If the setting has not been set, 'value' will be the default value."),
+    response_model_exclude_none=True,
+    response_model=dict[str, SettingResponse],
+)
+async def find(
+    db: Annotated[AsyncSession, Depends(get_db_session)],
+) -> Union[dict[str, SettingResponse], JSONResponse]:
+    settings: dict[str, SettingResponse] = {}
+
+    # First get all settings that have been set.
+    db_items = await setting.get_all(db)
+    for db_item in db_items:
+        try:
+            definition = parse_setting(db_item.key)
+        except ValueError:
+            continue  # Ignore settings that have been removed.
+
+        settings[db_item.key] = SettingResponse(
+            value=db_item.value,
+            is_set=True,
+            type=definition.type,
+        )
+
+    # Then get all settings that have not been set.
+    for settingdef in SETTINGS.values():
+        if settingdef.key not in settings:
+            settings[settingdef.key] = SettingResponse(
+                value=settingdef.default,
+                is_set=False,
+                type=settingdef.type,
+            )
+
+    return settings
+
+
+@router.websocket(
+    "/{key}",
+    name="Listen to setting changes",
+)
+async def notify(
+    websocket: WebSocket,
+    key: str,
+) -> None:
+    try:
+        parse_setting(key)
+    except ValueError as e:
+        await websocket.close(code=4040, reason=str(e))
+        return
+
+    await websocket.accept()
+    websocket_manager.connect(("setting", str(key)), websocket)
+    try:
+        while True:
+            await asyncio.sleep(0.5)
+            if await websocket.receive_text():
+                await websocket.send_json({"status": "healthy"})
+    except WebSocketDisconnect:
+        websocket_manager.disconnect(("setting", str(key)), websocket)
+
+
+@router.post(
+    "/{key}",
+    name="Set setting",
+    description=(
+        "Set the value of a setting. The body must match the JSON type of the setting. "
+        "An empty body or a body containing only 'null' will reset the setting to its default value. "
+        "The new value will be returned."
+    ),
+    response_model_exclude_none=True,
+    response_model=SettingResponse,
+    responses={404: {"model": Message}},
+)
+async def update(
+    db: Annotated[AsyncSession, Depends(get_db_session)],
+    key: str,
+    body: Annotated[str, Body()],
+) -> Union[SettingResponse, JSONResponse]:
+    try:
+        definition = parse_setting(key)
+    except ValueError as e:
+        return JSONResponse(status_code=404, content=Message(message=str(e)).dict())
+
+    if body and body != "null":
+        try:
+            definition.validate_type(body)
+        except ValueError as e:
+            return JSONResponse(status_code=400, content=Message(message=str(e)).dict())
+
+        await setting.update(db=db, definition=definition, value=body)
+        logger.info('Setting "%s" has been set to "%s".', key, body)
+    else:
+        await setting.delete(db=db, definition=definition)
+        logger.info('Setting "%s" has been unset.', key)
+
+    await db.commit()
+
+    # Get the new value of the setting.
+    try:
+        db_item = await setting.get(db, definition)
+        value = db_item.value
+        is_set = True
+    except ItemNotFoundError:
+        value = definition.default
+        is_set = False
+
+    return SettingResponse(
+        value=value,
+        is_set=is_set,
+        type=definition.type,
+    )
--- a/spoolman/database/models.py
+++ b/spoolman/database/models.py
@@ -3,7 +3,7 @@
 from datetime import datetime
 from typing import Optional

-from sqlalchemy import ForeignKey, Integer, String
+from sqlalchemy import ForeignKey, Integer, String, Text
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship


@@ -58,3 +58,11 @@ class Spool(Base):
    lot_nr: Mapped[Optional[str]] = mapped_column(String(64))
    comment: Mapped[Optional[str]] = mapped_column(String(1024))
    archived: Mapped[Optional[bool]] = mapped_column()
+
+
+class Setting(Base):
+    __tablename__ = "setting"
+
+    key: Mapped[str] = mapped_column(String(64), primary_key=True, index=True)
+    value: Mapped[str] = mapped_column(Text())
+    last_updated: Mapped[datetime] = mapped_column()
--- a/spoolman/database/setting.py
+++ b/spoolman/database/setting.py
@@ -0,0 +1,69 @@
+"""Helper functions for interacting with vendor database objects."""
+
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from spoolman.api.v1.models import EventType, SettingEvent, SettingKV
+from spoolman.database import models
+from spoolman.exceptions import ItemNotFoundError
+from spoolman.settings import SettingDefinition
+from spoolman.ws import websocket_manager
+
+SETTING_MAX_LENGTH = 2**16 - 1
+
+
+async def update(
+    *,
+    db: AsyncSession,
+    definition: SettingDefinition,
+    value: str,
+) -> None:
+    """Set a setting in the database."""
+    if len(value) > SETTING_MAX_LENGTH:
+        raise ValueError(f"Setting value is too big, max size is {SETTING_MAX_LENGTH} characters.")
+
+    setting = models.Setting(
+        key=definition.key,
+        value=value,
+        last_updated=datetime.utcnow().replace(microsecond=0),
+    )
+    await db.merge(setting)
+    await setting_changed(definition, value, EventType.UPDATED)
+
+
+async def get(db: AsyncSession, definition: SettingDefinition) -> models.Setting:
+    """Get a specific setting from the database."""
+    setting = await db.get(models.Setting, definition.key)
+    if setting is None:
+        raise ItemNotFoundError(f"Setting with key {definition.key} has not been set.")
+    return setting
+
+
+async def get_all(db: AsyncSession) -> list[models.Setting]:
+    """Get all set settings in the database."""
+    stmt = select(models.Setting)
+    rows = await db.execute(stmt)
+    return list(rows.scalars().all())
+
+
+async def delete(db: AsyncSession, definition: SettingDefinition) -> None:
+    """Delete a setting from the database."""
+    setting = await get(db, definition)
+    await db.delete(setting)
+    await setting_changed(definition, None, EventType.DELETED)
+
+
+async def setting_changed(definition: SettingDefinition, set_value: Optional[str], typ: EventType) -> None:
+    """Notify websocket clients that a setting has changed."""
+    await websocket_manager.send(
+        ("setting", str(definition.key)),
+        SettingEvent(
+            type=typ,
+            resource="setting",
+            date=datetime.utcnow(),
+            payload=SettingKV.from_db(definition, set_value),
+        ),
+    )
--- a/spoolman/settings.py
+++ b/spoolman/settings.py
@@ -0,0 +1,64 @@
+"""Settings that can be changed by the user.
+
+All settings are JSON encoded and stored in the database.
+"""
+
+import json
+from dataclasses import dataclass
+from enum import Enum
+
+
+class SettingType(Enum):
+    """The type of a setting."""
+
+    BOOLEAN = "boolean"
+    NUMBER = "number"
+    STRING = "string"
+    ARRAY = "array"
+    OBJECT = "object"
+
+
+@dataclass
+class SettingDefinition:
+    """A setting that can be changed by the user."""
+
+    key: str
+    type: SettingType
+    default: str
+
+    def validate_type(self, value: str) -> None:  # noqa: C901
+        """Validate that the value has the correct type."""
+        obj = json.loads(value)
+        if self.type == SettingType.BOOLEAN:
+            if not isinstance(obj, bool):
+                raise ValueError(f"Setting {self.key} must be a boolean.")
+        elif self.type == SettingType.NUMBER:
+            if not isinstance(obj, (int, float)):
+                raise ValueError(f"Setting {self.key} must be a number.")
+        elif self.type == SettingType.STRING:
+            if not isinstance(obj, str):
+                raise ValueError(f"Setting {self.key} must be a string.")
+        elif self.type == SettingType.ARRAY:
+            if not isinstance(obj, list):
+                raise ValueError(f"Setting {self.key} must be an array.")
+        elif self.type == SettingType.OBJECT:  # noqa: SIM102
+            if not isinstance(obj, dict):
+                raise ValueError(f"Setting {self.key} must be an object.")
+
+
+SETTINGS: dict[str, SettingDefinition] = {}
+
+
+def register_setting(key: str, typ: SettingType, default: str) -> None:
+    """Register a setting."""
+    SETTINGS[key] = SettingDefinition(key, typ, default)
+
+
+def parse_setting(key: str) -> SettingDefinition:
+    """Parse a setting key."""
+    if key not in SETTINGS:
+        raise ValueError(f"Setting {key} does not exist.")
+    return SETTINGS[key]
+
+
+register_setting("currency", SettingType.STRING, json.dumps("EUR"))
--- a/tests_integration/tests/setting/init.py
+++ b/tests_integration/tests/setting/init.py
@@ -0,0 +1 @@
+"""Tests for the setting API."""
--- a/tests_integration/tests/setting/test_get.py
+++ b/tests_integration/tests/setting/test_get.py
@@ -0,0 +1,44 @@
+"""Integration tests for the Vendor API endpoint."""
+
+import httpx
+
+URL = "http://spoolman:8000"
+
+
+def test_get_currency():
+    """Test getting the currency setting."""
+    # Execute
+    result = httpx.get(f"{URL}/api/v1/setting/currency")
+    result.raise_for_status()
+
+    # Verify
+    setting = result.json()
+    assert setting == {
+        "value": '"EUR"',
+        "is_set": False,
+        "type": "string",
+    }
+
+
+def test_get_unknown():
+    """Test getting an unknown setting."""
+    # Execute
+    result = httpx.get(f"{URL}/api/v1/setting/unknown")
+    assert result.status_code == 404
+
+
+def test_get_all():
+    """Test getting all settings."""
+    # Execute
+    result = httpx.get(f"{URL}/api/v1/setting/")
+    result.raise_for_status()
+
+    # Verify
+    settings = result.json()
+    assert settings == {
+        "currency": {
+            "value": '"EUR"',
+            "is_set": False,
+            "type": "string",
+        },
+    }
--- a/tests_integration/tests/setting/test_set.py
+++ b/tests_integration/tests/setting/test_set.py
@@ -0,0 +1,115 @@
+"""Integration tests for the Vendor API endpoint."""
+
+import json
+
+import httpx
+
+URL = "http://spoolman:8000"
+
+
+def test_set_currency():
+    """Test setting the currency setting."""
+    # Execute
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json='"SEK"',
+    )
+    result.raise_for_status()
+
+    # Verify
+    setting = result.json()
+    assert setting == {
+        "value": '"SEK"',
+        "is_set": True,
+        "type": "string",
+    }
+
+    # Cleanup
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json="",
+    )
+    result.raise_for_status()
+
+
+def test_unset_currency():
+    """Test un-setting the currency setting."""
+    # Execute set
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json='"SEK"',
+    )
+    result.raise_for_status()
+
+    # Verify set
+    setting = result.json()
+    assert setting == {
+        "value": '"SEK"',
+        "is_set": True,
+        "type": "string",
+    }
+
+    # Execute unset
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json="",
+    )
+    result.raise_for_status()
+
+    # Verify unset
+    setting = result.json()
+    assert setting == {
+        "value": '"EUR"',
+        "is_set": False,
+        "type": "string",
+    }
+
+
+def test_set_unknown():
+    """Test setting an invalid setting."""
+    # Execute
+    result = httpx.post(
+        f"{URL}/api/v1/setting/not-a-setting",
+        json='"SEK"',
+    )
+    assert result.status_code == 404
+
+
+def test_set_currency_wrong_type():
+    """Test setting the currency setting with the wrong type."""
+    # Execute
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json=123,
+    )
+    assert result.status_code == 400
+
+
+def test_set_big_value():
+    """Test setting a setting to a long string which should be saved correctly."""
+    long_string = "a" * (2**16 - 1 - 2)  # Backend guarantees that it can handle strings up to 65535 characters long.
+    # Remove 2 characters to account for the quotes.
+
+    # Execute
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json=json.dumps(long_string),
+    )
+    result.raise_for_status()
+
+    # Verify
+    result = httpx.get(f"{URL}/api/v1/setting/currency")
+    result.raise_for_status()
+    setting = result.json()
+    assert setting == {
+        "value": json.dumps(long_string),
+        "is_set": True,
+        "type": "string",
+    }
+
+    # Cleanup
+    result = httpx.post(
+        f"{URL}/api/v1/setting/currency",
+        json="",
+    )
+    result.raise_for_status()