From 44da991d47f81adb22b780e7879051ee99e5206b Mon Sep 17 00:00:00 2001 From: Donkie Date: Wed, 3 Jan 2024 18:44:09 +0100 Subject: [PATCH] Added a new backend settings system This can be used for saving runtime-configurable settings in the database. --- migrations/README.md | 13 +- ..._1346-ccbb17aeda7c_added_settings_table.py | 32 +++ spoolman/api/v1/models.py | 31 +++ spoolman/api/v1/router.py | 3 +- spoolman/api/v1/setting.py | 194 ++++++++++++++++++ spoolman/database/models.py | 10 +- spoolman/database/setting.py | 69 +++++++ spoolman/settings.py | 64 ++++++ tests_integration/tests/setting/__init__.py | 1 + tests_integration/tests/setting/test_get.py | 44 ++++ tests_integration/tests/setting/test_set.py | 115 +++++++++++ 11 files changed, 573 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py create mode 100644 spoolman/api/v1/setting.py create mode 100644 spoolman/database/setting.py create mode 100644 spoolman/settings.py create mode 100644 tests_integration/tests/setting/__init__.py create mode 100644 tests_integration/tests/setting/test_get.py create mode 100644 tests_integration/tests/setting/test_set.py diff --git a/migrations/README.md b/migrations/README.md index cb1b35b..5113989 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,7 +1,18 @@ # Migrations -Creating a new version: +Migrations are used to create and update the database schema. They are run automatically every time Spoolman starts. + +To create a new migration, edit the tables as desired in `spoolman/database/models.py`, then start the Spoolman server to update your local sqlite database. + ```bash pdm run python -m spoolman.main +``` + +Stop the server once it's up. + +Then, let Alembic automatically create a new migration file: +```bash pdm run alembic revision -m "some title" --autogenerate ``` + +Go into the created migration and make sure it looks good, that the column changes etc are as desired. Format it with Black and Ruff. Commit. diff --git a/migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py b/migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py new file mode 100644 index 0000000..788375b --- /dev/null +++ b/migrations/versions/2024_01_03_1346-ccbb17aeda7c_added_settings_table.py @@ -0,0 +1,32 @@ +"""Added Settings table. + +Revision ID: ccbb17aeda7c +Revises: 92793c8a937c +Create Date: 2024-01-03 13:46:41.362341 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ccbb17aeda7c" +down_revision = "92793c8a937c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Perform the upgrade.""" + op.create_table( + "setting", + sa.Column("key", sa.String(length=64), nullable=False), + sa.Column("value", sa.Text(), nullable=False), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("key"), + ) + op.create_index(op.f("ix_setting_key"), "setting", ["key"], unique=False) + + +def downgrade() -> None: + """Perform the downgrade.""" + op.drop_index(op.f("ix_setting_key"), table_name="setting") + op.drop_table("setting") diff --git a/spoolman/api/v1/models.py b/spoolman/api/v1/models.py index 0be5a14..4288e12 100644 --- a/spoolman/api/v1/models.py +++ b/spoolman/api/v1/models.py @@ -9,6 +9,7 @@ from pydantic import Field from spoolman.database import models from spoolman.math import length_from_weight +from spoolman.settings import SettingDefinition, SettingType def datetime_to_str(dt: datetime) -> str: @@ -31,6 +32,29 @@ class Message(BaseModel): message: str = Field() +class SettingResponse(BaseModel): + value: str = Field(description="Setting value.") + is_set: bool = Field(description="Whether the setting has been set. If false, 'value' contains the default value.") + type: SettingType = Field(description="Setting type. This corresponds with JSON types.") + + +class SettingKV(BaseModel): + key: str = Field(description="Setting key.") + setting: SettingResponse = Field(description="Setting value.") + + @staticmethod + def from_db(definition: SettingDefinition, set_value: Optional[str]) -> "SettingKV": + """Create a new Pydantic vendor object from a database vendor object.""" + return SettingKV( + key=definition.key, + setting=SettingResponse( + value=set_value if set_value is not None else definition.default, + is_set=set_value is not None, + type=definition.type, + ), + ) + + class Vendor(BaseModel): id: int = Field(description="Unique internal ID of this vendor.") registered: datetime = Field(description="When the vendor was registered in the database. UTC Timezone.") @@ -274,3 +298,10 @@ class VendorEvent(Event): payload: Vendor = Field(description="Updated vendor.") resource: Literal["vendor"] = Field(description="Resource type.") + + +class SettingEvent(Event): + """Event.""" + + payload: SettingKV = Field(description="Updated setting.") + resource: Literal["setting"] = Field(description="Resource type.") diff --git a/spoolman/api/v1/router.py b/spoolman/api/v1/router.py index f4e9731..812db8a 100644 --- a/spoolman/api/v1/router.py +++ b/spoolman/api/v1/router.py @@ -15,7 +15,7 @@ from spoolman.database.database import backup_global_db from spoolman.exceptions import ItemNotFoundError from spoolman.ws import websocket_manager -from . import filament, models, other, spool, vendor +from . import filament, models, other, setting, spool, vendor logger = logging.getLogger(__name__) @@ -107,4 +107,5 @@ async def notify( app.include_router(filament.router) app.include_router(spool.router) app.include_router(vendor.router) +app.include_router(setting.router) app.include_router(other.router) diff --git a/spoolman/api/v1/setting.py b/spoolman/api/v1/setting.py new file mode 100644 index 0000000..a1d1029 --- /dev/null +++ b/spoolman/api/v1/setting.py @@ -0,0 +1,194 @@ +"""Vendor related endpoints.""" + +import asyncio +import logging +from typing import Annotated, Union + +from fastapi import APIRouter, Body, Depends, WebSocket, WebSocketDisconnect +from fastapi.responses import JSONResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from spoolman.api.v1.models import Message, SettingEvent, SettingResponse +from spoolman.database import setting +from spoolman.database.database import get_db_session +from spoolman.exceptions import ItemNotFoundError +from spoolman.settings import SETTINGS, parse_setting +from spoolman.ws import websocket_manager + +router = APIRouter( + prefix="/setting", + tags=["setting"], +) + +# ruff: noqa: D103,B008 + +logger = logging.getLogger(__name__) + + +@router.websocket( + "", + name="Listen to setting changes", +) +async def notify_any( + websocket: WebSocket, +) -> None: + await websocket.accept() + websocket_manager.connect(("setting",), websocket) + try: + while True: + await asyncio.sleep(0.5) + if await websocket.receive_text(): + await websocket.send_json({"status": "healthy"}) + except WebSocketDisconnect: + websocket_manager.disconnect(("setting",), websocket) + + +@router.get( + "/{key}", + name="Get setting", + description=( + "Get a specific setting. If the setting has not been set, the default value will be returned." + "A websocket is served on the same path to listen for changes to the setting. " + "See the HTTP Response code 299 for the content of the websocket messages." + ), + response_model_exclude_none=True, + response_model=SettingResponse, + responses={404: {"model": Message}, 299: {"model": SettingEvent, "description": "Websocket message"}}, +) +async def get( + db: Annotated[AsyncSession, Depends(get_db_session)], + key: str, +) -> Union[SettingResponse, JSONResponse]: + try: + definition = parse_setting(key) + except ValueError as e: + return JSONResponse(status_code=404, content=Message(message=str(e)).dict()) + + try: + db_item = await setting.get(db, definition) + value = db_item.value + is_set = True + except ItemNotFoundError: + value = definition.default + is_set = False + + return SettingResponse( + value=value, + is_set=is_set, + type=definition.type, + ) + + +@router.get( + "/", + name="Get all settings", + description=("Get all settings, set or not. If the setting has not been set, 'value' will be the default value."), + response_model_exclude_none=True, + response_model=dict[str, SettingResponse], +) +async def find( + db: Annotated[AsyncSession, Depends(get_db_session)], +) -> Union[dict[str, SettingResponse], JSONResponse]: + settings: dict[str, SettingResponse] = {} + + # First get all settings that have been set. + db_items = await setting.get_all(db) + for db_item in db_items: + try: + definition = parse_setting(db_item.key) + except ValueError: + continue # Ignore settings that have been removed. + + settings[db_item.key] = SettingResponse( + value=db_item.value, + is_set=True, + type=definition.type, + ) + + # Then get all settings that have not been set. + for settingdef in SETTINGS.values(): + if settingdef.key not in settings: + settings[settingdef.key] = SettingResponse( + value=settingdef.default, + is_set=False, + type=settingdef.type, + ) + + return settings + + +@router.websocket( + "/{key}", + name="Listen to setting changes", +) +async def notify( + websocket: WebSocket, + key: str, +) -> None: + try: + parse_setting(key) + except ValueError as e: + await websocket.close(code=4040, reason=str(e)) + return + + await websocket.accept() + websocket_manager.connect(("setting", str(key)), websocket) + try: + while True: + await asyncio.sleep(0.5) + if await websocket.receive_text(): + await websocket.send_json({"status": "healthy"}) + except WebSocketDisconnect: + websocket_manager.disconnect(("setting", str(key)), websocket) + + +@router.post( + "/{key}", + name="Set setting", + description=( + "Set the value of a setting. The body must match the JSON type of the setting. " + "An empty body or a body containing only 'null' will reset the setting to its default value. " + "The new value will be returned." + ), + response_model_exclude_none=True, + response_model=SettingResponse, + responses={404: {"model": Message}}, +) +async def update( + db: Annotated[AsyncSession, Depends(get_db_session)], + key: str, + body: Annotated[str, Body()], +) -> Union[SettingResponse, JSONResponse]: + try: + definition = parse_setting(key) + except ValueError as e: + return JSONResponse(status_code=404, content=Message(message=str(e)).dict()) + + if body and body != "null": + try: + definition.validate_type(body) + except ValueError as e: + return JSONResponse(status_code=400, content=Message(message=str(e)).dict()) + + await setting.update(db=db, definition=definition, value=body) + logger.info('Setting "%s" has been set to "%s".', key, body) + else: + await setting.delete(db=db, definition=definition) + logger.info('Setting "%s" has been unset.', key) + + await db.commit() + + # Get the new value of the setting. + try: + db_item = await setting.get(db, definition) + value = db_item.value + is_set = True + except ItemNotFoundError: + value = definition.default + is_set = False + + return SettingResponse( + value=value, + is_set=is_set, + type=definition.type, + ) diff --git a/spoolman/database/models.py b/spoolman/database/models.py index 45df367..685deb0 100644 --- a/spoolman/database/models.py +++ b/spoolman/database/models.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import ForeignKey, Integer, String +from sqlalchemy import ForeignKey, Integer, String, Text from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -58,3 +58,11 @@ class Spool(Base): lot_nr: Mapped[Optional[str]] = mapped_column(String(64)) comment: Mapped[Optional[str]] = mapped_column(String(1024)) archived: Mapped[Optional[bool]] = mapped_column() + + +class Setting(Base): + __tablename__ = "setting" + + key: Mapped[str] = mapped_column(String(64), primary_key=True, index=True) + value: Mapped[str] = mapped_column(Text()) + last_updated: Mapped[datetime] = mapped_column() diff --git a/spoolman/database/setting.py b/spoolman/database/setting.py new file mode 100644 index 0000000..e05415b --- /dev/null +++ b/spoolman/database/setting.py @@ -0,0 +1,69 @@ +"""Helper functions for interacting with vendor database objects.""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from spoolman.api.v1.models import EventType, SettingEvent, SettingKV +from spoolman.database import models +from spoolman.exceptions import ItemNotFoundError +from spoolman.settings import SettingDefinition +from spoolman.ws import websocket_manager + +SETTING_MAX_LENGTH = 2**16 - 1 + + +async def update( + *, + db: AsyncSession, + definition: SettingDefinition, + value: str, +) -> None: + """Set a setting in the database.""" + if len(value) > SETTING_MAX_LENGTH: + raise ValueError(f"Setting value is too big, max size is {SETTING_MAX_LENGTH} characters.") + + setting = models.Setting( + key=definition.key, + value=value, + last_updated=datetime.utcnow().replace(microsecond=0), + ) + await db.merge(setting) + await setting_changed(definition, value, EventType.UPDATED) + + +async def get(db: AsyncSession, definition: SettingDefinition) -> models.Setting: + """Get a specific setting from the database.""" + setting = await db.get(models.Setting, definition.key) + if setting is None: + raise ItemNotFoundError(f"Setting with key {definition.key} has not been set.") + return setting + + +async def get_all(db: AsyncSession) -> list[models.Setting]: + """Get all set settings in the database.""" + stmt = select(models.Setting) + rows = await db.execute(stmt) + return list(rows.scalars().all()) + + +async def delete(db: AsyncSession, definition: SettingDefinition) -> None: + """Delete a setting from the database.""" + setting = await get(db, definition) + await db.delete(setting) + await setting_changed(definition, None, EventType.DELETED) + + +async def setting_changed(definition: SettingDefinition, set_value: Optional[str], typ: EventType) -> None: + """Notify websocket clients that a setting has changed.""" + await websocket_manager.send( + ("setting", str(definition.key)), + SettingEvent( + type=typ, + resource="setting", + date=datetime.utcnow(), + payload=SettingKV.from_db(definition, set_value), + ), + ) diff --git a/spoolman/settings.py b/spoolman/settings.py new file mode 100644 index 0000000..4f35191 --- /dev/null +++ b/spoolman/settings.py @@ -0,0 +1,64 @@ +"""Settings that can be changed by the user. + +All settings are JSON encoded and stored in the database. +""" + +import json +from dataclasses import dataclass +from enum import Enum + + +class SettingType(Enum): + """The type of a setting.""" + + BOOLEAN = "boolean" + NUMBER = "number" + STRING = "string" + ARRAY = "array" + OBJECT = "object" + + +@dataclass +class SettingDefinition: + """A setting that can be changed by the user.""" + + key: str + type: SettingType + default: str + + def validate_type(self, value: str) -> None: # noqa: C901 + """Validate that the value has the correct type.""" + obj = json.loads(value) + if self.type == SettingType.BOOLEAN: + if not isinstance(obj, bool): + raise ValueError(f"Setting {self.key} must be a boolean.") + elif self.type == SettingType.NUMBER: + if not isinstance(obj, (int, float)): + raise ValueError(f"Setting {self.key} must be a number.") + elif self.type == SettingType.STRING: + if not isinstance(obj, str): + raise ValueError(f"Setting {self.key} must be a string.") + elif self.type == SettingType.ARRAY: + if not isinstance(obj, list): + raise ValueError(f"Setting {self.key} must be an array.") + elif self.type == SettingType.OBJECT: # noqa: SIM102 + if not isinstance(obj, dict): + raise ValueError(f"Setting {self.key} must be an object.") + + +SETTINGS: dict[str, SettingDefinition] = {} + + +def register_setting(key: str, typ: SettingType, default: str) -> None: + """Register a setting.""" + SETTINGS[key] = SettingDefinition(key, typ, default) + + +def parse_setting(key: str) -> SettingDefinition: + """Parse a setting key.""" + if key not in SETTINGS: + raise ValueError(f"Setting {key} does not exist.") + return SETTINGS[key] + + +register_setting("currency", SettingType.STRING, json.dumps("EUR")) diff --git a/tests_integration/tests/setting/__init__.py b/tests_integration/tests/setting/__init__.py new file mode 100644 index 0000000..6cb5e4d --- /dev/null +++ b/tests_integration/tests/setting/__init__.py @@ -0,0 +1 @@ +"""Tests for the setting API.""" diff --git a/tests_integration/tests/setting/test_get.py b/tests_integration/tests/setting/test_get.py new file mode 100644 index 0000000..bb8c006 --- /dev/null +++ b/tests_integration/tests/setting/test_get.py @@ -0,0 +1,44 @@ +"""Integration tests for the Vendor API endpoint.""" + +import httpx + +URL = "http://spoolman:8000" + + +def test_get_currency(): + """Test getting the currency setting.""" + # Execute + result = httpx.get(f"{URL}/api/v1/setting/currency") + result.raise_for_status() + + # Verify + setting = result.json() + assert setting == { + "value": '"EUR"', + "is_set": False, + "type": "string", + } + + +def test_get_unknown(): + """Test getting an unknown setting.""" + # Execute + result = httpx.get(f"{URL}/api/v1/setting/unknown") + assert result.status_code == 404 + + +def test_get_all(): + """Test getting all settings.""" + # Execute + result = httpx.get(f"{URL}/api/v1/setting/") + result.raise_for_status() + + # Verify + settings = result.json() + assert settings == { + "currency": { + "value": '"EUR"', + "is_set": False, + "type": "string", + }, + } diff --git a/tests_integration/tests/setting/test_set.py b/tests_integration/tests/setting/test_set.py new file mode 100644 index 0000000..8cb1cd1 --- /dev/null +++ b/tests_integration/tests/setting/test_set.py @@ -0,0 +1,115 @@ +"""Integration tests for the Vendor API endpoint.""" + +import json + +import httpx + +URL = "http://spoolman:8000" + + +def test_set_currency(): + """Test setting the currency setting.""" + # Execute + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json='"SEK"', + ) + result.raise_for_status() + + # Verify + setting = result.json() + assert setting == { + "value": '"SEK"', + "is_set": True, + "type": "string", + } + + # Cleanup + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json="", + ) + result.raise_for_status() + + +def test_unset_currency(): + """Test un-setting the currency setting.""" + # Execute set + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json='"SEK"', + ) + result.raise_for_status() + + # Verify set + setting = result.json() + assert setting == { + "value": '"SEK"', + "is_set": True, + "type": "string", + } + + # Execute unset + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json="", + ) + result.raise_for_status() + + # Verify unset + setting = result.json() + assert setting == { + "value": '"EUR"', + "is_set": False, + "type": "string", + } + + +def test_set_unknown(): + """Test setting an invalid setting.""" + # Execute + result = httpx.post( + f"{URL}/api/v1/setting/not-a-setting", + json='"SEK"', + ) + assert result.status_code == 404 + + +def test_set_currency_wrong_type(): + """Test setting the currency setting with the wrong type.""" + # Execute + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json=123, + ) + assert result.status_code == 400 + + +def test_set_big_value(): + """Test setting a setting to a long string which should be saved correctly.""" + long_string = "a" * (2**16 - 1 - 2) # Backend guarantees that it can handle strings up to 65535 characters long. + # Remove 2 characters to account for the quotes. + + # Execute + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json=json.dumps(long_string), + ) + result.raise_for_status() + + # Verify + result = httpx.get(f"{URL}/api/v1/setting/currency") + result.raise_for_status() + setting = result.json() + assert setting == { + "value": json.dumps(long_string), + "is_set": True, + "type": "string", + } + + # Cleanup + result = httpx.post( + f"{URL}/api/v1/setting/currency", + json="", + ) + result.raise_for_status()