From f24b9610e85d73fe9960c2baafb24d7057666726 Mon Sep 17 00:00:00 2001 From: Donkie Date: Tue, 4 Jul 2023 14:59:05 +0200 Subject: [PATCH] Added automatic nightly SQLite backups Resolves #22 --- README.md | 3 + requirements.txt | 1 + spoolman/api/v1/models.py | 9 ++ spoolman/api/v1/router.py | 12 ++ spoolman/database/database.py | 112 +++++++++++++++++- spoolman/env.py | 30 +++++ spoolman/main.py | 9 +- .../docker-compose-cockroachdb.yml | 2 + tests_integration/docker-compose-mariadb.yml | 2 + tests_integration/docker-compose-postgres.yml | 2 + tests_integration/docker-compose-sqlite.yml | 2 + tests_integration/tests/conftest.py | 23 ++++ tests_integration/tests/test_backup.py | 18 +++ 13 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 tests_integration/tests/test_backup.py diff --git a/README.md b/README.md index 1620737..410e22e 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ services: - ./data:/home/app/.local/share/spoolman ports: - "7912:8000" + environment: + - TZ=Europe/Stockholm # Optional, defaults to UTC ``` With this example, you should first create a folder called `data` in the same directory as the docker-compose.yml, then you should run `chown 1000:1000 data` on it in order to give it the correct permissions for the user inside the docker container. @@ -56,6 +58,7 @@ If you want to connect with an external database instead, specify the `SPOOLMAN_ | SPOOLMAN_DB_PASSWORD | Database password | | SPOOLMAN_DB_QUERY | Query parameters for the database connection, e.g. set to `unix_socket=/path/to/mysql.sock` to connect using a MySQL socket. | | SPOOLMAN_LOGGING_LEVEL | Logging level, any of: `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`, defaults to `INFO`. | +| SPOOLMAN_AUTOMATIC_BACKUP | Automatic nightly DB backups for SQLite databases. Enabled by default, set to `FALSE` to disable. | ## Configuration ### Moonraker diff --git a/requirements.txt b/requirements.txt index 63e0cae..49cfe19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ SQLAlchemy[asyncio,aiomysql,postgresql_asyncpg,aiosqlite]==2.0.15 pydantic==1.10.7 platformdirs==3.2.0 alembic==1.11.1 +scheduler==0.8.4 # CockroachDB support sqlalchemy-cockroachdb==2.0.1 diff --git a/spoolman/api/v1/models.py b/spoolman/api/v1/models.py index d4417c7..7b5bd88 100644 --- a/spoolman/api/v1/models.py +++ b/spoolman/api/v1/models.py @@ -159,3 +159,12 @@ class Spool(BaseModel): class HealthCheck(BaseModel): status: str = Field(example="healthy") + + +class BackupResponse(BaseModel): + success: bool = Field(description="Whether the backup was created successfully.", example=True) + path: Optional[str] = Field( + default=None, + description="Path to the created backup file.", + example="/home/app/.local/share/spoolman/backups/spoolman.db", + ) diff --git a/spoolman/api/v1/router.py b/spoolman/api/v1/router.py index 6915b5c..794c4b0 100644 --- a/spoolman/api/v1/router.py +++ b/spoolman/api/v1/router.py @@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse from starlette.requests import Request from starlette.responses import Response +from spoolman.database.database import backup_task from spoolman.exceptions import ItemNotFoundError from . import filament, models, spool, vendor @@ -39,6 +40,17 @@ async def health() -> models.HealthCheck: return models.HealthCheck(status="healthy") +# Add endpoint for triggering a db backup +@app.post("/backup", summary="Trigger a database backup. Only applicable for SQLite databases.") +async def backup() -> models.BackupResponse: + """Trigger a database backup.""" + path = await backup_task() + if path is None: + return models.BackupResponse(success=False) + return models.BackupResponse(success=True, path=str(path)) + + +# Add routers app.include_router(filament.router) app.include_router(spool.router) app.include_router(vendor.router) diff --git a/spoolman/database/database.py b/spoolman/database/database.py index 996ae1c..eb3c823 100644 --- a/spoolman/database/database.py +++ b/spoolman/database/database.py @@ -1,9 +1,15 @@ """SQLAlchemy database setup.""" +import datetime import logging +import shutil +import sqlite3 from collections.abc import AsyncGenerator -from typing import Optional +from os import PathLike +from pathlib import Path +from typing import Optional, Union +from scheduler.asyncio.scheduler import Scheduler from sqlalchemy import URL from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine @@ -56,6 +62,14 @@ class Database: """Construct the Database wrapper and set config parameters.""" self.connection_url = connection_url + def is_file_based_sqlite(self: "Database") -> bool: + """Return True if the database is file based.""" + return ( + self.connection_url.drivername[:6] == "sqlite" + and self.connection_url.database is not None + and self.connection_url.database != ":memory:" + ) + def connect(self: "Database") -> None: """Connect to the database.""" if env.get_logging_level() == logging.DEBUG: @@ -72,6 +86,69 @@ class Database: ) self.session_maker = async_sessionmaker(self.engine, autocommit=False, autoflush=True, expire_on_commit=False) + def backup(self: "Database", target_path: Union[str, PathLike[str]]) -> None: + """Backup the database.""" + if not self.is_file_based_sqlite() or self.connection_url.database is None: + return + + logger.info("Backing up SQLite database to %s", target_path) + + def progress(_: int, remaining: int, total: int) -> None: + logger.info("Copied %d of %d pages.", total - remaining, total) + + if self.connection_url.database == target_path: + raise ValueError("Cannot backup database to itself.") + if Path(target_path).exists(): + raise ValueError("Backup target file already exists.") + + with sqlite3.connect(self.connection_url.database) as src, sqlite3.connect(target_path) as dst: + src.backup(dst, pages=1, progress=progress) + + logger.info("Backup complete.") + + def backup_and_rotate( + self: "Database", + backup_folder: Union[str, PathLike[str]], + num_backups: int = 5, + ) -> Optional[Path]: + """Backup the database and rotate existing backups. + + Args: + backup_folder: The folder to store the backups in. + num_backups: The number of backups to keep. + + Returns: + The path to the created backup or None if no backup was created. + """ + if not self.is_file_based_sqlite() or self.connection_url.database is None: + logger.info("Skipping backup as the database is not SQLite.") + return None + + backup_folder = Path(backup_folder) + backup_folder.mkdir(parents=True, exist_ok=True) + + # Delete oldest backup + backup_path = backup_folder.joinpath(f"spoolman.db.{num_backups}") + if backup_path.exists(): + logger.info("Deleting oldest backup %s", backup_path) + backup_path.unlink() + + # Rotate existing backups + for i in range(num_backups - 1, -1, -1): + if i == 0: + backup_path = backup_folder.joinpath("spoolman.db") + else: + backup_path = backup_folder.joinpath(f"spoolman.db.{i}") + if backup_path.exists(): + logger.debug("Rotating backup %s to %s", backup_path, backup_folder.joinpath(f"spoolman.db.{i + 1}")) + shutil.move(backup_path, backup_folder.joinpath(f"spoolman.db.{i + 1}")) + + # Create new backup + backup_path = backup_folder.joinpath("spoolman.db") + self.backup(backup_path) + + return backup_path + __db: Optional[Database] = None @@ -87,8 +164,39 @@ def setup_db(connection_url: URL) -> None: __db.connect() +async def backup_task() -> Optional[Path]: + """Backup the database and rotate existing backups. + + Returns: + The path to the created backup or None if no backup was created. + """ + if __db is None: + raise RuntimeError("DB is not setup.") + return __db.backup_and_rotate(env.get_data_dir().joinpath("backups"), num_backups=5) + + +def schedule_tasks(scheduler: Scheduler) -> None: + """Schedule tasks to be executed by the provided scheduler. + + Args: + scheduler: The scheduler to use for scheduling tasks. + """ + if __db is None: + raise RuntimeError("DB is not setup.") + if not env.is_automatic_backup_enabled(): + return + if "sqlite" in __db.connection_url.drivername: + logger.info("Scheduling automatic database backup for midnight.") + # Schedule for midnight + scheduler.daily(datetime.time(hour=0, minute=0, second=0), backup_task) # type: ignore[arg-type] + + async def get_db_session() -> AsyncGenerator[AsyncSession, None]: - """Get a DB session to be used with FastAPI's dependency system.""" + """Get a DB session to be used with FastAPI's dependency system. + + Yields: + The database session. + """ if __db is None or __db.session_maker is None: raise RuntimeError("DB is not setup.") async with __db.session_maker() as session: diff --git a/spoolman/env.py b/spoolman/env.py index 0d68d92..0b17486 100644 --- a/spoolman/env.py +++ b/spoolman/env.py @@ -196,6 +196,24 @@ def is_debug_mode() -> bool: raise ValueError(f"Failed to parse SPOOLMAN_DEBUG_MODE variable: Unknown debug mode '{debug_mode}'.") +def is_automatic_backup_enabled() -> bool: + """Get whether automatic backup is enabled from environment variables. + + Returns True if no environment variable was set for automatic backup. + + Returns: + bool: Whether automatic backup is enabled. + """ + automatic_backup = os.getenv("SPOOLMAN_AUTOMATIC_BACKUP", "TRUE").upper() + if automatic_backup == "FALSE" or automatic_backup == "0": + return False + if automatic_backup == "TRUE" or automatic_backup == "1": + return True + raise ValueError( + f"Failed to parse SPOOLMAN_AUTOMATIC_BACKUP variable: Unknown automatic backup '{automatic_backup}'.", + ) + + def get_data_dir() -> Path: """Get the data directory. @@ -205,3 +223,15 @@ def get_data_dir() -> Path: data_dir = Path(user_data_dir("spoolman")) data_dir.mkdir(parents=True, exist_ok=True) return data_dir + + +def get_backups_dir() -> Path: + """Get the backups directory. + + Returns: + Path: The backups directory. + """ + data_dir = get_data_dir() + backups_dir = data_dir.joinpath("backups") + backups_dir.mkdir(parents=True, exist_ok=True) + return backups_dir diff --git a/spoolman/main.py b/spoolman/main.py index 6285110..d9ee574 100644 --- a/spoolman/main.py +++ b/spoolman/main.py @@ -11,10 +11,11 @@ import uvicorn from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles +from scheduler.asyncio.scheduler import Scheduler from spoolman import env from spoolman.api.v1.router import app as v1_app -from spoolman.database.database import get_connection_url, setup_db +from spoolman.database import database # Define a file logger with log rotation log_file = env.get_data_dir().joinpath("spoolman.log") @@ -75,7 +76,7 @@ if env.is_debug_mode(): async def startup() -> None: """Run the service's startup sequence.""" logger.info("Setting up database...") - setup_db(get_connection_url()) + database.setup_db(database.get_connection_url()) logger.info("Performing migrations...") # Run alembic in a subprocess. @@ -84,6 +85,10 @@ async def startup() -> None: project_root = Path(__file__).parent.parent subprocess.run(["alembic", "upgrade", "head"], check=True, cwd=project_root) # noqa: S603, S607, ASYNC101 + # Setup scheduler + schedule = Scheduler() + database.schedule_tasks(schedule) + logger.info("Startup complete.") diff --git a/tests_integration/docker-compose-cockroachdb.yml b/tests_integration/docker-compose-cockroachdb.yml index 3483d55..bc1ce04 100644 --- a/tests_integration/docker-compose-cockroachdb.yml +++ b/tests_integration/docker-compose-cockroachdb.yml @@ -33,5 +33,7 @@ services: image: donkie/spoolman-tester:latest volumes: - ./tests:/tester/tests + environment: + - DB_TYPE=cockroachdb depends_on: - spoolman diff --git a/tests_integration/docker-compose-mariadb.yml b/tests_integration/docker-compose-mariadb.yml index bee7d8f..517503c 100644 --- a/tests_integration/docker-compose-mariadb.yml +++ b/tests_integration/docker-compose-mariadb.yml @@ -37,5 +37,7 @@ services: image: donkie/spoolman-tester:latest volumes: - ./tests:/tester/tests + environment: + - DB_TYPE=mysql depends_on: - spoolman diff --git a/tests_integration/docker-compose-postgres.yml b/tests_integration/docker-compose-postgres.yml index 708dd52..f1e0980 100644 --- a/tests_integration/docker-compose-postgres.yml +++ b/tests_integration/docker-compose-postgres.yml @@ -20,5 +20,7 @@ services: image: donkie/spoolman-tester:latest volumes: - ./tests:/tester/tests + environment: + - DB_TYPE=postgres depends_on: - spoolman diff --git a/tests_integration/docker-compose-sqlite.yml b/tests_integration/docker-compose-sqlite.yml index a76eec5..f2eafdc 100644 --- a/tests_integration/docker-compose-sqlite.yml +++ b/tests_integration/docker-compose-sqlite.yml @@ -8,5 +8,7 @@ services: image: donkie/spoolman-tester:latest volumes: - ./tests:/tester/tests + environment: + - DB_TYPE=sqlite depends_on: - spoolman diff --git a/tests_integration/tests/conftest.py b/tests_integration/tests/conftest.py index 51e3e97..8e91d1f 100644 --- a/tests_integration/tests/conftest.py +++ b/tests_integration/tests/conftest.py @@ -1,6 +1,8 @@ """Test fixtures for integration tests.""" +import os import time +from enum import Enum from typing import Any import httpx @@ -11,6 +13,27 @@ TIMEOUT = 10 URL = "http://spoolman:8000" +class DbType(str, Enum): + """Enum for database types.""" + + SQLITE = "sqlite" + POSTGRES = "postgres" + MYSQL = "mysql" + COCKROACHDB = "cockroachdb" + + +def get_db_type() -> DbType: + """Return the database type from environment variables.""" + env_db_type = os.environ.get("DB_TYPE") + if env_db_type is None: + raise RuntimeError("DB_TYPE environment variable not set") + try: + db_type = DbType(env_db_type) + except ValueError as e: + raise RuntimeError(f"Unknown database type: {env_db_type}") from e + return db_type + + @pytest.fixture(scope="session", autouse=True) def _wait_for_server(): # noqa: ANN202 """Wait for the server to start up.""" diff --git a/tests_integration/tests/test_backup.py b/tests_integration/tests/test_backup.py new file mode 100644 index 0000000..f795b42 --- /dev/null +++ b/tests_integration/tests/test_backup.py @@ -0,0 +1,18 @@ +"""Integration tests for the Vendor API endpoint.""" + +import httpx + +from .conftest import DbType, get_db_type + +URL = "http://spoolman:8000" + + +def test_backup(): + """Test triggering an automatic database backup.""" + if get_db_type() != DbType.SQLITE: + return + + # Trigger backup + result = httpx.post(f"{URL}/api/v1/backup") + result.raise_for_status() + assert result.json()["success"] is True