Added automatic nightly SQLite backups

Resolves #22

README.md

@@ -38,6 +38,8 @@ services:
       - ./data:/home/app/.local/share/spoolman
     ports:
       - "7912:8000"
+    environment:
+      - TZ=Europe/Stockholm # Optional, defaults to UTC
 ```
 With this example, you should first create a folder called `data` in the same directory as the docker-compose.yml, then you should run `chown 1000:1000 data` on it in order to give it the correct permissions for the user inside the docker container.
 
@@ -56,6 +58,7 @@ If you want to connect with an external database instead, specify the `SPOOLMAN_
 | SPOOLMAN_DB_PASSWORD      | Database password                                                                                                            |
 | SPOOLMAN_DB_QUERY         | Query parameters for the database connection, e.g. set to `unix_socket=/path/to/mysql.sock` to connect using a MySQL socket. |
 | SPOOLMAN_LOGGING_LEVEL    | Logging level, any of: `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`, defaults to `INFO`.                                  |
+| SPOOLMAN_AUTOMATIC_BACKUP | Automatic nightly DB backups for SQLite databases. Enabled by default, set to `FALSE` to disable.                            |
 
 ## Configuration
 ### Moonraker

requirements.txt

@@ -8,6 +8,7 @@ SQLAlchemy[asyncio,aiomysql,postgresql_asyncpg,aiosqlite]==2.0.15
 pydantic==1.10.7
 platformdirs==3.2.0
 alembic==1.11.1
+scheduler==0.8.4
 
 # CockroachDB support
 sqlalchemy-cockroachdb==2.0.1

spoolman/api/v1/models.py

@@ -159,3 +159,12 @@ class Spool(BaseModel):
 
 class HealthCheck(BaseModel):
     status: str = Field(example="healthy")
+
+
+class BackupResponse(BaseModel):
+    success: bool = Field(description="Whether the backup was created successfully.", example=True)
+    path: Optional[str] = Field(
+        default=None,
+        description="Path to the created backup file.",
+        example="/home/app/.local/share/spoolman/backups/spoolman.db",
+    )

spoolman/api/v1/router.py

@@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse
 from starlette.requests import Request
 from starlette.responses import Response
 
+from spoolman.database.database import backup_task
 from spoolman.exceptions import ItemNotFoundError
 
 from . import filament, models, spool, vendor
@@ -39,6 +40,17 @@ async def health() -> models.HealthCheck:
     return models.HealthCheck(status="healthy")
 
 
+# Add endpoint for triggering a db backup
+@app.post("/backup", summary="Trigger a database backup. Only applicable for SQLite databases.")
+async def backup() -> models.BackupResponse:
+    """Trigger a database backup."""
+    path = await backup_task()
+    if path is None:
+        return models.BackupResponse(success=False)
+    return models.BackupResponse(success=True, path=str(path))
+
+
+# Add routers
 app.include_router(filament.router)
 app.include_router(spool.router)
 app.include_router(vendor.router)

spoolman/database/database.py

@@ -1,9 +1,15 @@
 """SQLAlchemy database setup."""
 
+import datetime
 import logging
+import shutil
+import sqlite3
 from collections.abc import AsyncGenerator
-from typing import Optional
+from os import PathLike
+from pathlib import Path
+from typing import Optional, Union
 
+from scheduler.asyncio.scheduler import Scheduler
 from sqlalchemy import URL
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
 
@@ -56,6 +62,14 @@ class Database:
         """Construct the Database wrapper and set config parameters."""
         self.connection_url = connection_url
 
+    def is_file_based_sqlite(self: "Database") -> bool:
+        """Return True if the database is file based."""
+        return (
+            self.connection_url.drivername[:6] == "sqlite"
+            and self.connection_url.database is not None
+            and self.connection_url.database != ":memory:"
+        )
+
     def connect(self: "Database") -> None:
         """Connect to the database."""
         if env.get_logging_level() == logging.DEBUG:
@@ -72,6 +86,69 @@ class Database:
         )
         self.session_maker = async_sessionmaker(self.engine, autocommit=False, autoflush=True, expire_on_commit=False)
 
+    def backup(self: "Database", target_path: Union[str, PathLike[str]]) -> None:
+        """Backup the database."""
+        if not self.is_file_based_sqlite() or self.connection_url.database is None:
+            return
+
+        logger.info("Backing up SQLite database to %s", target_path)
+
+        def progress(_: int, remaining: int, total: int) -> None:
+            logger.info("Copied %d of %d pages.", total - remaining, total)
+
+        if self.connection_url.database == target_path:
+            raise ValueError("Cannot backup database to itself.")
+        if Path(target_path).exists():
+            raise ValueError("Backup target file already exists.")
+
+        with sqlite3.connect(self.connection_url.database) as src, sqlite3.connect(target_path) as dst:
+            src.backup(dst, pages=1, progress=progress)
+
+        logger.info("Backup complete.")
+
+    def backup_and_rotate(
+        self: "Database",
+        backup_folder: Union[str, PathLike[str]],
+        num_backups: int = 5,
+    ) -> Optional[Path]:
+        """Backup the database and rotate existing backups.
+
+        Args:
+            backup_folder: The folder to store the backups in.
+            num_backups: The number of backups to keep.
+
+        Returns:
+            The path to the created backup or None if no backup was created.
+        """
+        if not self.is_file_based_sqlite() or self.connection_url.database is None:
+            logger.info("Skipping backup as the database is not SQLite.")
+            return None
+
+        backup_folder = Path(backup_folder)
+        backup_folder.mkdir(parents=True, exist_ok=True)
+
+        # Delete oldest backup
+        backup_path = backup_folder.joinpath(f"spoolman.db.{num_backups}")
+        if backup_path.exists():
+            logger.info("Deleting oldest backup %s", backup_path)
+            backup_path.unlink()
+
+        # Rotate existing backups
+        for i in range(num_backups - 1, -1, -1):
+            if i == 0:
+                backup_path = backup_folder.joinpath("spoolman.db")
+            else:
+                backup_path = backup_folder.joinpath(f"spoolman.db.{i}")
+            if backup_path.exists():
+                logger.debug("Rotating backup %s to %s", backup_path, backup_folder.joinpath(f"spoolman.db.{i + 1}"))
+                shutil.move(backup_path, backup_folder.joinpath(f"spoolman.db.{i + 1}"))
+
+        # Create new backup
+        backup_path = backup_folder.joinpath("spoolman.db")
+        self.backup(backup_path)
+
+        return backup_path
+
 
 __db: Optional[Database] = None
 
@@ -87,8 +164,39 @@ def setup_db(connection_url: URL) -> None:
     __db.connect()
 
 
+async def backup_task() -> Optional[Path]:
+    """Backup the database and rotate existing backups.
+
+    Returns:
+        The path to the created backup or None if no backup was created.
+    """
+    if __db is None:
+        raise RuntimeError("DB is not setup.")
+    return __db.backup_and_rotate(env.get_data_dir().joinpath("backups"), num_backups=5)
+
+
+def schedule_tasks(scheduler: Scheduler) -> None:
+    """Schedule tasks to be executed by the provided scheduler.
+
+    Args:
+        scheduler: The scheduler to use for scheduling tasks.
+    """
+    if __db is None:
+        raise RuntimeError("DB is not setup.")
+    if not env.is_automatic_backup_enabled():
+        return
+    if "sqlite" in __db.connection_url.drivername:
+        logger.info("Scheduling automatic database backup for midnight.")
+        # Schedule for midnight
+        scheduler.daily(datetime.time(hour=0, minute=0, second=0), backup_task)  # type: ignore[arg-type]
+
+
 async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
-    """Get a DB session to be used with FastAPI's dependency system."""
+    """Get a DB session to be used with FastAPI's dependency system.
+
+    Yields:
+        The database session.
+    """
     if __db is None or __db.session_maker is None:
         raise RuntimeError("DB is not setup.")
     async with __db.session_maker() as session:

spoolman/env.py

@@ -196,6 +196,24 @@ def is_debug_mode() -> bool:
     raise ValueError(f"Failed to parse SPOOLMAN_DEBUG_MODE variable: Unknown debug mode '{debug_mode}'.")
 
 
+def is_automatic_backup_enabled() -> bool:
+    """Get whether automatic backup is enabled from environment variables.
+
+    Returns True if no environment variable was set for automatic backup.
+
+    Returns:
+        bool: Whether automatic backup is enabled.
+    """
+    automatic_backup = os.getenv("SPOOLMAN_AUTOMATIC_BACKUP", "TRUE").upper()
+    if automatic_backup == "FALSE" or automatic_backup == "0":
+        return False
+    if automatic_backup == "TRUE" or automatic_backup == "1":
+        return True
+    raise ValueError(
+        f"Failed to parse SPOOLMAN_AUTOMATIC_BACKUP variable: Unknown automatic backup '{automatic_backup}'.",
+    )
+
+
 def get_data_dir() -> Path:
     """Get the data directory.
 
@@ -205,3 +223,15 @@ def get_data_dir() -> Path:
     data_dir = Path(user_data_dir("spoolman"))
     data_dir.mkdir(parents=True, exist_ok=True)
     return data_dir
+
+
+def get_backups_dir() -> Path:
+    """Get the backups directory.
+
+    Returns:
+        Path: The backups directory.
+    """
+    data_dir = get_data_dir()
+    backups_dir = data_dir.joinpath("backups")
+    backups_dir.mkdir(parents=True, exist_ok=True)
+    return backups_dir

spoolman/main.py

@@ -11,10 +11,11 @@ import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
+from scheduler.asyncio.scheduler import Scheduler
 
 from spoolman import env
 from spoolman.api.v1.router import app as v1_app
-from spoolman.database.database import get_connection_url, setup_db
+from spoolman.database import database
 
 # Define a file logger with log rotation
 log_file = env.get_data_dir().joinpath("spoolman.log")
@@ -75,7 +76,7 @@ if env.is_debug_mode():
 async def startup() -> None:
     """Run the service's startup sequence."""
     logger.info("Setting up database...")
-    setup_db(get_connection_url())
+    database.setup_db(database.get_connection_url())
 
     logger.info("Performing migrations...")
     # Run alembic in a subprocess.
@@ -84,6 +85,10 @@ async def startup() -> None:
     project_root = Path(__file__).parent.parent
     subprocess.run(["alembic", "upgrade", "head"], check=True, cwd=project_root)  # noqa: S603, S607, ASYNC101
 
+    # Setup scheduler
+    schedule = Scheduler()
+    database.schedule_tasks(schedule)
+
     logger.info("Startup complete.")
 
 

tests_integration/docker-compose-cockroachdb.yml

@@ -33,5 +33,7 @@ services:
     image: donkie/spoolman-tester:latest
     volumes:
       - ./tests:/tester/tests
+    environment:
+      - DB_TYPE=cockroachdb
     depends_on:
       - spoolman

tests_integration/docker-compose-mariadb.yml

@@ -37,5 +37,7 @@ services:
     image: donkie/spoolman-tester:latest
     volumes:
       - ./tests:/tester/tests
+    environment:
+      - DB_TYPE=mysql
     depends_on:
       - spoolman

tests_integration/docker-compose-postgres.yml

@@ -20,5 +20,7 @@ services:
     image: donkie/spoolman-tester:latest
     volumes:
       - ./tests:/tester/tests
+    environment:
+      - DB_TYPE=postgres
     depends_on:
       - spoolman

tests_integration/docker-compose-sqlite.yml

@@ -8,5 +8,7 @@ services:
     image: donkie/spoolman-tester:latest
     volumes:
       - ./tests:/tester/tests
+    environment:
+      - DB_TYPE=sqlite
     depends_on:
       - spoolman

tests_integration/tests/conftest.py

@@ -1,6 +1,8 @@
 """Test fixtures for integration tests."""
 
+import os
 import time
+from enum import Enum
 from typing import Any
 
 import httpx
@@ -11,6 +13,27 @@ TIMEOUT = 10
 URL = "http://spoolman:8000"
 
 
+class DbType(str, Enum):
+    """Enum for database types."""
+
+    SQLITE = "sqlite"
+    POSTGRES = "postgres"
+    MYSQL = "mysql"
+    COCKROACHDB = "cockroachdb"
+
+
+def get_db_type() -> DbType:
+    """Return the database type from environment variables."""
+    env_db_type = os.environ.get("DB_TYPE")
+    if env_db_type is None:
+        raise RuntimeError("DB_TYPE environment variable not set")
+    try:
+        db_type = DbType(env_db_type)
+    except ValueError as e:
+        raise RuntimeError(f"Unknown database type: {env_db_type}") from e
+    return db_type
+
+
 @pytest.fixture(scope="session", autouse=True)
 def _wait_for_server():  # noqa: ANN202
     """Wait for the server to start up."""

tests_integration/tests/test_backup.py

@@ -0,0 +1,18 @@
+"""Integration tests for the Vendor API endpoint."""
+
+import httpx
+
+from .conftest import DbType, get_db_type
+
+URL = "http://spoolman:8000"
+
+
+def test_backup():
+    """Test triggering an automatic database backup."""
+    if get_db_type() != DbType.SQLITE:
+        return
+
+    # Trigger backup
+    result = httpx.post(f"{URL}/api/v1/backup")
+    result.raise_for_status()
+    assert result.json()["success"] is True