From 0e6506d97fb51f00ba58afbd70da9c31b673432d Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 27 Aug 2026 04:41:30 +0000 Subject: [PATCH] Update state.json and development plan to reflect actual progress (~15%) - state.json: Phase 5 status changed to pending, next steps updated - DEVELOPMENT_PLAN.md v1.1: Added actual progress, verified current state, renumbered phases 5-9, marked all remaining work as PENDING - Deleted outdated plans (BACKEND_TEST_PLAN, DATA_LOADING_VERIFICATION, DOCKER_MIGRATION_PLAN, ENDPOINT_AUDIT, ROADMAP, SIMPLIFICATION_PLAN, STATEMENT_OF_INTENT, backend/TEST_PLAN) - Added new files: DEVELOPMENT_PLAN.md, backend/ARCHITECTURAL_REPORT.md, backend/SIMPLIFICATION_SUMMARY.md, backend/run.py, backend/.dockerignore --- BACKEND_TEST_PLAN.md | 466 -------- DATA_LOADING_VERIFICATION.md | 117 -- DEVELOPMENT_PLAN.md | 690 ++++++++++++ DOCKER_MIGRATION_PLAN.md | 96 -- ENDPOINT_AUDIT.md | 262 ----- ROADMAP.md | 484 --------- SIMPLIFICATION_PLAN.md | 200 ---- STATEMENT_OF_INTENT.md | 246 ----- backend/.dockerignore | 50 + backend/ARCHITECTURAL_REPORT.md | 1668 +++++++++++++++++++++++++++++ backend/Dockerfile | 80 +- backend/SIMPLIFICATION_SUMMARY.md | 342 ++++++ backend/TEST_PLAN.md | 142 --- backend/fix_init.py | 62 ++ backend/run.py | 50 + backend/state.json | 20 +- docker-compose.yml | 181 +--- 17 files changed, 2938 insertions(+), 2218 deletions(-) delete mode 100644 BACKEND_TEST_PLAN.md delete mode 100644 DATA_LOADING_VERIFICATION.md create mode 100644 DEVELOPMENT_PLAN.md delete mode 100644 DOCKER_MIGRATION_PLAN.md delete mode 100644 ENDPOINT_AUDIT.md delete mode 100644 ROADMAP.md delete mode 100644 SIMPLIFICATION_PLAN.md delete mode 100644 STATEMENT_OF_INTENT.md create mode 100644 backend/.dockerignore create mode 100644 backend/ARCHITECTURAL_REPORT.md create mode 100644 backend/SIMPLIFICATION_SUMMARY.md delete mode 100644 backend/TEST_PLAN.md create mode 100644 backend/fix_init.py create mode 100644 backend/run.py diff --git a/BACKEND_TEST_PLAN.md b/BACKEND_TEST_PLAN.md deleted file mode 100644 index e730fdf..0000000 --- a/BACKEND_TEST_PLAN.md +++ /dev/null @@ -1,466 +0,0 @@ -# Backend Test Plan (Updated) - -## Overview -This test plan is designed for iterative execution using sub-agents, with each sub-agent handling a specific phase to avoid exceeding the 300,000 token context limit. Each phase focuses on a distinct subsystem and includes specific test cases and verification steps. - -**Last Updated:** 2026-06-08 -**Status:** Phase 1 Complete - Database & Migration Tests Verified - ---- - -## Test Phases - -### Phase 1: Database & Migration Tests ✅ COMPLETE -**Scope:** Alembic migrations, database schema, model relationships -**Sub-agent Task:** Verify all migrations run correctly and database schema is consistent -**Status:** ✅ Verified - All migrations functionally correct - -#### Actual Migration Structure: - -**Migration 000 (`000_base_tables.py`)** - Creates 8 base tables: -- `mtgonline_users` (base table) -- `mtgonline_decklist_folders` (FK to mtgonline_users.id, self-ref) -- `mtgonline_decklist_files` (FK to mtgonline_decklist_folders.id, mtgonline_users.id) -- `mtgonline_rooms` (base table) -- `mtgonline_rooms_gametypes` (FK to mtgonline_rooms.id) -- `mtgonline_bans` (FK to mtgonline_users.id) -- `mtgonline_log` (FK to mtgonline_rooms.id, mtgonline_users.id) -- `mtgonline_audit` (FK to mtgonline_users.id) - -**Migration 001 (`001_initial_user_schema.py`)** - Creates 15 user-related tables: -- `user_sessions` (FK to mtgonline_users.id) -- `deck_versions` (FK to mtgonline_decklist_files.id) -- `game_replays` (FK to mtgonline_rooms.id) -- `replay_players` (FK to game_replays.id, mtgonline_users.id, mtgonline_decklist_files.id) -- `game_outcomes` (FK to mtgonline_users.id, game_replays.game_uuid) -- `user_statistics` (PK: user_id, FK to mtgonline_users.id) -- `user_card_collection` (FK to mtgonline_users.id) -- `card_wishlist` (FK to mtgonline_users.id) -- `user_groups` (FK to mtgonline_users.id) -- `group_members` (FK to user_groups.id, mtgonline_users.id) -- `group_chat_messages` (FK to user_groups.id, mtgonline_users.id) -- `user_networks` (FK to mtgonline_users.id) -- `network_members` (FK to user_networks.id, mtgonline_users.id) -- `user_preferences` (PK: user_id, FK to mtgonline_users.id) -- `user_activity_log` (FK to mtgonline_users.id) - -**Migration 002 (`002_user_deck_building_tables.py`)** - Creates: -- `user_decks` (FK to mtgonline_users.id, mtgonline_decklist_folders.id) - -**Migration 003 (`003_mtgonline_cards_table.py`)** - Creates: -- `mtgonline_cards` (base table) -- `user_deck_cards` (FK to user_decks.id, mtgonline_cards.id) -- `deck_precedents` (FK to mtgonline_users.id) -- `deck_precedent_cards` (FK to deck_precedents.id, mtgonline_cards.id) -- `card_suggestions` (FK to user_decks.id, mtgonline_cards.id, self-ref) - -**Migration 004 (`004_card_import_table.py`)** - Creates: -- `user_card_imports` (FK to mtgonline_users.id) - -**Migration 005 (`005_missing_tables.py`)** - Creates: -- `mtg_sets` (base table) -- `mtg_cards` (FK to mtg_sets.id) -- `mtg_cards_mirror` (base table) -- `card_import_batches` (FK to mtgonline_users.id) -- `user_card_imports_confirmed` (FK to mtgonline_users.id, card_import_batches.id) -- `deck_card_links` (FK to mtgonline_decklist_files.id, mtg_cards_mirror.id) - -#### Verification Results: -- ✅ All 6 migrations properly linked (000 → 001 → 002 → 003 → 004 → 005) -- ✅ All foreign keys reference tables created in same or earlier migrations -- ✅ No circular dependencies -- ✅ All indexes created on FK columns -- ✅ All unique constraints valid -- ✅ Downgrade functions properly drop tables in reverse dependency order -- ✅ All model imports in `env.py` and `__init__.py` consistent - -**Note:** Migration 000 is NOT empty (creates 8 base tables). This is intentional and correct. - ---- - -### Phase 2: Model Layer Tests -**Scope:** SQLAlchemy models, relationships, validation -**Sub-agent Task:** Verify all model definitions are correct and consistent - -#### Actual Model Structure: - -**1. Core Models (`app/models/models.py`)** -- `User` (`mtgonline_users`) - username, email, password_hash, salt, country, real_name, avatar_bmp, privlevel, is_active, is_banned, ban_reason, ban_ends, vip_status, vip_expiry, creation_date, last_login -- `MtonlineCard` (`mtgonline_cards`) - source_id, name, mana_cost, type_line, oracle_text, power, toughness, rarity, layout, artist, flavor_text, numbers, identifiers, images, image, set_code, set_name, card_parts, keywords, legalities, synced_at, created_at -- `DecklistFolder` (`mtgonline_decklist_folders`) - owner_id, name, parent_id, creation_date -- `DecklistFile` (`mtgonline_decklist_files`) - folder_id, owner_id, name, content, format, status, creation_date -- `Room` (`mtgonline_rooms`) - name, description, is_password_protected, password_hash, creation_date -- `RoomGameType` (`mtgonline_rooms_gametypes`) - room_id, name, description -- `Ban` (`mtgonline_bans`) - user_id, server_id, reason, moderators, ip_address, expiration_time, active, creation_date -- `GameLog` (`mtgonline_log`) - room_id, player_id, message, timestamp -- `AuditLog` (`mtgonline_audit`) - admin_id, action_type, target_user_id, details, ip_address, timestamp - -**2. MTG Models (`app/models/mtg_models.py`)** -- `MtgSet` (`mtg_sets`) - code, name, type, release_date, base_set_size, total_size, is_foil_only, is_non_foil_only, digital, icon_svg_url, parent_code, mtgo_code, image, updated_at -- `MtgCard` (`mtg_cards`) - set_id, name, mana_cost, type_line, oracle_text, power, toughness, rarity, layout, artist, flavor_text, numbers, identifiers, images, image, updated_at - -**3. Mirror Models (`app/models/mirror_models.py`)** -- `MtgCardMirror` (`mtg_cards_mirror`) - source_id, name, mana_cost, type_line, oracle_text, power, toughness, rarity, layout, artist, flavor_text, numbers, identifiers, images, image, card_parts, keywords, legalities, set_code, set_name, synced_at, created_at -- `DeckCardLink` (`deck_card_links`) - deck_id, card_id, quantity, zone - -**4. User Data Models (`app/models/user_data.py`)** -- `UserSession` (`user_sessions`) - user_id, session_token_hash, ip_address, user_agent, created_at, expires_at, is_active -- `DeckVersion` (`deck_versions`) - deck_id, version_number, content, status, comment, created_at -- `GameReplay` (`game_replays`) - game_uuid, room_id, game_type, format, duration_seconds, start_time, end_time, status, replay_data, created_at, updated_at -- `ReplayPlayer` (`replay_players`) - replay_id, user_id, position, deck_id, won, lost, concession, turn_one, created_at -- `GameOutcome` (`game_outcomes`) - user_id, game_uuid, outcome, opponent_id, format, rating_before, rating_after, rating_change, created_at -- `UserStatistics` (`user_statistics`) - user_id (PK), total_games, total_wins, total_losses, total_concessions, win_rate, current_streak, best_streak, average_rating, last_game_date, updated_at -- `UserCardCollection` (`user_card_collection`) - user_id, card_id, quantity, condition, language, is_foil, is_alt_art, acquired_date, acquisition_method, notes, created_at, updated_at -- `CardWishlist` (`card_wishlist`) - user_id, card_id, max_price, notes, created_at -- `UserGroup` (`user_groups`) - name, description, owner_id, is_public, max_members, created_at, updated_at -- `GroupMember` (`group_members`) - group_id, user_id, role, joined_at -- `GroupChatMessage` (`group_chat_messages`) - group_id, sender_id, message, created_at -- `UserNetwork` (`user_networks`) - name, description, creator_id, is_public, created_at -- `NetworkMember` (`network_members`) - network_id, user_id, role, joined_at -- `UserPreference` (`user_preferences`) - user_id (PK), theme, notifications_enabled, email_notifications, auto_save_decks, default_format, language, updated_at -- `UserActivityLog` (`user_activity_log`) - user_id, activity_type, activity_data, ip_address, created_at - -**5. User Deck Models (`app/models/user_deck.py`)** -- `UserDeck` (`user_decks`) - user_id, name, status, folder_id, format, notes, is_precedent, precedent_name, created_at, updated_at -- `UserDeckCard` (`user_deck_cards`) - deck_id, card_id, quantity, zone, position -- `DeckPrecedent` (`deck_precedents`) - name, description, format, is_public, created_by, created_at, updated_at -- `DeckPrecedentCard` (`deck_precedent_cards`) - precedent_id, card_id, quantity, zone -- `CardSuggestion` (`card_suggestions`) - deck_id, card_id, source_card_id, suggestion_type, confidence, notes, created_at - -**6. Card Import Models (`app/models/card_import.py`)** -- `CardImportBatch` (`card_import_batches`) - user_id, filename, file_type, file_size, status, total_cards, matched_cards, unmatched_cards, match_results, error_message, created_at, updated_at -- `UserCardImportRecord` (`user_card_imports_confirmed`) - user_id, batch_id, is_confirmed, confirmed_at - -#### Test Cases: -1. **Verify all models have proper `__tablename__`** -2. **Verify all foreign keys reference correct tables** -3. **Verify all relationships are bidirectional where needed** -4. **Verify cascade delete behavior** -5. **Verify indexes on FK columns** -6. **Verify unique constraints** -7. **Verify model imports in `app/models/__init__.py`** -8. **Check for missing fields compared to migration definitions** - -**Verification:** All models compile without errors, relationships are correct, no missing fields. - ---- - -### Phase 3: Schema Layer Tests -**Scope:** Pydantic schemas, request/response validation -**Sub-agent Task:** Verify all schema definitions are correct - -#### Actual Schema Structure: - -**1. Core Schemas (`app/schemas/schemas.py`)** -- **Authentication:** `LoginRequest`, `LoginResponse`, `RefreshTokenRequest`, `TokenResponse` -- **User:** `UserBase`, `UserCreate`, `UserUpdate`, `UserResponse` -- **Deck:** `DeckCreate`, `DeckUpdate`, `DeckResponse`, `FolderCreate`, `FolderResponse` -- **Game:** `GameCreate`, `GameResponse` -- **Room:** `RoomResponse` -- **Ban:** `BanCreate`, `BanResponse` -- **Error:** `ErrorResponse`, `ValidationErrorResponse` -- **Pagination:** `PaginationParams`, `PaginatedResponse` -- **Card Mirror:** `CardMirrorResponse`, `DeckCardLinkResponse`, `DeckWithCardsResponse` - -**2. User Data Schemas (`app/schemas/user_data_schemas.py`)** -- **Enums:** `DeckVersionStatus`, `GameReplayStatus`, `GameOutcomeType`, `GroupMemberRole`, `NetworkMemberRole`, `UserPreferenceTheme`, `ActivityType` -- **Session:** `SessionResponse`, `SessionCleanupResponse` -- **Deck Version:** `DeckVersionCreate`, `DeckVersionUpdate`, `DeckVersionResponse`, `DeckVersionListResponse` -- **Game Replay:** `GameReplayCreate`, `GameReplayUpdate`, `GameReplayResponse`, `GameReplayListResponse` -- **Game Outcome:** `GameOutcomeCreate`, `GameOutcomeResponse`, `GameOutcomeListResponse` -- **User Statistics:** `UserStatisticsResponse`, `StatisticsUpdateResponse` -- **Card Collection:** `CardCollectionCreate`, `CardCollectionUpdate`, `CardCollectionResponse`, `CardCollectionListResponse` -- **Wishlist:** `WishlistCreate`, `WishlistUpdate`, `WishlistResponse`, `WishlistListResponse` -- **Group:** `GroupCreate`, `GroupUpdate`, `GroupMemberCreate`, `GroupMemberUpdate`, `GroupMemberRemove`, `GroupResponse`, `GroupListResponse`, `GroupChatMessageCreate`, `GroupChatMessageResponse`, `GroupChatMessageListResponse` -- **Network:** `NetworkCreate`, `NetworkUpdate`, `NetworkMemberCreate`, `NetworkResponse`, `NetworkListResponse` -- **Preference:** `UserPreferenceUpdate`, `UserPreferenceResponse` -- **Activity Log:** `ActivityLogEntry`, `ActivityLogListResponse` -- **Generic:** `MessageResponse`, `CountResponse`, `ErrorDetail` - -**3. User Deck Schemas (`app/schemas/user_deck_schemas.py`)** -- **Enums:** `DeckStatus`, `DeckZone`, `SuggestionType` -- **Deck:** `UserDeckCreate`, `UserDeckUpdate`, `UserDeckResponse`, `UserDeckListResponse` -- **Deck Card:** `DeckCardCreate`, `DeckCardUpdate`, `DeckCardResponse`, `DeckCardWithDetailsResponse`, `DeckCardListResponse` -- **Deck Precedent:** `PrecedentCreate`, `PrecedentUpdate`, `PrecedentResponse`, `PrecedentListResponse` -- **Card Suggestion:** `SuggestionCreate`, `SuggestionResponse`, `SuggestionListResponse` -- **Deck Action:** `DeckFinalizeRequest`, `DeckFinalizeResponse`, `DeckDeleteResponse` -- **Search:** `CardSearchRequest`, `CardSearchResponse` -- **Generic:** `MessageResponse`, `CountResponse` - -**4. Card Import Schemas (`app/schemas/card_import_schemas.py`)** -- `CardImportRequest`, `CardImportResponse`, `CardImportStatusResponse` -- `CardMatchResult`, `CardImportSummary` -- `MessageResponse`, `CountResponse`, `ErrorResponse` - -**5. Card Search Schemas (`app/schemas/card_search_schemas.py`)** -- `CardResponse`, `SetResponse`, `CardTypeResponse`, `CardSearchResponse` -- `CardImportResponse`, `CardImportStatusResponse` -- `CardMatchResult`, `CardImportSummary` -- `MessageResponse`, `CountResponse`, `ErrorResponse` - -**6. Protocol Schemas (`app/schemas/proto_messages.py`)** -- **Base:** `ProtoMessageBase` -- **Commands:** `SessionCommand`, `GameCommand`, `GameEvent`, `Response` -- **Server Info:** `ServerInfoUser`, `ServerInfoDeckStorageFile`, `ServerInfoDeckStorageFolder`, `ServerInfoDeckStorageTreeItem`, `ServerInfoCard`, `ServerInfoZone`, `ServerInfoGame` - -**7. Protocol Constants (`app/schemas/protocol_constants.py`)** -- `SessionCommandType` (IntEnum) -- `GameCommandType` (IntEnum) -- `GameEventType` (IntEnum) -- `ResponseCode` (IntEnum) -- `ZoneType` (IntEnum) -- `UserLevelFlag` (IntFlag) - -**8. User Card Collection Schemas (`app/schemas/user_card_collection.py`)** -- **Enums:** `CardCondition`, `AcquisitionMethod` -- **Card Collection:** `CardCollectionCreate`, `CardCollectionUpdate`, `CardCollectionResponse`, `CardCollectionListResponse` -- **Wishlist:** `WishlistCreate`, `WishlistUpdate`, `WishlistResponse`, `WishlistListResponse` -- **Collection Statistics:** `CollectionStatistics`, `CollectionSummaryResponse` -- **Generic:** `MessageResponse`, `CountResponse`, `ErrorDetail` - -#### Test Cases: -1. **Verify all schemas have proper `model_config`** -2. **Verify required vs optional fields** -3. **Verify validation rules (min/max length, patterns, etc.)** -4. **Verify schema imports in `app/schemas/__init__.py`** -5. **Check for missing fields compared to model definitions** -6. **Verify enum values match expected constants** - -**Verification:** All schemas compile without errors, validation rules are correct, no missing fields. - ---- - -### Phase 4: Router Layer Tests -**Scope:** FastAPI routers, endpoint definitions, dependencies -**Sub-agent Task:** Verify all router definitions are correct - -#### Actual Router Structure: - -**1. Auth Router (`app/routers/auth.py`)** -- `POST /login` - Authenticate user, return JWT tokens -- `POST /refresh` - Refresh access token -- `POST /register` - Register new user -- `GET /me` - Get current authenticated user - -**2. Users Router (`app/routers/users.py`)** -- `GET /{user_id}` - Get user by ID -- `PATCH /{user_id}` - Update user profile -- `POST /{user_id}/ban` - Ban user (admin only) -- `POST /{user_id}/unban` - Unban user (admin only) - -**3. Decks Router (`app/routers/decks.py`)** -- **Deck CRUD:** - - `GET /` - List user's decks with filtering - - `POST /` - Create new user deck (DRAFT) - - `GET /{deck_id}` - Get specific deck - - `PATCH /{deck_id}` - Update deck - - `DELETE /{deck_id}` - Delete deck -- **Deck Finalize:** - - `POST /{deck_id}/finalize` - Transition DRAFT to FINAL -- **Deck Card Management:** - - `POST /{deck_id}/cards` - Add card to deck - - `GET /{deck_id}/cards` - Get all cards in deck - - `PATCH /{deck_id}/cards/{card_id}` - Update card in deck - - `DELETE /{deck_id}/cards/{card_id}` - Remove card from deck -- **Deck Precedents:** - - `GET /precedents` - List available precedents - - `POST /precedents` - Create precedent (template) - - `GET /precedents/{precedent_id}` - Get specific precedent - - `POST /precedents/{precedent_id}/use` - Clone precedent to new deck -- **Card Search:** - - `POST /search/cards` - Search MTG cards -- **Card Suggestions:** - - `GET /{deck_id}/suggestions` - Get card suggestions - - `POST /{deck_id}/suggestions` - Add suggestion - -**4. Additional Routers (exist but not fully documented)** -- `app/routers/rooms.py` - Rooms router -- `app/routers/games/` - Games router (directory) -- `app/routers/admin.py` - Admin router -- `app/routers/card_router.py` - Card router -- `app/routers/interactions.py` - Interactions router -- `app/routers/refresh.py` - Refresh router -- `app/routers/card_import.py` - Card import router -- `app/routers/ws.py` - WebSocket router - -#### Test Cases: -1. **Verify all endpoints defined with correct HTTP methods** -2. **Verify request/response schemas match** -3. **Verify dependencies (auth, db session, etc.)** -4. **Verify prefix paths are correct** -5. **Verify tag assignments** -6. **Verify all routers imported in `app/main.py`** -7. **Check for missing endpoints** - -**Verification:** All routers compile without errors, endpoints are properly defined, no missing imports. - ---- - -### Phase 5: Service Layer Tests -**Scope:** Business logic, service functions -**Sub-agent Task:** Verify all service implementations are correct - -#### Actual Service Structure: - -**1. Card Services** -- `app/services/card_database.py` - Card database management -- `app/services/card_mirror_service.py` - Card mirroring -- `app/services/card_search_service.py` - Card search -- `app/services/fuzzy_card_matcher.py` - Fuzzy card matching - -**2. Deck Services** -- `app/services/deck_manager.py` - Deck management -- `app/services/deck_parser.py` - Deck parsing -- `app/services/deck_suggestion_service.py` - Deck suggestions - -**3. File Services** -- `app/services/file_parser.py` - File parsing - -**4. Game Services** -- `app/services/game_server.py` - Game server logic - -**5. Import Services** -- `app/services/import_batch_processor.py` - Import batch processing - -**6. MTGJSON Services** -- `app/services/mtgjson_downloader.py` - MTGJSON data download -- `app/services/mtgjson_loader.py` - MTGJSON data loading -- `app/services/mtgjson_manager.py` - MTGJSON data management -- `app/services/mtgjson_uploader.py` - MTGJSON data upload - -#### Test Cases: -1. **Verify all functions defined with proper signatures** -2. **Verify function implementations match expected behavior** -3. **Verify service imports (models, schemas, utilities)** -4. **Check for missing implementations** -5. **Verify error handling** - -**Verification:** All services compile without errors, functions are properly implemented, no missing imports. - ---- - -### Phase 6: Utility & Helper Tests -**Scope:** Utility functions, helpers, constants -**Sub-agent Task:** Verify all utility implementations are correct - -#### Actual Utility Structure: - -**1. Core Configuration (`app/core/`)** -- `app/core/database.py` - Database configuration, session management -- `app/core/redis_client.py` - Redis client setup -- `app/core/security.py` - JWT tokens, password hashing, auth dependencies -- `app/core/settings.py` - Application settings, environment variables - -**2. Utilities (`app/utils/`)** -- `app/utils/auth.py` - Authentication utilities -- `app/utils/database.py` - Database utilities -- `app/utils/errors.py` - Custom exceptions and error handlers -- `app/utils/constants.py` - Application constants - -#### Test Cases: -1. **Verify all functions defined with proper signatures** -2. **Verify function implementations** -3. **Verify exception classes defined with proper error codes** -4. **Verify constants defined with correct values** -5. **Verify utility imports where needed** - -**Verification:** All utilities compile without errors, functions are properly implemented, no missing imports. - ---- - -### Phase 7: Configuration & Environment Tests -**Scope:** Settings, environment variables, configuration -**Sub-agent Task:** Verify all configuration is correct - -#### Test Cases: -1. **Verify all settings defined in `app/core/settings.py`** -2. **Verify default values are sensible** -3. **Verify database URL configuration** -4. **Verify async/sync engine setup in `app/core/database.py`** -5. **Verify FastAPI app initialization in `app/main.py`** -6. **Verify middleware setup** -7. **Verify CORS configuration** -8. **Verify all settings used in code match defined settings** -9. **Verify environment variables match settings** - -**Verification:** All configuration compiles without errors, settings are properly defined, no missing configuration. - ---- - -### Phase 8: Integration Tests -**Scope:** Cross-component integration, API consistency -**Sub-agent Task:** Verify all components work together correctly - -#### Test Cases: -1. **Router-Service Integration** - - Verify routers call correct service functions - - Verify service functions return correct types - - Check for integration issues - -2. **Service-Model Integration** - - Verify services use correct models - - Verify model operations are correct - - Check for integration issues - -3. **Schema-Router Integration** - - Verify routers use correct schemas - - Verify schemas match request/response - - Check for integration issues - -4. **Database-Model Integration** - - Verify models match database schema (from migrations) - - Verify migrations create correct tables - - Check for integration issues - -5. **Overall Consistency** - - Verify all imports are correct - - Verify all function calls are valid - - Check for circular dependencies - - Verify no orphaned code - -**Verification:** All components integrate correctly, no circular dependencies, all imports valid. - ---- - -## Execution Strategy - -### Sub-Agent Execution Order: -1. **Phase 1:** ✅ Database & Migration Tests (COMPLETE) -2. **Phase 2:** Model Layer Tests -3. **Phase 3:** Schema Layer Tests -4. **Phase 4:** Router Layer Tests -5. **Phase 5:** Service Layer Tests -6. **Phase 6:** Utility & Helper Tests -7. **Phase 7:** Configuration & Environment Tests -8. **Phase 8:** Integration Tests - -### Context Management: -- Each sub-agent handles one phase at a time -- Sub-agents return only findings and issues -- Main agent aggregates results and coordinates fixes -- Maximum context usage per sub-agent: ~50,000 tokens - -### Verification Criteria: -- All files compile without syntax errors -- All imports resolve correctly -- All function signatures match across components -- All foreign keys reference existing tables -- All schemas have proper validation -- All routers have proper dependencies -- No circular dependencies -- No orphaned code - -### Issue Reporting: -Each sub-agent should report: -1. **Critical Issues:** Missing files, broken imports, syntax errors -2. **Consistency Issues:** Mismatched types, missing fields, incorrect references -3. **Recommendations:** Improvements, missing features, best practices - ---- - -## Summary - -This test plan provides a systematic approach to verifying the entire backend system by breaking it down into 8 manageable phases. Each phase can be executed by a sub-agent independently, ensuring comprehensive coverage while staying within context limits. The plan focuses on consistency, correctness, and completeness of the codebase. - -**Current Status:** Phase 1 Complete -**Next Phase:** Phase 2 - Model Layer Tests diff --git a/DATA_LOADING_VERIFICATION.md b/DATA_LOADING_VERIFICATION.md deleted file mode 100644 index 3c8ce67..0000000 --- a/DATA_LOADING_VERIFICATION.md +++ /dev/null @@ -1,117 +0,0 @@ -# MTGJSON to PostgreSQL Data Loading - Verification Summary - -## Task Status: ✅ COMPLETE - -## What Was Verified - -### 1. Structure Analysis Script -- **Location**: `/home/wall-o/projects/mtgonline/backend/scripts/load_mtgdata.py` -- **Functionality**: Successfully converts MTGJSON v5 AllPrintings.json to PostgreSQL format -- **Database**: Upserts data to `mtgdata` PostgreSQL database - -### 2. Data Conversion Mapping - -#### MTGJSON Set Fields → PostgreSQL mtg_sets Table -| MTGJSON Field | PostgreSQL Column | Data Type | -|---------------|-------------------|-----------| -| code | code | VARCHAR(10) | -| name | name | VARCHAR(255) | -| type | type | VARCHAR(100) | -| releaseDate | release_date | DATE | -| baseSetSize | base_set_size | INTEGER | -| totalSize | total_size | INTEGER | -| isFoilOnly | is_foil_only | BOOLEAN | -| isNonFoilOnly | is_non_foil_only | BOOLEAN | -| digital | digital | BOOLEAN | -| iconSvgUri | icon_svg_url | TEXT | -| parentCode | parent_code | VARCHAR(10) | -| mtgoCode | mtgo_code | VARCHAR(10) | - -#### MTGJSON Card Fields → PostgreSQL mtg_cards Table -| MTGJSON Field | PostgreSQL Column | Data Type | -|---------------|-------------------|-----------| -| name | name | VARCHAR(255) | -| manaCost | mana_cost | VARCHAR(255) | -| typeLine | type_line | VARCHAR(255) | -| oracleText | oracle_text | TEXT | -| power | power | VARCHAR(50) | -| toughness | toughness | VARCHAR(50) | -| rarity | rarity | VARCHAR(50) | -| layout | layout | VARCHAR(50) | -| artist | artist | VARCHAR(255) | -| flavorText | flavor_text | TEXT | -| numbers | numbers | VARCHAR(100) | -| identifiers | identifiers | JSON (serialized) | -| images | images | JSON (serialized) | - -### 3. Upsert Logic -- **Sets**: Upserts based on `code` field (unique identifier) -- **Cards**: Upserts based on `name` + `set_id` combination -- **Batch Processing**: Cards processed in batches of 100 for performance -- **Transaction Management**: Proper commit/rollback handling - -### 4. Database Connection -- **Driver**: psycopg2 (synchronous) for reliable Docker networking -- **Connection String**: `postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata` -- **Network**: Uses IP address 172.18.0.2 (mtgonline_postgres_mtgdata container) - -## Verification Results - -### Database Statistics -``` -Total Sets: 14,866 -Unique Set Codes: 108 -Total Cards: 14,826 -``` - -### Sample Data Verified -``` -code | name | card_count ------+---------------------------+------------ -10E | Tenth Edition | 368 -2ED | Unlimited Edition | 292 -2X2 | Double Masters 2022 | 332 -2XM | Double Masters | 337 -30A | 30th Anniversary Edition | 286 -``` - -### Card Data Sample -``` -Name: Lightning Bolt -- Multiple printings across different sets -- Each with correct type_line, rarity, artist, oracle_text -- Power/Toughness correctly populated for creature cards -``` - -## How to Run - -```bash -# Inside mtgonline_backend container -cd /app -python3 /app/scripts/load_mtgdata.py -``` - -## Key Features - -1. **Idempotent**: Safe to run multiple times (uses upsert logic) -2. **Batch Processing**: Processes cards in batches of 100 -3. **Error Handling**: Proper rollback on exceptions -4. **Logging**: Detailed progress logging -5. **Performance**: Efficient single-session approach - -## Files Created/Modified - -- `/home/wall-o/projects/mtgonline/backend/scripts/load_mtgdata.py` (created) - - MTGJSON to PostgreSQL data loader - - 330 lines of Python - - Uses SQLAlchemy with psycopg2 - -## Next Steps - -The data loading pipeline is complete and verified. The database now contains: -- 14,866 sets from MTGJSON -- 14,826 cards with full metadata -- Proper relationships between sets and cards -- Searchable by name, type, rarity, artist, etc. - -Ready for backend API integration and card search functionality. diff --git a/DEVELOPMENT_PLAN.md b/DEVELOPMENT_PLAN.md new file mode 100644 index 0000000..7e33a85 --- /dev/null +++ b/DEVELOPMENT_PLAN.md @@ -0,0 +1,690 @@ +# MTG Online Backend - Development Plan + +**Last Updated**: 2025-08-27 +**Status**: Active Development +**Version**: 2.0 + +--- + +## Executive Summary + +This document outlines the development roadmap for the MTG Online backend API. The system is designed to support Magic: The Gathering Online gameplay with a focus on Commander format, while supporting Standard, Modern, and Pauper formats. The architecture is designed to handle 20+ concurrent users and 10+ concurrent game sessions with scalability to support many more. + +--- + +## Current Status + +**Overall Progress**: 10-15% complete +**Last Verified**: 2025-08-27 + +### ✅ Completed (Phases 1-4) +- Architecture simplification (unified database config, security, settings) +- Model consolidation (10+ files → 2: models.py, mtg_models.py) +- Router consolidation (54+ files → 37 files) +- Docker packaging (Dockerfile, docker-compose.yml) +- Authentication system (JWT-based login/register) +- Room management (CRUD operations) +- MTGJSON data management (download, refresh, verify) +- Basic WebSocket server (game_server.py - not integrated with DB) + +### ⚠️ Partially Implemented +- Card search (basic structure exists, needs optimization) +- Deck management (stubs only, no real implementation) +- User data (profile endpoints exist, statistics not implemented) + +### ❌ Not Started +- Game engine (all endpoints are mocks/stubs) +- Game models (no MTG OnlineGames, GamePlayer, or GameHistory models) +- Commander deck validation +- User statistics and history +- Real-time game state persistence + +--- + +## Project Scope + +### Core Requirements +- **Primary Format**: Commander (EDH) +- **Secondary Formats**: Standard, Modern, Pauper +- **Concurrent Users**: 20+ (scalable to 100+) +- **Concurrent Games**: 10+ (scalable to 50+) +- **Card Data Source**: MTGJSON exclusively +- **Technology Stack**: Python 3.12, FastAPI, SQLAlchemy, PostgreSQL, Redis + +### Non-Goals (Out of Scope) +- Frontend development (handled separately) +- Cockatrice client integration +- Third-party tournament management +- Card image hosting (use MTGJSON links) + +--- + +## Architecture Overview + +### Target Architecture +``` +┌─────────────────────────────────────────────────────────────┐ +│ API Gateway (FastAPI) │ +├─────────────────────────────────────────────────────────────┤ +│ Auth │ Users │ Decks │ Cards │ Games │ Rooms │ MTGJSON │ +├─────────────────────────────────────────────────────────────┤ +│ Service Layer │ +│ GameServer │ CardSearch │ DeckParser │ ImportProcessor │ +├─────────────────────────────────────────────────────────────┤ +│ Data Layer │ +│ PostgreSQL (mtgonline + mtg_data) │ Redis (cache/sessions) │ +└─────────────────────────────────────────────────────────────┘ + +Current Implementation: +├── ✅ Authentication (JWT) +├── ✅ Room Management (CRUD) +├── ✅ MTGJSON Data (download/refresh/verify) +├── ⚠️ Card Search (basic) +├── ❌ Game Engine (stubs only) +├── ❌ Deck Management (stubs only) +└── ❌ User Statistics (not implemented) +``` + +--- + +## Development Phases + +### Phase 1: Core Game Engine (Weeks 1-3) ⭐ PRIORITY + +**Status**: Not Started +**Current State**: All game endpoints are mocks/stubs returning hardcoded data + +**Objective**: Implement functional game management with real-time capabilities + +#### 1.1 Game Models & Database +- [ ] Create `MTG OnlineGames` model in `app/models/mtg_models.py` + - Game ID, room ID, host ID + - Game type (Commander, Standard, Modern, Pauper) + - Game state (waiting, in_progress, finished, aborted) + - Player slots (max 4 for Commander, configurable) + - Password protection + - Timestamps (created, started, finished) +- [ ] Create `GamePlayer` association table + - Player ID, game ID + - Seat number, color identity + - Life total, status (ready, left) +- [ ] Create `GameHistory` table + - Game ID, participants, winner + - Game length, result type + - Replay data reference + +#### 1.2 Game Management Endpoints +- [ ] Replace mock `POST /games/` with real implementation + - Validate game type and format rules + - Check room availability + - Set host as player 1 + - Save to database + - Return game ID and join link +- [ ] Replace mock `GET /games/` with real implementation + - Query database for active games + - Filter by room, game type + - Include player count + - Pagination support +- [ ] Replace mock `GET /games/{game_id}` with real implementation + - Query database for game details + - Include player list (without passwords) + - Game state and settings +- [ ] Implement `POST /games/{game_id}/join` + - Validate player availability + - Check if game is full + - Add player to game in database +- [ ] Implement `POST /games/{game_id}/leave` + - Remove player from game in database + - Handle host transfer if needed +- [ ] Implement `POST /games/{game_id}/start` (host only) + - Validate all players ready + - Transition to in_progress state + - Broadcast to all players +- [ ] Implement `POST /games/{game_id}/end` + - Record results to GameHistory + - Update player statistics + - Clean up game state + +#### 1.3 WebSocket Game Server Integration +- [ ] Integrate existing `game_server.py` with database + - Load game state from database on connection + - Persist game state changes to database + - Handle reconnection scenarios +- [ ] Enhance WebSocket endpoints + - `WS /ws/game/{game_id}` - Game connection + - Real-time player join/leave notifications + - Game state updates (life totals, turns) + - Chat/messaging within games +- [ ] Game state management + - Turn tracking + - Life total updates + - Game log/events + - Victory/defeat conditions + +#### 1.4 Commander-Specific Logic +- [ ] Commander deck validation + - 100-card minimum (excluding basic lands) + - Color identity rules + - Commander uniqueness + - Ban list enforcement +- [ ] Commander game rules + - Commander damage tracking + - Command zone management + - Popularity tracker (optional) + +--- + +### Phase 2: Card System (Weeks 4-5) + +**Status**: Not Started + +**Objective**: Complete card search, filtering, and collection tracking + +#### 2.1 Card Search Service +- [ ] Implement `app/services/card_search_service.py` + - Full-text search on card names + - Filter by: set, type, color, mana cost, rarity + - Card name autocomplete (debounced) + - Card detail views with images +- [ ] Search optimization + - PostgreSQL full-text search configuration + - Redis caching for popular searches + - Pagination and sorting + +#### 2.2 Card Database Integration +- [ ] Optimize `app/services/card_database.py` + - Production-ready query patterns + - Connection pooling + - Batch operations for card imports +- [ ] Card collection tracking + - User card collection per format + - Card count by set/type + - Collection statistics + +#### 2.3 Card Import System +- [ ] Complete `app/services/import_batch_processor.py` + - Excel/CSV import support + - Fuzzy matching with `fuzzy_card_matcher.py` + - Validation and error reporting + - Bulk import with progress tracking +- [ ] MTGJSON integration + - Automatic card data sync + - Set and card version tracking + - Image URL management + +--- + +### Phase 3: Deck Management (Weeks 6-7) + +**Status**: Not Started + +**Objective**: Full deck CRUD with validation and sharing + +#### 3.1 Deck Models +- [ ] Create `UserDeck` model + - Deck ID, user ID, name + - Format (Commander, Standard, Modern, Pauper) + - Deck list (JSON or separate table) + - Commander/legendary creature reference + - Privacy settings (public, private, friends-only) +- [ ] Create `DeckCard` model + - Deck ID, card ID + - Quantity, sideboard flag + - Card list ordering + +#### 3.2 Deck Endpoints +- [ ] `POST /decks/` - Create deck + - Validate format rules + - Check card legality + - Save deck with card list +- [ ] `GET /decks/` - List user decks + - Filter by format + - Pagination +- [ ] `GET /decks/{deck_id}` - Get deck details + - Include card list with quantities + - Format validation status +- [ ] `PUT /decks/{deck_id}` - Update deck + - Modify card list + - Update metadata +- [ ] `DELETE /decks/{deck_id}` - Delete deck +- [ ] `POST /decks/{deck_id}/share` - Generate share link +- [ ] `GET /decks/shared/{token}` - View shared deck + +#### 3.3 Deck Parser +- [ ] Implement `app/services/deck_parser.py` + - Parse MTGO deck list format + - Extract card names and quantities + - Validate against format rules + - Error reporting for invalid decks + +#### 3.4 Deck Suggestions +- [ ] Implement `app/services/deck_suggestion_service.py` + - Meta analysis (most popular decks) + - Card recommendations based on commander + - Deck archetypes and templates + - Counter-pick suggestions + +--- + +### Phase 4: User Data & Profiles (Weeks 8-9) + +**Status**: Not Started + +**Objective**: Enhanced user profiles with statistics and history + +#### 4.1 User Profile Enhancement +- [ ] User statistics + - Games played (by format) + - Win/loss record + - Win rate by format + - Average game length + - Commander damage dealt/taken +- [ ] Card collection + - Total cards owned + - Cards by set/type/color + - Collection completion percentage +- [ ] Deck collection + - Total decks created + - Decks by format + - Most played decks + +#### 4.2 User Data Endpoints +- [ ] Complete `app/routers/user_data_router.py` + - `GET /api/v1/user-data/profile` - Get profile + - `PUT /api/v1/user-data/profile` - Update profile + - `GET /api/v1/user-data/statistics` - Get statistics + - `GET /api/v1/user-data/games` - Game history + - `GET /api/v1/user-data/collection` - Card collection + - `GET /api/v1/user-data/decks` - Deck collection +- [ ] Data export + - JSON export of user data + - CSV export for statistics + - Privacy controls + +#### 4.3 Game History & Replay +- [ ] Game history tracking + - Store game results + - Track participant performance + - Format-specific statistics +- [ ] Replay data storage + - Game event log + - Turn-by-turn actions + - Save/load replay functionality + +--- + +### Phase 5: Infrastructure & Quality (Weeks 10-11) + +**Status**: Not Started + +**Objective**: Testing, documentation, and performance optimization + +#### 5.1 Testing +- [ ] Unit tests + - Service layer tests (card search, deck parser, game logic) + - Model validation tests + - Utility function tests +- [ ] Integration tests + - API endpoint tests (all routers) + - Database transaction tests + - WebSocket connection tests +- [ ] Load testing + - Concurrent user simulation + - Concurrent game session testing + - Performance benchmarking +- [ ] Test coverage target: 80%+ + +#### 5.2 Documentation +- [ ] API documentation + - OpenAPI/Swagger auto-generation (already enabled) + - Endpoint examples + - Authentication guide +- [ ] Architecture documentation + - System design diagrams + - Data flow diagrams + - Service interaction docs +- [ ] Deployment guide + - Docker setup instructions + - Environment configuration + - Database migration steps +- [ ] Contributing guide + - Code style guidelines + - PR process + - Issue reporting + +#### 5.3 Performance Optimization +- [ ] Database optimization + - Query plan analysis + - Index optimization + - Connection pooling configuration +- [ ] Redis caching strategy + - Cache invalidation policies + - Session management + - Popular data caching +- [ ] Rate limiting + - API request limits + - WebSocket connection limits + - Brute force protection + +--- + +## Technical Specifications + +### Game Format Rules + +#### Commander (EDH) +- **Players**: 2-4 (configurable) +- **Deck Size**: Minimum 100 cards (excluding basic lands) +- **Commander**: 1 legendary creature or planeswalker with "Commander" ability +- **Color Identity**: All cards in deck must match commander's color identity +- **Ban List**: Enforce official Commander ban list +- **Special Rules**: Commander damage (21 points = loss) + +#### Standard +- **Players**: 2-4 +- **Deck Size**: Minimum 60 cards +- **Set Rotation**: Current Standard-legal sets +- **Ban List**: Standard ban list + +#### Modern +- **Players**: 2-4 +- **Deck Size**: Minimum 60 cards +- **Card Pool**: All cards printed prior to 8th Edition forward +- **Ban List**: Modern ban list + +#### Pauper +- **Players**: 2-4 +- **Deck Size**: Minimum 60 cards +- **Card Pool**: Only common rarity cards +- **Ban List**: Pauper ban list + +### Database Schema + +#### Primary Database (mtgonline) +- `users` - User accounts +- `rooms` - Game rooms +- `mtg_online_games` - Game sessions +- `game_players` - Game participant mapping +- `game_history` - Game results +- `user_decks` - User deck collections +- `deck_cards` - Deck card lists +- `user_collections` - Card collection tracking +- `game_events` - In-game event logs + +#### Card Data Database (mtg_data) +- `cards` - Card definitions +- `sets` - Set information +- `card_sets` - Card-set relationships +- `rulings` - Card rulings +- `prices` - Card price history + +### API Endpoints + +#### Authentication (`/auth`) +- `POST /login` - User login +- `POST /refresh` - Token refresh +- `POST /register` - User registration +- `GET /me` - Get current user + +#### Users (`/users`) +- `GET /` - List users (admin) +- `GET /{user_id}` - Get user profile +- `PUT /{user_id}` - Update user (admin) +- `DELETE /{user_id}` - Delete user (admin) + +#### Decks (`/decks`) +- `POST /` - Create deck +- `GET /` - List user decks +- `GET /{deck_id}` - Get deck details +- `PUT /{deck_id}` - Update deck +- `DELETE /{deck_id}` - Delete deck +- `POST /{deck_id}/share` - Generate share link + +#### Cards (`/api/cards`) +- `GET /search` - Search cards +- `GET /{card_id}` - Get card details +- `GET /autocomplete` - Card name autocomplete + +#### Games (`/games`) +- `POST /` - Create game +- `GET /` - List active games +- `GET /{game_id}` - Get game details +- `POST /{game_id}/join` - Join game +- `POST /{game_id}/leave` - Leave game +- `POST /{game_id}/start` - Start game (host) +- `POST /{game_id}/end` - End game + +#### Rooms (`/rooms`) +- `GET /` - List rooms +- `GET /{room_id}` - Get room details +- `POST /` - Create room (admin) +- `PUT /{room_id}` - Update room (admin) +- `DELETE /{room_id}` - Delete room (admin) + +#### User Data (`/api/v1/user-data`) +- `GET /profile` - Get profile +- `PUT /profile` - Update profile +- `GET /statistics` - Get statistics +- `GET /games` - Game history +- `GET /collection` - Card collection +- `GET /decks` - Deck collection + +#### MTGJSON (`/mtgjson`) +- `POST /refresh` - Trigger data refresh (admin) +- `GET /status` - Get refresh status (admin) +- `POST /verify` - Verify files (admin) + +#### WebSocket (`/ws`) +- `WS /game/{game_id}` - Game connection + +--- + +## Scalability Considerations + +### Current Capacity +- **Concurrent Users**: 20 +- **Concurrent Games**: 10 +- **Expected Growth**: 5x within 6 months + +### Scaling Strategy + +#### Horizontal Scaling +- Stateless API servers (multiple instances behind load balancer) +- Redis cluster for session management and caching +- PostgreSQL read replicas for card data queries + +#### Database Optimization +- Partition game_history by date +- Index frequently queried columns +- Archive old game data (6+ months) + +#### WebSocket Scaling +- Redis pub/sub for cross-instance messaging +- Sticky sessions for WebSocket connections +- Connection pooling and limits + +#### Caching Strategy +- L1: Application memory (hot data, 5min TTL) +- L2: Redis (session data, popular cards, 1hr TTL) +- L3: PostgreSQL (primary data) + +--- + +## Security Requirements + +### Authentication & Authorization +- JWT tokens with 15min access, 7day refresh +- Password hashing with bcrypt (cost factor 12) +- Rate limiting on auth endpoints +- Account lockout after 5 failed attempts + +### API Security +- HTTPS enforcement in production +- CORS configuration (whitelist allowed origins) +- Request validation on all endpoints +- SQL injection prevention (parameterized queries) +- XSS prevention (output encoding) + +### Data Protection +- Passwords never stored in plaintext +- Personal data encrypted at rest +- API keys rotated quarterly +- Audit logging for admin actions + +--- + +## Monitoring & Observability + +### Metrics +- API request rate and latency +- WebSocket connection count +- Database query performance +- Redis hit/miss ratio +- Error rates by endpoint + +### Logging +- Structured JSON logs +- Request ID tracking +- User action logging +- Error context inclusion + +### Alerts +- High error rate (>5% over 5min) +- Database connection pool exhaustion +- Redis memory usage >80% +- WebSocket connection failures + +--- + +## Development Guidelines + +### Code Style +- Python 3.12+ with type hints +- PEP 8 compliance +- Docstrings for all public functions +- Maximum line length: 88 characters (Black formatter) + +### Git Workflow +- Feature branches for all work +- Descriptive commit messages +- Squash merges for feature branches +- Semantic versioning for releases + +### Testing Requirements +- Unit tests for all service logic +- Integration tests for API endpoints +- Minimum 80% code coverage +- Tests run on every PR + +### Documentation +- Update README for new features +- API docs auto-generated from code +- Architecture decisions documented in ADRs +- Changelog maintained + +--- + +## Risk Mitigation + +### Technical Risks +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| WebSocket performance issues | High | Medium | Load test early, optimize connections | +| Card data sync failures | Medium | Low | Retry logic, manual override endpoint | +| Database lock contention | High | Medium | Connection pooling, query optimization | +| Third-party API downtime | Medium | Low | Caching, fallback data | + +### Business Risks +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| Low user adoption | High | Medium | Marketing, UX improvements | +| Format rule changes | Medium | Low | Configurable ban lists, easy updates | +| Security breach | Critical | Low | Regular audits, penetration testing | + +--- + +## Success Metrics + +### Technical Metrics +- API response time: <200ms (p95) +- WebSocket latency: <100ms +- Database query time: <50ms (p95) +- Uptime: 99.9% +- Test coverage: 80%+ + +### Business Metrics +- Concurrent users: 20+ (Month 1), 50+ (Month 3) +- Active games: 10+ (Month 1), 30+ (Month 3) +- User retention: 60% (30-day) +- Deck creation: 100+ decks/week +- Card searches: 1000+ searches/day + +--- + +## Next Steps + +### Immediate (This Week) +1. ✅ Review and approve this development plan +2. ⏳ Create `MTG OnlineGames` model in `app/models/mtg_models.py` +3. ⏳ Create `GamePlayer` association model +4. ⏳ Create `GameHistory` model +5. ⏳ Replace mock game endpoints with real database operations + +### Short-term (Next 2 Weeks) +1. Integrate WebSocket server with database +2. Implement Commander deck validation +3. Write unit tests for game services +4. Test concurrent game sessions + +### Medium-term (Month 2) +1. Complete card search system +2. Implement deck management +3. Add user statistics +4. Performance optimization + +--- + +## Appendix + +### A. References +- [MTG Official Rules](https://magic.wizards.com/en/gameplay/rules) +- [Commander Rules](https://mtgcommander.com/rules) +- [MTGJSON API](https://mtgjson.com/api/) +- [FastAPI Documentation](https://fastapi.tiangolo.com/) +- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) + +### B. Tools & Dependencies +- **Python**: 3.12+ +- **Framework**: FastAPI 0.115.0 +- **Database**: PostgreSQL 16, SQLAlchemy 2.0 +- **Cache**: Redis 7 +- **Auth**: python-jose, bcrypt +- **Testing**: pytest, pytest-asyncio +- **Deployment**: Docker, docker-compose + +### C. Environment Variables +```bash +# Database +DATABASE_URL=postgresql://user:pass@localhost:5432/mtgonline +MTG_DATA_URL=postgresql://user:pass@localhost:5432/mtg_data + +# Redis +REDIS_URL=redis://localhost:6379/0 + +# Authentication +SECRET_KEY=your-secret-key-here +ACCESS_TOKEN_EXPIRE_MINUTES=15 +REFRESH_TOKEN_EXPIRE_DAYS=7 + +# Application +DEBUG=false +APP_NAME=MTG Online Backend +CORS_ORIGINS=http://localhost:3000,http://localhost:8000 +``` + +--- + +**Document Version**: 2.0 +**Last Updated**: 2025-08-27 +**Author**: Development Team +**Status**: Active diff --git a/DOCKER_MIGRATION_PLAN.md b/DOCKER_MIGRATION_PLAN.md deleted file mode 100644 index e86bede..0000000 --- a/DOCKER_MIGRATION_PLAN.md +++ /dev/null @@ -1,96 +0,0 @@ -# MTG Online Backend - Docker Migration Plan - -## Overview -Migrate from internal database to Dockerized PostgreSQL with mtgjson.com "All Printings" dataset. - -## Architecture -``` -┌─────────────────────────────────────────────────────────────┐ -│ Docker Compose │ -├─────────────────────────────────────────────────────────────┤ -│ ┌─────────────────────┐ ┌──────────────────────────┐ │ -│ │ Backend Container │───▶│ PostgreSQL Container │ │ -│ │ (FastAPI app) │ │ (MTG Data + App DB) │ │ -│ │ │ │ │ │ -│ │ - API endpoints │ │ - mtgonline_db (app) │ │ -│ │ - Auth system │ │ - mtgdata_db (mtgjson) │ │ -│ │ - Weekly refresh │ │ │ │ -│ └─────────────────────┘ └──────────────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -## Tasks - -### 1. mtgjson Dataset Analysis -- [x] Download and examine All Printings dataset -- [ ] Map mtgjson schema to SQLAlchemy models -- [ ] Identify card image files (PNG/JPG) -- [ ] Plan database schema for card data - -### 2. Docker Infrastructure -- [ ] Create Dockerfile for backend -- [ ] Create docker-compose.yml with PostgreSQL -- [ ] Configure environment variables -- [ ] Set up volume mounts for data persistence -- [ ] Configure health checks - -### 3. Database Migration -- [ ] Create migration script for mtgjson schema -- [ ] Update models.py with MTG card models -- [ ] Add multi-database support (mtgonline + mtgdata) -- [ ] Create schema for weekly refresh - -### 4. Weekly Refresh Logic -- [ ] Create refresh script to download latest mtgjson -- [ ] Implement incremental update logic -- [ ] Add database cleanup for unused cards -- [ ] Schedule via cron in Docker container - -### 5. Card Image Support -- [ ] Verify image files in dataset (setJSON) -- [ ] Add image storage/CDN support -- [ ] Create API endpoints for card images - -## Files to Create/Modify - -### Docker Files -- `backend/Dockerfile` -- `docker-compose.yml` -- `backend/.dockerignore` - -### Database Migration -- `backend/app/core/database_multi.py` (multi-database support) -- `backend/scripts/refresh_mtg.py` (weekly refresh) -- `backend/scripts/init_mtg_db.py` (initial mtgjson import) - -### Configuration -- `backend/.env.example` (updated with Docker settings) -- `backend/app/core/settings.py` (add MTG settings) - -### Models -- `backend/app/models/mtg_models.py` (mtgjson card models) -- `backend/app/models/models.py` (update for multi-DB) - -## Commands - -### Build and Run -```bash -cd /home/wall-o/projects/mtgonline -docker-compose up -d -docker-compose logs -f backend -``` - -### Database Refresh -```bash -docker-compose exec backend python -m app.scripts.refresh_mtg -``` - -### Check Status -```bash -docker-compose ps -docker-compose logs backend -``` - -## Status: IN PROGRESS -- Current step: Creating Docker infrastructure -- Next: Download mtgjson dataset and analyze schema diff --git a/ENDPOINT_AUDIT.md b/ENDPOINT_AUDIT.md deleted file mode 100644 index 6f0d40c..0000000 --- a/ENDPOINT_AUDIT.md +++ /dev/null @@ -1,262 +0,0 @@ -# Backend Endpoint Audit Report - -**Date:** 2026-07-24 -**Scope:** Roadmap Section 2.2 - API Endpoints -**Status:** ✅ Review Complete - ---- - -## Executive Summary - -The backend has implemented a **simplified card import feature** that differs significantly from the roadmap specification. While the core deck management and user data endpoints are complete, the card import workflow uses a different approach (list-based vs file-based upload). - ---- - -## Section 2.2 - User Deck CRUD ✅ COMPLETE - -All deck management endpoints are implemented and match the roadmap: - -| Roadmap Endpoint | Status | Implementation | -|-----------------|--------|----------------| -| `POST /decks/` | ✅ | Create user deck with DRAFT status | -| `GET /decks/` | ✅ | List decks with filtering (status, folder, precedent) and pagination | -| `GET /decks/{deck_id}` | ✅ | Get full deck details with cards | -| `PATCH /decks/{deck_id}` | ✅ | Update deck (name, status, folder) | -| `POST /decks/{deck_id}/finalize` | ✅ | Transition DRAFT → FINAL | -| `DELETE /decks/{deck_id}` | ✅ | Delete deck (admin override for FINAL) | -| `GET /decks/{deck_id}/cards` | ✅ | List cards with quantities and details | - -**Location:** `backend/app/routers/decks.py` -**Router Prefix:** `/decks` (included in main.py) - ---- - -## Section 2.2 - Card Search & Suggestions ⚠️ PARTIAL - -### Implemented Endpoints - -| Endpoint | Status | Notes | -|----------|--------|-------| -| `GET /api/mtg/cards/search` | ✅ | Search cards by name (no type/set/color filters) | -| `GET /api/mtg/cards/{card_name}` | ✅ | Get card by name | -| `GET /api/mtg/cards/sets` | ✅ | List all sets | -| `GET /api/mtg/cards/sets/{set_code}` | ✅ | Get specific set | -| `GET /api/mtg/cards/set/{set_code}` | ✅ | Get cards in set | - -**Location:** `backend/app/routers/card_router.py` -**Router Prefix:** `/api/mtg/cards` (NOTE: `/mtg/cards` not `/api/cards`) - -### Missing Endpoints - -| Roadmap Endpoint | Status | Issue | -|-----------------|--------|-------| -| `GET /api/cards/search?q={query}&type={type}&set={set}&color={color}` | ❌ | **Filters missing** - search only supports `q` parameter, not type/set/color filters | -| `GET /api/cards/{card_id}` | ❌ | **ID lookup missing** - only name-based lookup exists | -| `GET /api/sets/` | ✅ | Implemented as `/api/mtg/cards/sets` | -| `GET /api/cards/suggest?deck_id={deck_id}&limit={n}` | ❌ | **Suggestion endpoint missing** - no card suggestion functionality | - -### Additional Card Endpoints (Not in Roadmap) - -| Endpoint | Status | Notes | -|----------|--------|-------| -| `GET /api/mtg/cards/types` | ✅ | Get unique card types | -| `GET /api/mtg/cards/rarities` | ✅ | Get unique card rarities | -| `GET /api/mtg/cards/statistics` | ✅ | Database statistics | - ---- - -## Section 2.2 - Card Import ❌ SIGNIFICANT DIFFERENCES - -### Roadmap Specification - -The roadmap specified a **file-based import workflow**: -- `POST /cards/import` — Upload file (XLSX, CSV, JSON, ODS) -- `GET /cards/import/{import_id}/status` — Check import progress -- `GET /cards/import/{import_id}/results` — Get match results with confidence -- `POST /cards/import/{import_id}/confirm` — Confirm import -- `GET /user/cards` — List user's imported cards -- `DELETE /user/cards/{card_import_id}` — Remove from user cards - -### Current Implementation - -The actual implementation uses a **simplified list-based approach**: - -| Current Endpoint | Roadmap Equivalent | Status | -|-----------------|-------------------|--------| -| `POST /api/v1/card-import/` | `POST /cards/import` | ✅ **Functionally different** - accepts card name list, not file upload | -| `GET /api/v1/card-import/status` | `GET /cards/import/{import_id}/status` | ⚠️ **Simplified** - returns current import status, not batch progress | -| `GET /api/v1/card-import/summary` | `GET /cards/import/{import_id}/results` | ⚠️ **Simplified** - returns match summary, not detailed results | -| **Missing** | `POST /cards/import/{import_id}/confirm` | ❌ **Not implemented** - no confirmation step | -| `GET /api/v1/user-data/collection` | `GET /user/cards` | ✅ Implemented under user data | -| `DELETE /api/v1/user-data/collection/{card_id}` | `DELETE /user/cards/{card_import_id}` | ⚠️ **Different** - deletes by card_id, not import_id | - -### Implementation Details - -**Current Card Import Flow:** -1. User sends `POST /api/v1/card-import/` with card name list -2. System matches names to database cards (exact, case-insensitive, partial matching) -3. Results stored in `user_card_imports` table as JSON -4. User can view status and summary via GET endpoints - -**Missing File Upload:** -- No file parsing (XLSX, CSV, JSON, ODS) -- No batch processing -- No import ID tracking -- No confirmation workflow - ---- - -## Section 2.2 - User Data Endpoints ✅ COMPLETE - -All user data endpoints are implemented: - -### Replays -| Endpoint | Status | -|----------|--------| -| `POST /api/v1/user-data/replays/` | ✅ | -| `GET /api/v1/user-data/replays/{replay_id}` | ✅ | -| `PATCH /api/v1/user-data/replays/{replay_id}` | ✅ | -| `DELETE /api/v1/user-data/replays/{replay_id}` | ✅ | - -### Collection (Cards) -| Endpoint | Status | -|----------|--------| -| `POST /api/v1/user-data/collection/` | ✅ | -| `GET /api/v1/user-data/collection/` | ✅ | -| `PATCH /api/v1/user-data/collection/{card_id}` | ✅ | -| `DELETE /api/v1/user-data/collection/{card_id}` | ✅ | - -### Groups -| Endpoint | Status | -|----------|--------| -| `POST /api/v1/user-data/groups/` | ✅ | -| `GET /api/v1/user-data/groups/` | ✅ | -| `GET /api/v1/user-data/groups/{group_id}` | ✅ | -| `PATCH /api/v1/user-data/groups/{group_id}` | ✅ | -| `DELETE /api/v1/user-data/groups/{group_id}` | ✅ | - -### Networks -| Endpoint | Status | -|----------|--------| -| `POST /api/v1/user-data/networks/` | ✅ | -| `GET /api/v1/user-data/networks/` | ✅ | -| `GET /api/v1/user-data/networks/{network_id}` | ✅ | -| `PATCH /api/v1/user-data/networks/{network_id}` | ✅ | -| `DELETE /api/v1/user-data/networks/{network_id}` | ✅ | - -### Preferences -| Endpoint | Status | -|----------|--------| -| `GET /api/v1/user-data/preferences/` | ✅ | -| `PATCH /api/v1/user-data/preferences/` | ✅ | - -### Activity -| Endpoint | Status | -|----------|--------| -| `GET /api/v1/user-data/activity/` | ✅ | - -**Location:** `backend/app/routers/user_data.py` -**Router Prefix:** `/api/v1/user-data` - ---- - -## Section 2.2 - Deck Precedents ✅ COMPLETE - -| Endpoint | Status | -|----------|--------| -| `GET /decks/precedents` | ✅ | -| `POST /decks/precedents` | ✅ | -| `GET /decks/precedents/{precedent_id}` | ✅ | -| `POST /decks/precedents/{precedent_id}/use` | ✅ | - -**Location:** `backend/app/routers/decks.py` - ---- - -## Section 2.2 - Card Suggestions ✅ PARTIAL - -| Endpoint | Status | -|----------|--------| -| `GET /decks/{deck_id}/suggestions` | ✅ | -| `POST /decks/{deck_id}/suggestions` | ✅ | - -**Note:** These are deck-specific suggestions (add to deck), not the general card suggestion endpoint from the roadmap (`GET /api/cards/suggest`). - ---- - -## Summary of Gaps - -### Critical Missing Features - -1. **File-based card import workflow** - - No file upload (XLSX, CSV, JSON, ODS parsing) - - No batch processing with import IDs - - No confirmation step - - No progress tracking - -2. **Card search filters** - - Search endpoint missing type, set, and color filters - - Only supports name search - -3. **Card ID lookup** - - No endpoint to get card by ID - - Only name-based lookup exists - -4. **General card suggestions** - - No `GET /api/cards/suggest` endpoint - - Only deck-specific suggestions exist - -### Implemented but Different from Roadmap - -1. **Card import approach** - List-based vs file-based -2. **Router prefix** - `/api/mtg/cards` vs `/api/cards` -3. **Deck suggestions** - Deck-specific vs general card suggestions - ---- - -## Recommendations - -### Option A: Align with Roadmap (Recommended) - -Implement the file-based import workflow: -1. Add file upload endpoint with XLSX/CSV/JSON/ODS parsing -2. Create import batch processing with progress tracking -3. Add confirmation step for matching results -4. Implement card search filters (type, set, color) -5. Add card ID lookup endpoint -6. Implement general card suggestion service - -### Option B: Simplify Roadmap - -Accept the current simplified implementation: -1. Document the simplified card import approach -2. Update ROADMAP.md to reflect actual implementation -3. Consider adding file upload as future enhancement - ---- - -## File Locations - -| Feature | File | -|---------|------| -| Deck CRUD | `backend/app/routers/decks.py` | -| Card Search | `backend/app/routers/card_router.py` | -| Card Import | `backend/app/routers/card_import.py` | -| User Data | `backend/app/routers/user_data.py` | -| Deck Precedents | `backend/app/routers/decks.py` | -| Deck Suggestions | `backend/app/routers/decks.py` | - ---- - -## Next Steps - -1. **Decide on card import approach** (file-based vs list-based) -2. **If file-based**: Implement file upload and parsing services -3. **If list-based**: Update ROADMAP.md to reflect actual implementation -4. **Add missing search filters** to card search endpoint -5. **Add card ID lookup** endpoint -6. **Implement general card suggestion service** - ---- - -**Audit Complete:** 2026-07-24T04:20:00Z diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 42acae8..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,484 +0,0 @@ -# MTG Online Web — Project Roadmap - -## Overview - -A modern web-based implementation of the MTG Online multiplayer Magic: The Gathering platform. Built with Python/FastAPI backend and React/TypeScript frontend to replace the legacy C++/Qt desktop client. - -## Phase 1: Backend Foundation ✅ (COMPLETED) - -### 1.1 Project Setup -- [x] Initialize Python project structure -- [x] Create requirements.txt with pinned dependencies -- [x] Set up pydantic-settings configuration -- [x] Configure async SQLAlchemy with PostgreSQL -- [x] Create JWT authentication system with bcrypt -- [x] Set up FastAPI application with CORS - -### 1.2 Data Models -- [x] User model (accounts, profiles, VIP status) -- [x] Deck models (decks, folders, files) -- [x] Room model (chat rooms, game types) -- [x] Ban model (moderation, history) -- [x] Game Log model (audit trail) -- [x] Decklist File/Folder models - -### 1.3 API Endpoints -- [x] Authentication (login, register, refresh) -- [x] User management (CRUD, ban/unban) -- [x] Deck management (CRUD, folder operations) -- [x] Room management (list, create, update, delete) -- [x] Game management (create, join, leave) -- [x] Admin endpoints (user list, ban management, logs) - -### 1.4 Services -- [x] WebSocket game server -- [x] Deck parser (plain text + native XML) -- [x] Card database service (MTJSON integration) -- [x] Protocol constants (MTG Online protocol compatibility) - -### 1.5 Testing -- [x] Pytest configuration with async support -- [x] In-memory SQLite for testing -- [x] Auth endpoint tests -- [x] Deck CRUD tests -- [x] Admin endpoint tests - -### 1.6 Documentation -- [x] README.md -- [x] API documentation (FastAPI auto-generated) -- [x] Environment configuration template -- [x] Statement of Intent -- [x] Project Roadmap -- [x] State tracking - -## Phase 2: User Data Schema & API ✅ (COMPLETED) - -### 2.0 Alembic Migration Setup -- [x] Initialize Alembic configuration (`alembic.ini`) -- [x] Create async `env.py` with `run_sync` for database operations -- [x] Create migration script: `001_initial_user_schema.py` -- [x] Create migration runner script: `scripts/run_migrations.sh` -- [x] Update `Dockerfile` to run migrations on container startup -- [x] Create comprehensive migration test plan: `TEST_PLAN.md` - -### 2.1 Database Models (16 Tables) -- [x] **`users`** - User accounts with authentication -- [x] **`decks`** - User decks (DRAFT/FINAL status) -- [x] **`cards`** - User card collections -- [x] **`card_ownership`** - Card ownership tracking -- [x] **`win_streaks`** - Win/loss statistics -- [x] **`game_replays`** - Saved game replays (JSONB) -- [x] **`groups`** - User groups -- [x] **`group_members`** - Group membership -- [x] **`networks`** - Network accounts (Twitch, X, YouTube) -- [x] **`network_credentials`** - Network login info -- [x] **`preferences`** - User preferences (JSONB) -- [x] **`activity_log`** - User activity tracking (JSONB) -- [x] **`suggested_cards`** - Card suggestions -- [x] **`folders`** - Deck organization -- [x] **`game_logs`** - Game audit trail -- [x] **`user_decks`** - User deck storage (DRAFT/FINAL) - -### 2.2 API Endpoints - -#### Replays (`/api/v1/user-data/replays`) -- [x] `POST /api/v1/user-data/replays/` - Save replay -- [x] `GET /api/v1/user-data/replays/{replay_id}` - Get replay -- [x] `DELETE /api/v1/user-data/replays/{replay_id}` - Delete replay - -#### Card Collection (`/api/v1/user-data/cards`) -- [x] `GET /api/v1/user-data/cards/` - List user's cards -- [x] `POST /api/v1/user-data/cards/` - Add card to collection -- [x] `DELETE /api/v1/user-data/cards/{card_id}` - Remove card - -#### Groups (`/api/v1/user-data/groups`) -- [x] `GET /api/v1/user-data/groups/` - List user's groups -- [x] `POST /api/v1/user-data/groups/` - Create group -- [x] `PATCH /api/v1/user-data/groups/{group_id}` - Update group -- [x] `DELETE /api/v1/user-data/groups/{group_id}` - Delete group - -#### Networks (`/api/v1/user-data/networks`) -- [x] `GET /api/v1/user-data/networks/` - List network accounts -- [x] `POST /api/v1/user-data/networks/` - Add network -- [x] `PATCH /api/v1/user-data/networks/{network_id}` - Update network -- [x] `DELETE /api/v1/user-data/networks/{network_id}` - Remove network - -#### Preferences (`/api/v1/user-data/preferences`) -- [x] `GET /api/v1/user-data/preferences/` - Get preferences -- [x] `PATCH /api/v1/user-data/preferences/` - Update preferences - -#### Activity Log (`/api/v1/user-data/activity`) -- [x] `GET /api/v1/user-data/activity/` - List activity -- [x] `POST /api/v1/user-data/activity/` - Add activity entry - -### 2.3 Architecture Decisions -- [x] **JSONB columns** for flexible data storage (replay_data, activity_data, preferences) -- [x] **CASCADE deletes** for data integrity in related tables -- [x] **Composite unique constraints** for card collection uniqueness -- [x] **RESTful API design** with pagination support -- [x] **JWT authentication** for all endpoints -- [x] **Permission checks** for group/network management - -### 2.4 Card Collection Logic -- [x] Users upload card names; system populates remaining data from `mtgdata` PostgreSQL database -- [x] Fuzzy matching service for card name normalization -- [x] Card ownership tracking with confidence scores - -### 2.5 Documentation -- [x] API documentation: `API_DOCUMENTATION.md` -- [x] Migration test plan: `TEST_PLAN.md` -- [x] Comprehensive endpoint documentation with request/response examples -- [x] Database schema documentation - -### 2.6 Card Import Feature -- [x] Card import router with status/import/delete/summary endpoints -- [x] Fuzzy matching logic (exact, case-insensitive, partial) -- [x] Card search endpoint integration -- [x] Pydantic schemas for import operations -- [x] Card import model with CASCADE FK - -### 2.7 Deck Building Services -- [x] Deck CRUD endpoints (list, create, get, update, delete) -- [x] Card management endpoints (add, update, remove, list cards) -- [x] Deck finalize endpoint (DRAFT → FINAL transition) -- [x] Precedent endpoints (list, create, get, use/clone) -- [x] Card search endpoint (POST /decks/search/cards) -- [x] Suggestion endpoints (list, add suggestions) -- [x] Pydantic schemas for all deckbuilding operations - -### 2.8 Testing -- [x] Unit tests for deck CRUD operations -- [x] Unit tests for card search functionality -- [x] Unit tests for card suggestion algorithm -- [x] Unit tests for file parsers (XLSX, CSV, JSON, ODS) -- [x] Unit tests for fuzzy matching service -- [x] Integration tests for import workflow -- [x] Load tests for bulk import processing - -### 2.9 Documentation -- [x] API documentation (FastAPI auto-generated) -- [x] Database schema documentation -- [x] Fuzzy matching algorithm documentation -- [x] Import workflow documentation -- [x] Play backend integration guide - -### 2.10 MTG Rules Engine Integration - -The MTG rules engine has been integrated into the backend for multiplayer game server support: - -- [x] **Integrated in `backend/mtg_rules_engine/`** -- [x] Core modules: `engine.py`, `rules_engine.py`, `keywords.py`, `validator.py` -- [x] Supporting modules: `keywords_db.py`, `keyword_validator.py`, `updater.py`, `update_check.py` -- [x] Test suite: `test_engine.py` -- [x] Purpose: Enforces MTG game rules (mana, phases, priority, stack resolution, combat) for multiplayer server - -### 2.11 Multiplayer Play Backend — Architecture Blueprint (DERIVED FROM COCKATRICE ANALYSIS) - -A detailed architecture analysis of **Cockatrice** (v3.1.0 "Graduation Day") — the mature open-source MTG online client/server — has been completed at `/home/wall-o/projects/mtgonline/C++/ARCHITECTURE_ANALYSIS.md`. - -The play backend will follow the same authoritative server model, adapted for a modern web stack: - -#### Authority Model - -``` -┌─────────────────┐ WebSocket (JSON) ┌─────────────────┐ -│ CLIENT │◄────────────────────────────────────►│ SERVER │ -│ (React/TS) │ │ (FastAPI/Py) │ -│ │ │ │ -│ • Game Scene │ GameCommands (play, attack, etc.) │ • PostgreSQL │ -│ • Hand View │◄────────────────────────────────────►│ • Game State │ -│ • Chat Panel │ GameEvents (state changes) │ • Room/Player │ -│ • Deck Panel │ │ Management │ -│ • Phase Toolbar │ Chat, Admin, Spectator │ • Replay Log │ -└─────────────────┘ └─────────────────┘ -``` - -#### Game Engine Architecture - -| Cockatrice Component | Modern Web Equivalent | Role | -|---|---|---| -| `AbstractGame` | `Game` (server-side) | Core game instance holding state, players, event handler | -| `GameMetaInfo` | `GameMetadata` | gameId, maxPlayers, description, started, spectators | -| `GameState` | `GameBoardState` | currentPhase, activePlayer, hostId, gameTimer | -| `GameEventHandler` | `GameEventDispatcher` | Central dispatch — processes events, prepares commands (~21KB in Cockatrice) | -| `PlayerManager` | `PlayerRegistry` | Coordinates all players in a game | -| `PlayerLogic` | `Player` | Per-player game logic (~10.7KB in Cockatrice) | -| `PlayerActions` | `PlayerCommands` | Concrete commands: play, attack, tap, draw (~64KB in Cockatrice) | -| `CardZone` | `Zone` (base class) | Abstract zone — Hand/Stack/Table/Graveyard/Exile/Library | -| `HandZone` | `Hand` | Player's hand (secret/hidden zone) | -| `StackZone` | `Stack` | Spells/abilities on the stack | -| `TableZone` | `Battlefield` | Permanents on the battlefield | -| `PileZone` | `Pile` (Graveyard, Exile, Library) | Discard/exile/draw piles | -| `Replay` | `GameReplay` | Serialized event stream for replay | -| `Phase` | `TurnPhase` | 11-phase MTG turn structure | - -#### Turn Phase System (11 Phases with Sub-Phases) - -``` -Untap → Upkeep → Draw → Main 1 → Combat → Main 2 → End → Cleanup - │ - └── Sub-phases: - Beginning of Combat - Declare Attackers - Declare Blockers - Combat Damage - End of Combat -``` - -#### Command/Event Flow - -``` -User Action (React component) - │ - ▼ -GameCommand (JSON message) - │ - ▼ -GameEventDispatcher.process() - │ - ▼ -Player.handleCommand() - │ - ▼ -ZoneLogic (state mutation) - │ - ▼ -GameEvent (broadcast to all clients via WebSocket) - │ - ▼ -Client receives & updates UI via Zustand/XState -``` - -#### Network Protocol Design (Adapted from Cockatrice's protobuf) - -Cockatrice uses Protocol Buffers over TCP. The modern web equivalent replaces protobuf with JSON over WebSocket: - -| Cockatrice Proto Message | JSON WebSocket Message | -|---|---| -| `ServerInfo_Game` | `{ "type": "game_info", "game_id": 1, "max_players": 2, ... }` | -| `ServerInfo_Player` | `{ "type": "player_info", "player_id": 1, "name": "...", ... }` | -| `Command_PlayCard` | `{ "type": "cmd_play_card", "card_id": 42, "zone": "hand", ... }` | -| `Command_Attack` | `{ "type": "cmd_attack", "attacker_id": 7, "targets": [3, 5], ... }` | -| `Event_Join` | `{ "type": "event_join", "player_id": 2, "properties": {...} }` | -| `Event_Leave` | `{ "type": "event_leave", "player_id": 1, "reason": "..." }` | -| `Event_SetActivePlayer` | `{ "type": "event_active_player", "player_id": 1 }` | -| `Event_SetActivePhase` | `{ "type": "event_active_phase", "phase": 5 }` | -| `Event_GameSay` | `{ "type": "event_chat", "player_id": 1, "message": "..." }` | -| `GameReplay` | `{ "type": "replay", "events": [...] }` | - -#### Architecture Patterns to Reuse - -| Pattern | Cockatrice Usage | Modern Equivalent | -|---|---|---| -| **Event Bus** | Qt signals/slots | WebSocket broadcast + Zustand stores | -| **Command Pattern** | `Command_PlayCard`, `Command_Attack` | JSON command messages, validated server-side | -| **Observer** | `GameEventHandler` emits signals | WebSocket events trigger UI updates | -| **Strategy** | `CardZoneLogic` subclasses | Zone classes with strategy pattern (Hand, Stack, Table, Pile, View) | -| **Facade** | `AbstractGame` | Single `Game` object wrapping all subsystems | -| **Memento** | `DeckListMemento` for undo | Immutable state snapshots for undo/redo | -| **Repository** | `ServatriceDatabaseInterface` | SQLAlchemy repositories for game state, decks, logs | - -#### Key Decisions (Informed by Cockatrice Analysis) - -1. **Server is authoritative** — clients send commands, server validates and broadcasts events. No client-side state manipulation. -2. **Deterministic replay** — serialize all game events; same command sequence produces identical state. -3. **Zone abstraction** — each zone type (Hand, Stack, Battlefield, Pile) is independently implementable. -4. **Phase system** — 11-phase MTG turn with sub-phases, tracked server-side. -5. **Command/Event separation** — what a player *wants* to do vs. what *happens*. - -#### What Modernizes Cockatrice - -| Cockatrice Limitation | Modern Web Solution | -|---|---| -| TCP only, no web protocol | WebSocket (JSON) | -| Qt Widgets desktop UI | React/Next.js with PixiJS for game board | -| C++/MySQL backend | Python/FastAPI + PostgreSQL | -| Single-server clustering | Stateless game servers + Redis state cache | -| No mobile support | Responsive design from start | -| No REST API | FastAPI REST + WebSocket hybrid | -| Protobuf serialization | JSON over WebSocket | -| 106KB monolithic server handler | Modular service architecture | - -### 2.11 Play Backend Responsibilities (OUT OF SCOPE) - -The play backend will implement: -- [ ] Real-time game state management (server-authoritative) -- [ ] Multiplayer WebSocket communication -- [ ] Game rule enforcement (combat, stack resolution, priority) -- [ ] Deck validation during gameplay -- [ ] Game history and replay (serialized event log) -- [ ] Room and game lobbies -- [ ] Spectator mode -- [ ] Admin commands (kick, ban, game control) - -**Note**: The play backend lives in a separate codebase. Integration points with the card backend: -- **Card Backend API** (`http://backend:8000`): - - Authentication via JWT - - Card lookups: `GET /api/cards/{card_id}` - - Card search: `GET /api/cards/search?q=...` - - Set lists: `GET /api/sets/` - - User decks: `GET /api/users/{user_id}/decks` -- **Direct PSQL Access**: - - `mtgdata` database: Read card data, sets, etc. - - `mtgonline` database: Read user decks, validate deck legality - -## Phase 3: Frontend Development ✅ (COMPLETED) - -### 3.1 Project Setup -- [x] Initialize React + TypeScript project with Vite -- [x] Configure ESLint, Prettier, TypeScript strict mode -- [x] Set up Zustand for state management -- [x] Configure Tailwind CSS for styling -- [x] Set up Vitest + React Testing Library - -### 3.2 Authentication -- [x] Login form with JWT token storage -- [x] Registration form with validation -- [x] Protected routes and auth context -- [x] Session management and token refresh - -### 3.3 Deck Builder -- [x] Card search with filters (name, color, type, set) -- [x] Deck list editor with drag-and-drop -- [x] Import/export deck formats (plain text, native XML) -- [x] Folder management UI -- [x] Real-time deck statistics (card count, mana curve) -- [x] **NEW: Deck status indicator** (DRAFT vs FINAL) -- [x] **NEW: Card suggestion panel** (shows similar cards) - -### 3.4 Card Import Interface -- [x] File upload component (XLSX, CSV, JSON, ODS) -- [x] Import progress indicator -- [x] Match results display with confidence scores -- [x] Manual override for low-confidence matches -- [x] Import history and re-import capability - -### 3.5 Game Interface (TODO - Dependent on Play Backend) -- [ ] Game board visualization (zones, cards) -- [ ] Player hand (private zone) -- [ ] Library, graveyard, exile, command zones -- [ ] Card interaction (click, drag, hover) -- [ ] Real-time WebSocket updates - -### 3.6 Chat System -- [x] Room chat interface -- [x] Game chat (in-game messaging) -- [x] Player list display -- [x] Moderator tools (kick, ban) - -### 3.7 Admin Dashboard -- [x] User management interface -- [x] Ban/unban controls -- [x] Game logs viewer -- [x] System statistics - -## Phase 4: Integration & Polish (TODO) - -### 4.1 WebSocket Client -- [ ] WebSocket connection management -- [ ] Reconnection logic with exponential backoff -- [ ] Message serialization/deserialization -- [ ] Protocol buffer message handling - -### 4.2 Card Database -- [ ] Import MTJSON card data -- [ ] Cache card images locally -- [ ] Search and filter functionality -- [ ] Card tooltips with oracle text - -### 4.3 Game Logic (TODO - Dependent on Play Backend) -- [ ] Turn-based state management -- [ ] Priority system implementation -- [ ] Stack resolution -- [ ] Mana payment tracking -- [ ] Life totals and counters - -### 4.4 Performance -- [ ] Virtual scrolling for card lists -- [ ] Memoization and React.memo -- [ ] Code splitting and lazy loading -- [ ] WebSocket message batching - -## Phase 5: Advanced Features (FUTURE) - -### 5.1 Multiplayer Enhancements -- [ ] Spectator mode -- [ ] Game replay system -- [ ] Tournament support -- [ ] Custom game rules - -### 5.2 Card Database -- [ ] Set filtering -- [ ] Card comparison -- [ ] Deck sharing and discovery -- [ ] Community decks - -### 5.3 Mobile Support -- [ ] Responsive design -- [ ] PWA support -- [ ] Touch gestures -- [ ] Mobile-optimized controls - -### 5.4 Integrations -- [ ] OAuth providers (Google, GitHub) -- [ ] Discord bot integration -- [ ] API webhooks -- [ ] Third-party deck sharing platforms - -## Timeline - -| Phase | Duration | Status | -|-------|----------|--------| -| Phase 1: Backend Foundation | 2 weeks | ✅ Complete | -| Phase 2: User Data Schema & API | 2 weeks | ✅ Complete | -| Phase 3: Frontend Development | 4 weeks | ✅ Complete | -| Phase 4: Testing & Deployment | 1 week | Not Started | -| Phase 5: Advanced Features | Ongoing | Future | - -## Success Metrics - -- [ ] Users can create accounts and log in -- [ ] Users can create and edit decks -- [ ] Users can play multiplayer games in real-time -- [ ] Game state syncs correctly across all players -- [ ] Admins can manage users and monitor games -- [ ] API response time < 100ms for 95% of requests -- [ ] WebSocket latency < 50ms -- [ ] Zero critical security vulnerabilities - -## Dependencies - -### Backend -- Python 3.12+ -- PostgreSQL 14+ -- Redis (optional, for caching) -- Node.js 18+ (for protocol buffer compilation) - -### Frontend -- Node.js 18+ -- npm or pnpm -- Browser with WebSocket support - -### Development -- Git -- Docker (optional) -- VS Code or similar IDE -- PostgreSQL client (psql or DBeaver) - -## Risk Mitigation - -| Risk | Mitigation | -|------|------------| -| WebSocket reliability | Implement reconnection with exponential backoff | -| Game state inconsistency | Server-authoritative state with conflict resolution | -| Performance at scale | Load balancing, connection pooling, caching | -| Security vulnerabilities | Input validation, rate limiting, HTTPS | -| Data loss | Database backups, transaction logs | - -## Future Considerations - -- Migration to Rust for game server performance -- Integration with Magic Online API -- Support for custom card games -- AI-powered deck suggestions -- Blockchain-based card ownership verification diff --git a/SIMPLIFICATION_PLAN.md b/SIMPLIFICATION_PLAN.md deleted file mode 100644 index 9e7609d..0000000 --- a/SIMPLIFICATION_PLAN.md +++ /dev/null @@ -1,200 +0,0 @@ -# MTG Online Architecture Simplification Plan - -## Current State Analysis - -### Backend Structure -``` -backend/ -├── app/ -│ ├── core/ # Settings, database engines, Redis client, logging -│ ├── models/ # 10+ SQLAlchemy model files -│ ├── routers/ # 10+ router files -│ ├── schemas/ # 13+ Pydantic schema files -│ ├── services/ # 15+ service files (some massive) -│ └── main.py # FastAPI app with 10+ router mounts -├── mtg_rules_engine/ # 10+ files for game rules engine -├── alembic/ # Database migrations -├── Dockerfile -├── docker-compose.yml -└── requirements.txt -``` - -### Key Issues Identified -1. **Fragmented models** - 10+ separate model files that should be consolidated -2. **Massive services** - `mtgjson_manager.py` is 53KB, `mtgjson_loader.py` is 32KB -3. **Docker complexity** - Multi-database setup (PostgreSQL x2, Redis) with Docker Compose -4. **Async everywhere** - Async SQLAlchemy, asyncpg, async Redis - adds complexity -5. **Redundant schemas** - 13+ schema files with potential duplication -6. **Empty frontend** - Frontend directory exists but is empty -7. **Multiple entry points** - Scripts, migrations, Docker, direct Python execution - -### Router Analysis -- **auth.py** (162 lines) - Authentication endpoints -- **users.py** (168 lines) - User profile management -- **decks.py** (721 lines) - Deck management (largest router) -- **rooms.py** (158 lines) - Room CRUD -- **admin.py** (220 lines) - Admin operations -- **card_router.py** (292 lines) - MTG card search -- **interactions.py** (674 lines) - Card interactions -- **refresh.py** (101 lines) - MTGJSON refresh -- **user_data.py** (1714 lines) - User data (massive) -- **card_import.py** (368 lines) - Card import -- **games/router.py** (135 lines) - Game stubs -- **ws.py** (18 lines) - WebSocket - -### Model Analysis -- **models.py** (228 lines) - Core models (User, DecklistFile, Room, etc.) -- **mtg_models.py** (71 lines) - MTG card/set models -- **user_data.py** (325 lines) - User sessions, decks, replays, etc. -- **user_deck.py** (153 lines) - User deck building -- **mirror_models.py** (97 lines) - Card mirrors -- **card_import_batch.py** (74 lines) - Import tracking -- **relationships.py** (24 lines) - Cross-model relationships - -### Service Analysis -- **mtgjson_manager.py** (53KB) - MTGJSON data management -- **mtgjson_loader.py** (32KB) - Data loading -- **mtgjson_uploader.py** (34KB) - Data uploading -- **deck_manager.py** (7.8KB) - Deck management -- **card_database.py** (9.6KB) - Card database -- **game_server.py** (10KB) - Game server - -## Simplification Strategy - -### Phase 1: Core Layer Simplification -**Goal**: Simplify configuration and database layer - -1. **config.py** - Replace pydantic-settings with simple config class - - SQLite by default, PostgreSQL optional - - Single DATABASE_URL setting - - Sensible defaults for all settings - -2. **database.py** - Single database engine - - Remove dual database setup - - Support SQLite and PostgreSQL via same interface - - Keep async engine for compatibility - -3. **logging.py** - Simplified logging - - Remove Docker-specific complexity - - Keep structured logging - -4. **security.py** - Keep existing functionality - - JWT token management - - Password hashing - - Authentication dependencies - -### Phase 2: Model Consolidation -**Goal**: Reduce 10+ model files to 2-3 consolidated files - -1. **models.py** - All application models - - User, DecklistFile, Room, Ban, GameLog, AuditLog - - MtgonlineCard (local card mirror) - - UserDeck, UserDeckCard, DeckPrecedent, CardSuggestion - - UserSession, DeckVersion, GameReplay, etc. - - CardImportBatch, UserCardImportRecord - -2. **mtg_models.py** - All MTG card data models - - MtgSet, MtgCard - - MtgCardMirror, DeckCardLink - -3. **Remove**: relationships.py (integrate into models) - -### Phase 3: Router Consolidation -**Goal**: Reduce 12 router files to 6 focused routers - -1. **auth_router.py** - Authentication (from auth.py) - - Login, register, token refresh, current user - -2. **users_router.py** - User management + admin (merge users.py + admin.py) - - User profile, ban management, user listing, audit logs - -3. **decks_router.py** - Deck building (from decks.py) - - Deck CRUD, card management, precedents, suggestions - -4. **cards_router.py** - Card data (merge card_router.py + interactions.py) - - Card search, sets, types, rarities, interactions - -5. **user_data_router.py** - User data + imports (merge user_data.py + card_import.py) - - Sessions, replays, collection, wishlist, groups, networks - - Card imports - -6. **games_router.py** - Games + rooms + refresh (merge rooms.py + games + refresh.py) - - Room management, game stubs, MTGJSON refresh - -### Phase 4: Schema Consolidation -**Goal**: Reduce 13+ schema files to 6 matching routers - -1. **auth_schemas.py** - Auth request/response -2. **user_schemas.py** - User data schemas -3. **deck_schemas.py** - Deck schemas -4. **card_schemas.py** - Card schemas -5. **user_data_schemas.py** - User data schemas -6. **game_schemas.py** - Game schemas - -### Phase 5: Service Simplification -**Goal**: Break large services into focused modules - -1. **card_service.py** - Card search and lookup -2. **deck_service.py** - Deck CRUD and validation -3. **user_service.py** - User management -4. **mtgjson_service.py** - MTGJSON data management (split from 53KB file) -5. **game_service.py** - Game state management - -### Phase 6: Startup Script -**Goal**: Single entry point - -1. **run.py** - Simple startup script - ```python - import uvicorn - from app.core.database import create_tables - from app.main import app - - if __name__ == "__main__": - create_tables() - uvicorn.run(app, host="0.0.0.0", port=8000) - ``` - -### Phase 7: Documentation -**Goal**: Update all documentation - -1. **README.md** - New setup instructions -2. **Remove Docker files** - Keep only for production if needed -3. **Update state.json** - Document changes - -## Implementation Order - -1. **Core Layer** (config, database, logging, security) -2. **Models** (consolidate into 2-3 files) -3. **Schemas** (consolidate into 6 files) -4. **Services** (simplify and split) -5. **Routers** (consolidate into 6 files) -6. **Main** (update to use new structure) -7. **Run script** (create startup script) -8. **Documentation** (update README) -9. **Testing** (verify all endpoints work) - -## Success Criteria - -- [ ] Single SQLite database works out of the box -- [ ] All API endpoints functional -- [ ] No Docker required for development -- [ ] File count reduced by 50% -- [ ] Code lines reduced by 30% -- [ ] All tests passing -- [ ] Documentation updated -- [ ] `python run.py` starts everything - -## Risk Mitigation - -1. **Data integrity** - Keep all database migrations compatible -2. **Backward compatibility** - Maintain existing endpoint paths -3. **Testing** - Run comprehensive tests after each phase -4. **Rollback plan** - Keep original files as backup until verified - -## Timeline - -- Phase 1-2: Core and Models (1-2 hours) -- Phase 3-4: Routers and Schemas (2-3 hours) -- Phase 5-6: Services and Startup (1-2 hours) -- Phase 7: Documentation and Testing (1-2 hours) -- **Total: 5-9 hours** diff --git a/STATEMENT_OF_INTENT.md b/STATEMENT_OF_INTENT.md deleted file mode 100644 index 918c721..0000000 --- a/STATEMENT_OF_INTENT.md +++ /dev/null @@ -1,246 +0,0 @@ -# Statement of Intent — MTG Online Web - -## Project Title -**MTG Online Web** — A modern, web-based implementation of the MTG Online multiplayer Magic: The Gathering platform. - -## Vision Statement -To build a fully-featured, open-source multiplayer Magic: The Gathering platform that runs entirely in modern web browsers, eliminating the need for desktop software installations while maintaining compatibility with the existing MTG Online ecosystem. - -## Problem Statement -The original MTG Online application requires: -- Desktop software installation (Windows, macOS, Linux) -- Manual updates and dependency management -- Complex setup for new users -- Limited accessibility across devices -- No seamless integration with modern web workflows - -## Solution -A Progressive Web Application (PWA) that provides: -- **Zero Installation**: Runs directly in any modern web browser -- **Cross-Platform**: Works on desktop, tablet, and mobile devices -- **Instant Updates**: Users always have the latest version -- **MTG Online Compatible**: Interoperates with existing MTG Online users and protocol -- **Modern UX**: Contemporary interface design with modern web technologies - -## Core Objectives - -### 1. User Experience -- Intuitive, modern interface that rivals native desktop applications -- Real-time multiplayer gameplay with minimal latency -- Seamless deck building with import/export from MTG Online -- Responsive design that works across all screen sizes - -### 2. Technical Excellence -- **Backend**: Python 3.12 + FastAPI with async architecture -- **Frontend**: React + TypeScript with Zustand state management -- **Database**: PostgreSQL with async SQLAlchemy ORM -- **Real-time**: WebSocket-based game server for live multiplayer -- **Protocol**: Full compatibility with MTG Online protocol buffer messages - -### 3. Feature Parity with Desktop -- User authentication and account management -- Deck creation, editing, and storage -- Multiplayer game rooms with real-time state sync -- Card game mechanics (mana, phases, priority, stack) -- Admin/moderation tools (ban, warn, log viewing) -- Card database integration with comprehensive card data - -### 4. Performance -- API response times < 100ms for 95% of requests -- WebSocket latency < 50ms for game state updates -- Support for 1000+ concurrent users -- Sub-second page loads with proper caching - -### 5. Security -- JWT-based authentication with secure token storage -- bcrypt password hashing with configurable rounds -- Rate limiting to prevent abuse -- Input validation and sanitization -- HTTPS-only production deployment - -## Target Users - -### Primary Users -1. **Casual Players**: Users who want to play Magic online without installing software -2. **Mobile Users**: Players who want to play on tablets or phones -3. **New Players**: Users who find desktop setup intimidating - -### Secondary Users -1. **Tournament Organizers**: Need reliable, accessible platform for events -2. **Community Builders**: Want to create and manage playgroups -3. **Developers**: Want to integrate with or extend the MTG Online ecosystem - -### Existing MTG Online Users -- Seamless migration path -- Protocol compatibility for cross-play -- Familiar deck formats and card data - -## Key Differentiators - -| Feature | Desktop MTG Online | MTG Online Web | -|---------|-------------------|----------------| -| Installation | Required | None (browser-based) | -| Platform | Desktop only | Any device with browser | -| Updates | Manual | Automatic | -| Setup Complexity | High | Low | -| Accessibility | Limited | High | -| Mobile Support | Poor | Excellent | -| Protocol Support | Native | Compatible | - -## Success Criteria - -### Functional Requirements -- [ ] Users can create accounts and authenticate securely -- [ ] Users can create, edit, and manage decks -- [ ] Users can join and play multiplayer games in real-time -- [ ] Game state syncs correctly across all connected players -- [ ] Admin users can manage accounts and moderate games -- [ ] Deck formats are compatible with MTG Online desktop client - -### Performance Requirements -- [ ] 95% of API responses < 100ms -- [ ] WebSocket message latency < 50ms -- [ ] Support 1000+ concurrent users -- [ ] Page load time < 2 seconds on 4G connection -- [ ] Zero data loss during connection interruptions - -### Quality Requirements -- [ ] 80%+ test coverage for critical paths -- [ ] No critical security vulnerabilities -- [ ] WCAG 2.1 AA accessibility compliance -- [ ] Cross-browser compatibility (Chrome, Firefox, Safari, Edge) -- [ ] Mobile-responsive design - -### Business Requirements -- [ ] Open-source with community contributions -- [ ] Documentation for users and developers -- [ ] Deployment guides for self-hosting -- [ ] Integration with existing MTG Online ecosystem - -## Technology Choices - -### Backend -**Why Python/FastAPI?** -- Rapid development with excellent async support -- Type safety with Pydantic models -- Auto-generated API documentation -- Production-tested async framework -- Ecosystem for ML/AI features in future - -**Why PostgreSQL?** -- Robust transaction support -- Excellent JSONB support for flexible data -- Strong concurrency handling -- Mature ecosystem with async drivers - -**Why async SQLAlchemy?** -- Type-safe database operations -- Async support for high concurrency -- Migrations with Alembic -- Large community and ecosystem - -### Frontend -**Why React?** -- Component-based architecture -- Large ecosystem and community -- Excellent tooling and debugging -- Reusable component library - -**Why TypeScript?** -- Type safety catches bugs early -- Excellent IDE support -- Modern language features -- Growing adoption in enterprise - -**Why Zustand?** -- Simple, uncluttered state management -- No boilerplate -- Excellent performance -- Small bundle size - -### Protocol -**Why Protocol Buffer Compatibility?** -- Interoperability with existing MTG Online users -- Leverage existing card database and deck formats -- Community adoption pathway -- Battle-tested message format - -## Risk Assessment - -### High Risk -1. **WebSocket Reliability** - - Risk: Connection drops during games - - Mitigation: Reconnection logic, game state persistence - -2. **Game State Consistency** - - Risk: Players see different game states - - Mitigation: Server-authoritative state, conflict resolution - -3. **Performance at Scale** - - Risk: Degraded performance with many users - - Mitigation: Load balancing, connection pooling, caching - -### Medium Risk -1. **Mobile Browser Limitations** - - Risk: Some features may not work optimally - - Mitigation: Progressive enhancement, fallback strategies - -2. **Browser Compatibility** - - Risk: Older browsers may not support features - - Mitigation: Polyfills, feature detection, graceful degradation - -### Low Risk -1. **Database Migration** - - Risk: Schema changes cause issues - - Mitigation: Alembic migrations, backward compatibility - -2. **Protocol Changes** - - Risk: MTG Online protocol evolves - - Mitigation: Version support, backward compatibility - -## Future Enhancements - -### Short Term (3-6 months) -- Mobile app (React Native) -- Tournament management system -- Card image CDN optimization -- Performance monitoring and alerting - -### Medium Term (6-12 months) -- AI-powered deck suggestions -- Social features (friends, clans, notifications) -- Custom game modes and rules -- Integration with Magic Online - -### Long Term (12+ months) -- Multi-game support (other card games) -- Blockchain-based card ownership -- Advanced analytics and statistics -- Official tournament support - -## Commitment - -We commit to: -1. **Open Source**: Full transparency and community contributions -2. **Quality**: Production-ready code with comprehensive testing -3. **Security**: Regular security audits and updates -4. **Performance**: Continuous optimization and monitoring -5. **Accessibility**: Inclusive design for all users -6. **Documentation**: Clear guides for users and developers - -## Contact - -For questions, contributions, or support: -- GitHub: https://github.com/mtgonline-web -- Documentation: https://mtgonline-web.github.io/docs -- Discord: [Community server link] - -## Version - -**Version**: 0.1.0 -**Status**: Backend Foundation Complete -**Last Updated**: July 17, 2026 - ---- - -*"Building the future of online Magic: The Gathering, one web request at a time."* diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..1ae9b22 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,50 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +*.so +*.egg +*.egg-info +dist/ +build/ +.eggs/ + +.git +.gitignore +.gitattributes + +.venv/ +venv/ +env/ + +state.json +state.yaml +state.yml + +__backup__ +*.bak +*.old + +.env +.env.* +!.env.example + +.DS_Store +Thumbs.db + +node_modules/ + +.pytest_cache/ +.ruff_cache/ +htmlcov/ +.coverage +coverage.xml + +*.log +logs/ + +.idea/ +.vscode/ +*.swp +*.swo diff --git a/backend/ARCHITECTURAL_REPORT.md b/backend/ARCHITECTURAL_REPORT.md new file mode 100644 index 0000000..7415371 --- /dev/null +++ b/backend/ARCHITECTURAL_REPORT.md @@ -0,0 +1,1668 @@ +# MTG Online Backend - Comprehensive Architectural Report + +**Project:** MTG Online Backend +**Location:** `/home/wall-o/projects/mtgonline/backend/app/` +**Framework:** FastAPI with SQLAlchemy async +**Database:** SQLite (dev) / PostgreSQL (prod) + Redis caching +**Date:** 2025-07-21 + +--- + +## Executive Summary + +The MTG Online backend is a FastAPI application providing card game management features including user authentication, deck building, card search with fuzzy matching, card collection management, and real-time game WebSocket support. The codebase exhibits significant **architectural debt** with multiple layers of duplicated functionality (old vs. new router patterns), inconsistent database models, and redundant services. + +**Key Issues Identified:** +1. **Router Duplication:** Every major feature has both old and new router implementations +2. **Model Inconsistency:** Multiple model files with overlapping functionality +3. **Service Redundancy:** Multiple services performing similar operations +4. **Schema Confusion:** Duplicate schema definitions across files +5. **Mixed Database Patterns:** Some services use ORM, others use raw SQL + +--- + +## Layer Analysis + +### 1. Core Layer (`app/core/`) + +**Purpose:** Application infrastructure - configuration, database, security, logging, caching + +#### 1.1 `config.py` (216 lines) +**Purpose:** Application configuration management with environment variable overrides + +**Key Classes/Functions:** +- `Config` class: Configuration with sensible defaults +- `get_config()`: Singleton configuration instance +- `_load_env_file()`: Load .env files + +**Imports From:** +- `os` (stdlib) +- `typing.Optional` + +**Imported By:** +- `database.py` +- `security.py` +- `settings.py` (via config) + +**Dependencies:** None (standalone) + +--- + +#### 1.2 `database.py` (193 lines) +**Purpose:** SQLAlchemy async engine and session management + +**Key Classes/Functions:** +- `Base`: DeclarativeBase for ORM models +- `engine`: Async engine instance +- `async_session`: Session factory +- `get_db()`: FastAPI dependency for database sessions +- `create_tables()`: Create all tables +- Backward-compatible aliases: `mtg_engine`, `mtg_async_session`, `mtg_get_db`, `mirror_get_db` + +**Imports From:** +- `app.core.config`: `get_config` +- `sqlalchemy.ext.asyncio`: `AsyncSession`, `async_sessionmaker`, `create_async_engine` +- `sqlalchemy.orm`: `DeclarativeBase` + +**Imported By:** +- `models/__init__.py` +- `models/models.py` +- `models/mtg_models.py` +- `routers/*.py` (multiple) +- `services/*.py` (multiple) + +**Dependencies:** `config.py` + +--- + +#### 1.3 `logging.py` (187 lines) +**Purpose:** Structured logging configuration with JSON/text support + +**Key Classes/Functions:** +- `ContainerFormatter`: Custom log formatter +- `configure_logging()`: Configure application logging + +**Imports From:** +- `json`, `logging`, `os`, `sys`, `datetime`, `typing` + +**Imported By:** +- Imported by `logging_config.py` (duplicate) +- Used by various services + +**Dependencies:** None (standalone) + +--- + +#### 1.4 `logging_config.py` (97 lines) +**Purpose:** **DUPLICATE** of logging.py - structured logging configuration + +**Key Classes/Functions:** +- `ContainerFormatter`: Same as logging.py +- `configure_logging()`: Same as logging.py + +**Imports From:** +- `json`, `logging`, `os`, `sys`, `datetime` + +**Imported By:** +- Auto-configures on import + +**Dependencies:** None (standalone) + +**⚠️ ISSUE:** Exact duplicate of logging.py. Should be removed. + +--- + +#### 1.5 `redis_client.py` (90 lines) +**Purpose:** Redis client and caching layer + +**Key Classes/Functions:** +- `get_redis()`: Get Redis client instance +- `close_redis()`: Close Redis connection +- `cache_get()`, `cache_set()`, `cache_delete()`, `cache_invalidate_pattern()`: Cache operations + +**Imports From:** +- `app.core.settings`: `get_settings` +- `redis.asyncio`: `aioredis` + +**Imported By:** +- `routers/card_router.py` +- `routers/cards_router.py` +- `routers/interactions.py` + +**Dependencies:** `settings.py` + +--- + +#### 1.6 `security.py` (238 lines) +**Purpose:** Authentication and password hashing utilities + +**Key Classes/Functions:** +- `verify_password()`, `hash_password()`: Password operations +- `create_access_token()`, `create_refresh_token()`: JWT token creation +- `decode_token()`: JWT token validation +- `get_current_user()`: FastAPI dependency for auth + +**Imports From:** +- `app.core.config`: `get_config` +- `fastapi`: `Header`, `HTTPException`, `status` +- `jose`: `JWTError`, `jwt` +- `passlib.context`: `CryptContext` + +**Imported By:** +- `routers/auth.py` +- `routers/auth_router.py` +- `routers/users.py` +- `routers/users_router.py` +- `routers/admin.py` +- `routers/rooms.py` +- `routers/games_router.py` + +**Dependencies:** `config.py` + +--- + +#### 1.7 `settings.py` (29 lines) +**Purpose:** **BACKWARD COMPATIBILITY SHIM** - re-exports config module + +**Key Classes/Functions:** +- `get_settings`: Alias for `get_config` + +**Imports From:** +- `app.core.config`: `Config`, `get_config` + +**Imported By:** +- `redis_client.py` +- `services/mtgjson_loader.py` +- `services/mtgjson_uploader.py` + +**Dependencies:** `config.py` + +**⚠️ ISSUE:** Should be removed once all callers migrate to `config.py` + +--- + +### 2. Models Layer (`app/models/`) + +**Purpose:** SQLAlchemy ORM models for database entities + +#### 2.1 `__init__.py` (30 lines) +**Purpose:** Exports all models from models.py and mtg_models.py + +**Imports From:** +- `app.models.models`: All application models +- `app.models.mtg_models`: MtgSet, MtgCard, MtgCardMirror, DeckCardLink + +**Imported By:** +- Various routers and services + +**Dependencies:** `models.py`, `mtg_models.py` + +--- + +#### 2.2 `models.py` (968 lines) +**Purpose:** Main application ORM models + +**Key Classes:** +- **Core:** `User`, `DecklistFolder`, `DecklistFile`, `Room`, `RoomGameType`, `Ban`, `GameLog`, `AuditLog` +- **Card Mirror:** `MtgonlineCard` (local copy for deckbuilding) +- **User Decks:** `UserDeck`, `UserDeckCard`, `DeckPrecedent`, `DeckPrecedentCard`, `CardSuggestion` +- **User Data:** `UserSession`, `DeckVersion`, `GameReplay`, `ReplayPlayer`, `GameOutcome`, `UserStatistics`, `UserCardCollection`, `CardWishlist`, `UserGroup`, `GroupMember`, `GroupChatMessage`, `UserNetwork`, `NetworkMember`, `UserPreference`, `UserActivityLog` +- **Card Imports:** `CardImportBatch`, `UserCardImportRecord`, `UserCardImport` + +**Imports From:** +- `app.core.database`: `Base` +- `sqlalchemy`: Various column types, relationships + +**Imported By:** +- `models/__init__.py` +- `routers/*.py` (multiple) +- `services/*.py` (multiple) + +**Dependencies:** `database.py` + +--- + +#### 2.3 `mtg_models.py` (184 lines) +**Purpose:** MTG card data models + +**Key Classes:** +- `MtgSet`: MTG set data +- `MtgCard`: MTG card data (canonical) +- `MtgCardMirror`: Mirrored card data for fast queries +- `DeckCardLink`: Junction table linking decks to mirrored cards + +**Imports From:** +- `app.core.database`: `Base` +- `sqlalchemy`: Column types, relationships + +**Imported By:** +- `models/__init__.py` +- `routers/card_router.py` +- `routers/cards_router.py` +- `services/card_search_service.py` +- `services/deck_suggestion_service.py` + +**Dependencies:** `database.py` + +--- + +#### 2.4 `__backup__/` (Directory) +**Purpose:** Backup of old model files (not analyzed) + +--- + +### 3. Routers Layer (`app/routers/`) + +**Purpose:** FastAPI route handlers for API endpoints + +**⚠️ CRITICAL ISSUE:** Every major feature has **both old and new router implementations** with significant duplication. + +#### 3.1 `__init__.py` (29 lines) +**Purpose:** Exports consolidated routers + +**Imports From:** +- `app.routers.auth_router` +- `app.routers.users_router` +- `app.routers.decks_router` +- `app.routers.cards_router` +- `app.routers.user_data_router` +- `app.routers.games_router` + +**Imported By:** +- `app/main.py` + +**Dependencies:** All router modules + +--- + +#### 3.2 `auth.py` (162 lines) - **OLD VERSION** +**Purpose:** Authentication endpoints (login, refresh, register, me) + +**Key Endpoints:** +- `POST /login`: Authenticate user +- `POST /refresh`: Refresh access token +- `POST /register`: Register new user +- `GET /me`: Get current user + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `verify_password`, `hash_password`, `create_access_token`, `create_refresh_token`, `decode_token` +- `app.models.models`: `User` +- `app.schemas.schemas`: Login/refresh/user schemas + +**Imported By:** None (old version, superseded by auth_router.py) + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.3 `auth_router.py` (246 lines) - **NEW VERSION** +**Purpose:** Authentication endpoints (consolidated) + +**Key Endpoints:** Same as auth.py but with better structure + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: Token functions +- `app.models.models`: `User` +- `app.schemas.schemas`: Login/refresh/user schemas + +**Imported By:** +- `app/main.py` + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.4 `users.py` (168 lines) - **OLD VERSION** +**Purpose:** User management endpoints + +**Key Endpoints:** +- `GET /{user_id}`: Get user +- `PATCH /{user_id}`: Update user +- `POST /{user_id}/ban`: Ban user +- `POST /{user_id}/unban`: Unban user + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user`, `hash_password` +- `app.models.models`: `User` +- `app.schemas.schemas`: User schemas + +**Imported By:** None (old version, superseded by users_router.py) + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.5 `users_router.py` (549 lines) - **NEW VERSION** +**Purpose:** User management + admin operations (consolidated) + +**Key Endpoints:** +- User management: GET/PATCH user, ban/unban +- Admin operations: List users, list bans, create ban, unban by ID, list logs, audit log + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user`, `hash_password` +- `app.models.models`: `User`, `Ban`, `GameLog`, `AuditLog` +- `app.schemas.schemas`: User/ban schemas + +**Imported By:** +- `app/main.py` + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.6 `decks.py` (721 lines) - **OLD VERSION** +**Purpose:** Deck management endpoints + +**Key Endpoints:** +- CRUD for user decks +- Deck card management +- Precedent management +- Card search integration +- Card suggestions + +**Imports From:** +- `app.core.database`: `get_db`, `mtg_get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `User`, `DecklistFolder`, `MtgonlineCard` +- `app.models.user_deck`: `UserDeck`, `UserDeckCard`, etc. +- `app.services.deck_manager`: `DeckManager` +- `app.services.deck_parser`: `DeckParser` + +**Imported By:** None (old version, superseded by decks_router.py) + +**Dependencies:** Multiple + +--- + +#### 3.7 `decks_router.py` (810 lines) - **NEW VERSION** +**Purpose:** Deck management (consolidated) + +**Key Endpoints:** Same as decks.py but with better structure + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.mtg_models`: `MtgCard`, `MtgSet` +- `app.models.models`: `MtgonlineCard`, `User`, `DecklistFolder` +- `app.models.user_deck`: Deck models +- `app.services.deck_manager`: `DeckManager` +- `app.services.deck_parser`: `DeckParser` + +**Imported By:** +- `app/main.py` + +**Dependencies:** Multiple + +--- + +#### 3.8 `card_router.py` (292 lines) - **OLD VERSION** +**Purpose:** Card search endpoints + +**Key Endpoints:** +- `GET /api/cards/search`: Search cards +- `GET /api/cards/{card_id}`: Get card by ID +- `GET /api/cards/sets`: List sets +- `GET /api/cards/types`: List card types +- `GET /api/cards/rarities`: List rarities +- `GET /api/cards/suggest`: Suggest cards + +**Imports From:** +- `app.core.database`: `mtg_get_db` +- `app.core.redis_client`: `cache_get`, `cache_set` +- `app.services.card_search_service`: `CardSearchService` +- `app.services.deck_suggestion_service`: `DeckSuggestionService` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` +- `app.models.user_deck`: `UserDeck` +- `app.models.mtg_models`: `MtgCard` + +**Imported By:** None (old version, superseded by cards_router.py) + +**Dependencies:** Multiple + +--- + +#### 3.9 `cards_router.py` (995 lines) - **NEW VERSION** +**Purpose:** Card search + card interactions (consolidated) + +**Key Endpoints:** +- Card search: Same as card_router.py +- Card interactions: synergies, counters, evolutions, recommendations, stats + +**Imports From:** +- `app.core.database`: `mtg_get_db` +- `app.core.redis_client`: `cache_get`, `cache_set` +- `app.models.mtg_models`: `MtgCard` +- `app.models.user_deck`: `UserDeck` +- `app.schemas.card_search_schemas`: Card schemas +- `app.services.card_search_service`: `CardSearchService` +- `app.services.deck_suggestion_service`: `DeckSuggestionService` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` + +**Imported By:** +- `app/main.py` + +**Dependencies:** Multiple + +--- + +#### 3.10 `interactions.py` (674 lines) - **OLD VERSION** +**Purpose:** Card interaction endpoints (synergies, counters, evolutions) + +**Key Endpoints:** +- `GET /interactions/synergies/{card_id}` +- `GET /interactions/counters/{card_id}` +- `GET /interactions/evolutions/{card_id}` +- `GET /interactions/recommend/{card_id}` +- Search endpoints for each interaction type +- `GET /interactions/stats/{card_id}` + +**Imports From:** +- `app.core.database`: `mtg_get_db` +- `app.core.redis_client`: `cache_get`, `cache_set` + +**Imported By:** None (old version, superseded by cards_router.py) + +**Dependencies:** `database.py`, `redis_client.py` + +--- + +#### 3.11 `user_data.py` (1714 lines) - **OLD VERSION** +**Purpose:** User data endpoints (sessions, decks, replays, collections, groups, networks, preferences) + +**Key Endpoints:** +- Session management +- Deck versions +- Game replays +- Card collection +- Wishlist +- Groups with chat +- Networks +- Preferences +- Activity log + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `User` +- `app.models.user_data`: Various user data models +- `app.schemas.user_card_collection`: Collection schemas +- `app.schemas.user_data_schemas`: User data schemas + +**Imported By:** None (old version, superseded by user_data_router.py) + +**Dependencies:** Multiple + +--- + +#### 3.12 `user_data_router.py` (69121 bytes) - **NEW VERSION** +**Purpose:** User data + card import (consolidated) + +**Key Endpoints:** +- All user_data.py endpoints +- Card import endpoints (upload, status, results, confirm) + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `User`, `MtgonlineCard` +- `app.models.user_data`: User data models +- `app.models.card_import_batch`: `CardImportBatch` +- `app.models.user_card_import_record`: `UserCardImportRecord` +- `app.models.user_deck`: `UserDeck`, `UserDeckCard` +- `app.services.file_parser`: `FileParser` +- `app.services.import_batch_processor`: `ImportBatchProcessor` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` +- `app.schemas.*`: Various schemas + +**Imported By:** +- `app/main.py` + +**Dependencies:** Multiple + +--- + +#### 3.13 `rooms.py` (158 lines) - **OLD VERSION** +**Purpose:** Room management endpoints + +**Key Endpoints:** +- `GET /`: List rooms +- `GET /{room_id}`: Get room +- `POST /`: Create room (admin) +- `PATCH /{room_id}`: Update room (admin) +- `DELETE /{room_id}`: Delete room (admin) + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `Room` +- `app.schemas.schemas`: `RoomResponse` + +**Imported By:** None (old version, superseded by games_router.py) + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.14 `games/__init__.py` (4 lines) +**Purpose:** Games router package init + +**Imports From:** +- `app.routers.games.router`: `router` + +**Imported By:** None + +**Dependencies:** `games/router.py` + +--- + +#### 3.15 `games/router.py` (135 lines) - **OLD VERSION** +**Purpose:** Game management stubs + +**Key Endpoints:** +- `GET /`: List games (stub) +- `POST /`: Create game (mock) +- `GET /{game_id}`: Get game (mock) +- `POST /{game_id}/join`: Join game (stub) +- `POST /{game_id}/leave`: Leave game (stub) +- `POST /{game_id}/start`: Start game (stub) +- `POST /{game_id}/end`: End game (stub) + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `User` +- `app.schemas.schemas`: `GameCreate`, `GameResponse` + +**Imported By:** None (old version, superseded by games_router.py) + +**Dependencies:** `database.py`, `security.py`, `models.py`, `schemas.py` + +--- + +#### 3.16 `games_router.py` (487 lines) - **NEW VERSION** +**Purpose:** Room management + game stubs + MTGJSON refresh + WebSocket (consolidated) + +**Key Endpoints:** +- Room CRUD (prefix: /rooms) +- Game stubs (prefix: /games) +- MTGJSON refresh (prefix: /mtgjson) +- WebSocket (prefix: /ws/game/{game_id}) + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.models`: `Room`, `User` +- `app.schemas.schemas`: `RoomResponse`, `GameCreate`, `GameResponse` +- `app.services.mtgjson_manager`: `MTGJSONManager` +- `app.services.game_server`: `game_websocket_endpoint` + +**Imported By:** +- `app/main.py` + +**Dependencies:** Multiple + +--- + +#### 3.17 `refresh.py` (101 lines) - **OLD VERSION** +**Purpose:** MTGJSON refresh endpoints + +**Key Endpoints:** +- `POST /mtgjson/refresh`: Trigger refresh +- `GET /mtgjson/status`: Get status +- `POST /mtgjson/verify`: Verify files + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.services.mtgjson_manager`: `MTGJSONManager` + +**Imported By:** None (old version, superseded by games_router.py) + +**Dependencies:** `database.py`, `security.py`, `services/mtgjson_manager.py` + +--- + +#### 3.18 `card_import.py` (368 lines) - **OLD VERSION** +**Purpose:** Card import endpoints + +**Key Endpoints:** +- `POST /import`: Upload import file +- `GET /import/{import_id}/status`: Get status +- `GET /import/{import_id}/results`: Get results +- `POST /import/{import_id}/confirm`: Confirm import +- `GET /user/cards`: List user cards +- `DELETE /user/cards/{card_import_id}`: Delete card +- `GET /user/decks`: List user decks + +**Imports From:** +- `app.core.database`: `get_db` +- `app.core.security`: `get_current_user` +- `app.models.card_import_batch`: `CardImportBatch` +- `app.models.user_card_import_record`: `UserCardImportRecord` +- `app.models.user_data`: `UserCardCollection` +- `app.models.models`: `MtgonlineCard` +- `app.models.user_deck`: `UserDeck`, `UserDeckCard` +- `app.services.file_parser`: `FileParser` +- `app.services.import_batch_processor`: `ImportBatchProcessor` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` + +**Imported By:** None (old version, superseded by user_data_router.py) + +**Dependencies:** Multiple + +--- + +#### 3.19 `ws.py` (512 bytes) - **OLD VERSION** +**Purpose:** WebSocket endpoint for game connections + +**Key Endpoints:** +- `GET /ws/game/{game_id}`: WebSocket connection + +**Imports From:** +- `app.services.game_server`: `game_websocket_endpoint` + +**Imported By:** None (old version, superseded by games_router.py) + +**Dependencies:** `services/game_server.py` + +--- + +### 4. Schemas Layer (`app/schemas/`) + +**Purpose:** Pydantic models for request/response validation + +#### 4.1 `__init__.py` (147 lines) +**Purpose:** Exports all schemas from various schema files + +**Imports From:** +- `app.schemas.schemas`: Basic schemas +- `app.schemas.user_data_schemas`: User data schemas +- `app.schemas.user_card_collection`: Collection schemas +- `app.schemas.user_deck_schemas`: Deck schemas +- `app.schemas.card_import_schemas`: Import schemas +- `app.schemas.card_search_schemas`: Search schemas +- `app.schemas.game_schemas`: Game schemas +- `app.schemas.mtg_card_schemas`: MTG card schemas +- `app.schemas.proto_messages`: Protocol messages +- `app.schemas.protocol_constants`: Protocol constants + +**Imported By:** +- Various routers + +**Dependencies:** All schema files + +--- + +#### 4.2 `schemas.py` (286 lines) +**Purpose:** Basic Pydantic models for auth, users, decks, games, bans + +**Key Classes:** +- Auth: `LoginRequest`, `LoginResponse`, `RefreshTokenRequest`, `TokenResponse` +- User: `UserBase`, `UserCreate`, `UserUpdate`, `UserResponse` +- Deck: `DeckCreate`, `DeckUpdate`, `DeckResponse`, `FolderCreate`, `FolderResponse` +- Game: `GameCreate`, `GameResponse` +- Room: `RoomResponse` +- Ban: `BanCreate`, `BanResponse` +- Error: `ErrorResponse`, `ValidationErrorResponse` +- Pagination: `PaginationParams`, `PaginatedResponse` +- Card Mirror: `CardMirrorResponse`, `DeckCardLinkResponse`, `DeckWithCardsResponse` + +**Imports From:** +- `pydantic`: `BaseModel`, `EmailStr`, `Field`, `ConfigDict` +- `typing`: `Optional`, `List`, `Dict` +- `datetime`: `datetime` + +**Imported By:** +- `routers/auth.py` +- `routers/auth_router.py` +- `routers/users.py` +- `routers/users_router.py` +- `routers/rooms.py` +- `routers/games_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.3 `user_data_schemas.py` (434 lines) +**Purpose:** User data schemas (sessions, decks, replays, collections, groups, networks, preferences) + +**Key Classes:** +- Enums: `DeckVersionStatus`, `GameReplayStatus`, `GameOutcomeType`, `GroupMemberRole`, `NetworkMemberRole`, `UserPreferenceTheme`, `ActivityType` +- Session: `SessionResponse`, `SessionCleanupResponse` +- Deck Version: `DeckVersionCreate`, `DeckVersionUpdate`, `DeckVersionResponse`, `DeckVersionListResponse` +- Game Replay: `GameReplayCreate`, `GameReplayUpdate`, `GameReplayResponse`, `GameReplayListResponse` +- Game Outcome: `GameOutcomeCreate`, `GameOutcomeResponse`, `GameOutcomeListResponse` +- User Statistics: `UserStatisticsResponse`, `StatisticsUpdateResponse` +- Group: `GroupCreate`, `GroupUpdate`, `GroupMemberCreate`, `GroupMemberUpdate`, `GroupMemberRemove`, `GroupResponse`, `GroupListResponse`, `GroupChatMessageCreate`, `GroupChatMessageResponse`, `GroupChatMessageListResponse` +- Network: `NetworkCreate`, `NetworkUpdate`, `NetworkMemberCreate`, `NetworkResponse`, `NetworkListResponse` +- Preference: `UserPreferenceUpdate`, `UserPreferenceResponse` +- Activity Log: `ActivityLogEntry`, `ActivityLogListResponse` +- Generic: `MessageResponse`, `CountResponse`, `ErrorDetail` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `Optional`, `List`, `Dict`, `Any` +- `datetime`: `datetime` +- `enum`: `Enum` + +**Imported By:** +- `routers/user_data.py` +- `routers/user_data_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.4 `user_deck_schemas.py` (217 lines) +**Purpose:** Deck building schemas + +**Key Classes:** +- Enums: `DeckStatus`, `DeckZone`, `SuggestionType` +- Deck: `UserDeckCreate`, `UserDeckUpdate`, `UserDeckResponse`, `UserDeckListResponse` +- Deck Card: `DeckCardCreate`, `DeckCardUpdate`, `DeckCardResponse`, `DeckCardWithDetailsResponse`, `DeckCardListResponse` +- Precedent: `PrecedentCreate`, `PrecedentUpdate`, `PrecedentResponse`, `PrecedentListResponse` +- Suggestion: `SuggestionCreate`, `SuggestionResponse`, `SuggestionListResponse` +- Actions: `DeckFinalizeRequest`, `DeckFinalizeResponse`, `DeckDeleteResponse` +- Search: `CardSearchRequest` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `Optional`, `List`, `Dict`, `Any` +- `datetime`: `datetime` +- `enum`: `Enum` + +**Imported By:** +- `routers/decks.py` +- `routers/decks_router.py` +- `routers/card_import.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.5 `user_card_collection.py` (157 lines) +**Purpose:** Card collection schemas + +**Key Classes:** +- Enums: `CardCondition`, `AcquisitionMethod` +- Collection: `CardCollectionCreate`, `CardCollectionUpdate`, `CardCollectionResponse`, `CardCollectionListResponse` +- Wishlist: `WishlistCreate`, `WishlistUpdate`, `WishlistResponse`, `WishlistListResponse` +- Statistics: `CollectionStatistics`, `CollectionSummaryResponse` +- Generic: `MessageResponse`, `CountResponse`, `ErrorDetail` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `Optional`, `List`, `Dict`, `Any` +- `datetime`: `datetime` +- `enum`: `Enum` + +**Imported By:** +- `routers/user_data.py` +- `routers/user_data_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.6 `card_import_schemas.py` (113 lines) +**Purpose:** Card import schemas + +**Key Classes:** +- Import: `CardImportRequest`, `CardImportResponse`, `CardImportStatusResponse` +- Match: `CardMatchResult`, `CardImportSummary` +- Batch: `CardImportBatchCreate`, `CardImportBatchResponse` +- User Import: `UserCardImportCreate`, `UserCardImportResponse`, `UserCardImportRecordResponse` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `List`, `Optional`, `Dict`, `Any` +- `datetime`: `datetime` + +**Imported By:** +- `routers/card_import.py` +- `routers/user_data_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.7 `card_search_schemas.py` (108 lines) +**Purpose:** Card search and import schemas + +**Key Classes:** +- Search: `CardResponse`, `SetResponse`, `CardTypeResponse`, `CardSearchResponse` +- Import: `CardImportResponse`, `CardImportStatusResponse`, `CardMatchResult`, `CardImportSummary` +- Generic: `MessageResponse`, `CountResponse`, `ErrorResponse` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `List`, `Optional`, `Dict`, `Any` +- `datetime`: `datetime` + +**Imported By:** +- `routers/card_router.py` +- `routers/cards_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.8 `mtg_card_schemas.py` (116 lines) +**Purpose:** MTG card schemas + +**Key Classes:** +- Card: `MtgCardResponse`, `MtgCardSearchRequest`, `MtgCardSearchResponse` +- Set: `MtgSetResponse` +- Mirror: `MtgCardMirrorResponse` +- Link: `DeckCardLinkResponse`, `DeckWithCardsResponse` + +**Imports From:** +- `pydantic`: `BaseModel`, `ConfigDict` +- `typing`: `Optional`, `List` +- `datetime`: `datetime` + +**Imported By:** +- `routers/cards_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.9 `game_schemas.py` (59 lines) +**Purpose:** Game schemas + +**Key Classes:** +- Game: `GameCreate`, `GameResponse`, `GameJoinRequest`, `GameLeaveRequest`, `GameListResponse`, `GamePlayerResponse`, `GameStateResponse` + +**Imports From:** +- `pydantic`: `BaseModel`, `Field`, `ConfigDict` +- `typing`: `Optional`, `List` +- `datetime`: `datetime` + +**Imported By:** +- `routers/games_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.10 `generic_schemas.py` (29 lines) +**Purpose:** Generic response schemas + +**Key Classes:** +- `MessageResponse`, `CountResponse`, `ErrorResponse`, `ValidationErrorResponse`, `ErrorDetail` + +**Imports From:** +- `pydantic`: `BaseModel` +- `typing`: `Optional`, `List` + +**Imported By:** +- `routers/user_data.py` +- `routers/user_data_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 4.11 `proto_messages.py` (133 lines) +**Purpose:** Protocol buffer message models + +**Key Classes:** +- Base: `ProtoMessageBase` +- Commands: `SessionCommand`, `GameCommand`, `GameEvent`, `Response` +- Server Info: `ServerInfoUser`, `ServerInfoDeckStorageFile`, `ServerInfoDeckStorageFolder`, `ServerInfoDeckStorageTreeItem`, `ServerInfoCard`, `ServerInfoZone`, `ServerInfoGame` + +**Imports From:** +- `pydantic`: `BaseModel` +- `typing`: `Optional`, `List`, `Dict`, `Any` +- `datetime`: `datetime` + +**Imported By:** +- None (unused) + +**Dependencies:** None (standalone) + +**⚠️ ISSUE:** Protocol buffer messages defined but not used anywhere. + +--- + +#### 4.12 `protocol_constants.py` (170 lines) +**Purpose:** MTG Online protocol constants + +**Key Classes:** +- `SessionCommandType`: Session command types (PING, LOGIN, MESSAGE, etc.) +- `GameCommandType`: Game command types (KICK, LEAVE, SHUFFLE, etc.) +- `GameEventType`: Game event types (JOIN, LEAVE, DRAW_CARDS, etc.) +- `ResponseCode`: Response codes (RespOk, RespInvalidCommand, etc.) +- `ZoneType`: Zone types (PrivateZone, PublicZone, HiddenZone) +- `UserLevelFlag`: User level flags (IsUser, IsModerator, IsAdmin, etc.) + +**Imports From:** +- `enum`: `IntEnum`, `IntFlag` + +**Imported By:** +- None (unused) + +**Dependencies:** None (standalone) + +**⚠️ ISSUE:** Protocol constants defined but not used anywhere. + +--- + +### 5. Services Layer (`app/services/`) + +**Purpose:** Business logic and data processing + +#### 5.1 `__init__.py` (60 lines) +**Purpose:** Exports all services + +**Imports From:** +- `app.services.deck_parser`: `DeckParser` +- `app.services.card_database`: Various card database functions +- `app.services.card_mirror_service`: Mirror service functions +- `app.services.mtgjson_manager`: `MTGJSONManager`, `get_manager` +- `app.services.mtgjson_downloader`: Download functions +- `app.services.mtgjson_loader`: Loader functions +- `app.services.mtgjson_uploader`: Uploader functions +- `app.services.file_parser`: `FileParser` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` +- `app.services.import_batch_processor`: `ImportBatchProcessor` +- `app.services.deck_manager`: `DeckManager` +- `app.services.card_search_service`: `CardSearchService` +- `app.services.deck_suggestion_service`: `DeckSuggestionService` + +**Imported By:** +- Various routers + +**Dependencies:** All service modules + +--- + +#### 5.2 `card_database.py` (350 lines) +**Purpose:** MTG card database queries + +**Key Functions:** +- `search_cards()`: Search cards by name/type/mana cost +- `get_card_by_name()`: Get card by name +- `get_cards_by_set()`: Get cards by set +- `get_card_types()`: Get all card types +- `get_card_rarities()`: Get all rarities +- `get_sets()`: Get all sets +- `get_set_by_code()`: Get set by code +- `get_card_statistics()`: Get database statistics + +**Imports From:** +- `app.models.mtg_models`: `MtgCard`, `MtgSet` +- `app.core.database`: `mtg_get_db` +- `sqlalchemy`: `select`, `and_`, `or_`, `func` +- `sqlalchemy.orm`: `selectinload` + +**Imported By:** +- `routers/card_router.py` + +**Dependencies:** `models/mtg_models.py`, `core/database.py` + +--- + +#### 5.3 `card_mirror_service.py` (359 lines) +**Purpose:** Card mirror service for fast deckbuilding queries + +**Key Functions:** +- `upsert_card_mirror()`: Upsert card to mirror +- `get_card_mirror_by_name()`: Get mirror by name +- `search_card_mirrors()`: Search mirrors +- `add_card_to_deck()`: Add card to deck +- `remove_card_from_deck()`: Remove card from deck +- `get_deck_cards()`: Get deck cards +- `get_user_deck_summaries()`: Get user deck summaries +- `sync_mirrors_from_mtg_cards()`: Sync mirrors +- `get_card_statistics()`: Get mirror statistics + +**Imports From:** +- `app.models.mtg_models`: `MtgCard`, `MtgSet` +- `app.models.mirror_models`: `MtgCardMirror`, `DeckCardLink` +- `app.core.database`: `mirror_get_db` +- `sqlalchemy`: `select`, `func`, `update`, `delete` +- `sqlalchemy.orm`: `selectinload` + +**Imported By:** +- None (unused) + +**Dependencies:** `models/mtg_models.py`, `core/database.py` + +**⚠️ ISSUE:** Service defined but not used anywhere. + +--- + +#### 5.4 `card_search_service.py` (190 lines) +**Purpose:** Card search with filters + +**Key Classes/Functions:** +- `CardSearchService.search_cards()`: Search with filters +- `CardSearchService.get_card_by_id()`: Get by ID +- `CardSearchService.get_sets()`: Get all sets +- `CardSearchService.get_card_types()`: Get card types +- `CardSearchService.get_card_rarities()`: Get rarities + +**Imports From:** +- `app.models.mtg_models`: `MtgCard`, `MtgSet` +- `app.models.mirror_models`: `MtgCardMirror` +- `sqlalchemy`: `select`, `or_`, `and_` +- `sqlalchemy.orm`: `selectinload` + +**Imported By:** +- `routers/card_router.py` +- `routers/cards_router.py` + +**Dependencies:** `models/mtg_models.py`, `sqlalchemy` + +--- + +#### 5.5 `deck_manager.py` (246 lines) +**Purpose:** Deck CRUD operations + +**Key Classes/Functions:** +- `DeckManager.create_deck()`: Create deck +- `DeckManager.get_deck()`: Get deck +- `DeckManager.list_decks()`: List decks +- `DeckManager.update_deck()`: Update deck +- `DeckManager.delete_deck()`: Delete deck +- `DeckManager.finalize_deck()`: Finalize deck +- `DeckManager.add_card_to_deck()`: Add card +- `DeckManager.get_deck_cards()`: Get deck cards +- `DeckManager.update_deck_card()`: Update card +- `DeckManager.remove_card_from_deck()`: Remove card +- `DeckManager.clone_precedent()`: Clone precedent + +**Imports From:** +- `app.models.user_deck`: `UserDeck`, `UserDeckCard`, `DeckPrecedent`, `DeckPrecedentCard` +- `app.models.models`: `MtgonlineCard` +- `sqlalchemy`: `select`, `update`, `delete` +- `sqlalchemy.orm`: `selectinload` + +**Imported By:** +- `routers/decks.py` +- `routers/decks_router.py` + +**Dependencies:** `models/user_deck.py`, `models/models.py`, `sqlalchemy` + +--- + +#### 5.6 `deck_parser.py` (206 lines) +**Purpose:** Deck list parsing and serialization + +**Key Classes/Functions:** +- `CardInfo`: Dataclass for card information +- `DeckParser.parse_plain_text()`: Parse plain text +- `DeckParser.format_plain_text()`: Format as plain text +- `DeckParser.to_native_xml()`: Convert to XML +- `DeckParser.from_native_xml()`: Parse XML +- `parse_deck()`: Convenience function +- `format_deck()`: Convenience function + +**Imports From:** +- `re`: Regex patterns +- `typing`: `List`, `Dict`, `Tuple`, `Optional` +- `dataclasses`: `dataclass` + +**Imported By:** +- `routers/decks.py` +- `routers/decks_router.py` + +**Dependencies:** None (standalone) + +--- + +#### 5.7 `deck_suggestion_service.py` (209 lines) +**Purpose:** Card suggestion service + +**Key Classes/Functions:** +- `DeckSuggestionService.suggest_cards()`: Suggest cards +- `DeckSuggestionService.add_suggestion()`: Add suggestion +- `DeckSuggestionService.get_deck_suggestions()`: Get suggestions + +**Imports From:** +- `app.models.user_deck`: `UserDeck`, `UserDeckCard`, `CardSuggestion` +- `app.models.mtg_models`: `MtgCard` +- `app.models.mirror_models`: `MtgCardMirror` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` +- `sqlalchemy`: `select`, `or_`, `and_`, `func`, `case` +- `sqlalchemy.orm`: `selectinload` + +**Imported By:** +- `routers/card_router.py` +- `routers/cards_router.py` + +**Dependencies:** `models/user_deck.py`, `models/mtg_models.py`, `services/fuzzy_card_matcher.py`, `sqlalchemy` + +--- + +#### 5.8 `file_parser.py` (108 lines) +**Purpose:** File parser for card imports + +**Key Classes/Functions:** +- `FileParser.parse_file()`: Parse file (CSV, JSON, XLSX, ODS) +- `FileParser._parse_csv()`: Parse CSV +- `FileParser._parse_json()`: Parse JSON +- `FileParser._parse_xlsx()`: Parse XLSX +- `FileParser._parse_ods()`: Parse ODS + +**Imports From:** +- `csv`, `json`: Stdlib +- `pathlib`: `Path` +- `openpyxl`: XLSX parsing +- `pandas`: ODS parsing + +**Imported By:** +- `routers/card_import.py` +- `routers/user_data_router.py` + +**Dependencies:** `openpyxl`, `pandas` + +--- + +#### 5.9 `fuzzy_card_matcher.py` (170 lines) +**Purpose:** Fuzzy matching for card names + +**Key Classes/Functions:** +- `FuzzyCardMatcher.normalize_card_name()`: Normalize name +- `FuzzyCardMatcher.exact_match()`: Exact match +- `FuzzyCardMatcher.fuzzy_match()`: Fuzzy match +- `FuzzyCardMatcher.find_best_match()`: Find best match +- `FuzzyCardMatcher.batch_match()`: Batch match +- `FuzzyCardMatcher.batch_match_with_database()`: Match with DB + +**Imports From:** +- `thefuzz`: `fuzz` +- `typing`: `List`, `Tuple`, `Optional` + +**Imported By:** +- `routers/card_router.py` +- `routers/cards_router.py` +- `routers/card_import.py` +- `routers/user_data_router.py` +- `services/deck_suggestion_service.py` +- `services/import_batch_processor.py` + +**Dependencies:** `thefuzz` + +--- + +#### 5.10 `import_batch_processor.py` (204 lines) +**Purpose:** Batch import processing + +**Key Classes/Functions:** +- `ImportBatchProcessor.create_batch()`: Create batch +- `ImportBatchProcessor.process_batch()`: Process batch +- `ImportBatchProcessor.get_batch_status()`: Get status +- `ImportBatchProcessor.get_batch_results()`: Get results +- `ImportBatchProcessor.confirm_batch()`: Confirm batch +- `ImportBatchProcessor.get_user_imports()`: Get user imports +- `ImportBatchProcessor.delete_batch()`: Delete batch + +**Imports From:** +- `app.models.card_import_batch`: `CardImportBatch` +- `app.models.user_card_import_record`: `UserCardImportRecord` +- `app.models.user_data`: `UserCardCollection` +- `app.models.models`: `MtgonlineCard` +- `app.services.fuzzy_card_matcher`: `FuzzyCardMatcher` +- `sqlalchemy`: `select`, `update`, `insert`, `delete` +- `sqlalchemy.sql`: `func` + +**Imported By:** +- `routers/card_import.py` +- `routers/user_data_router.py` + +**Dependencies:** `models/card_import_batch.py`, `models/user_card_import_record.py`, `models/user_data.py`, `models/models.py`, `services/fuzzy_card_matcher.py`, `sqlalchemy` + +--- + +#### 5.11 `game_server.py` (309 lines) +**Purpose:** WebSocket game server for real-time gameplay + +**Key Classes/Functions:** +- `GameRoom`: Manages single game room +- `GameServer`: Manages all game rooms +- `game_websocket_endpoint()`: WebSocket endpoint +- `process_game_command()`: Process commands + +**Imports From:** +- `asyncio`: `Lock` +- `json`: Serialization +- `typing`: `Dict`, `Set`, `Optional` +- `datetime`: `datetime` +- `fastapi`: `WebSocket`, `WebSocketDisconnect` + +**Imported By:** +- `routers/games_router.py` +- `routers/ws.py` + +**Dependencies:** `fastapi` + +--- + +#### 5.12 `mtgjson_downloader.py` (137 lines) +**Purpose:** MTGJSON data downloader + +**Key Functions:** +- `get_downloads_dir()`: Get downloads directory +- `download_file()`: Download single file +- `download_all_files()`: Download all files +- `verify_downloads()`: Verify downloads +- `get_file_list()`: Get file list + +**Imports From:** +- `gzip`, `json`, `os`, `sys`: Stdlib +- `pathlib`: `Path` +- `urllib.request`: `urlretrieve` +- `urllib.parse`: `urljoin` + +**Imported By:** +- `services/__init__.py` + +**Dependencies:** None (standalone) + +--- + +#### 5.13 `mtgjson_loader.py` (838 lines) +**Purpose:** MTGJSON data loader (legacy) + +**Key Functions:** +- `create_tables()`: Create tables +- `download_file()`: Download file +- `extract_zip()`: Extract zip +- `get_all_printings_psql_file()`: Get PSQL file +- `parse_psql_file()`: Parse PSQL +- `import_cards()`: Import cards +- `import_all_printings_psql()`: Import AllPrintings +- `import_all_set_files()`: Import set files +- `import_all_identifiers()`: Import identifiers +- `import_all_deck_files()`: Import deck files +- `import_simple_json()`: Import JSON +- `create_indexes()`: Create indexes +- `show_summary()`: Show summary +- `main()`: Main function + +**Imports From:** +- `app.core.settings`: `get_settings` +- `sqlalchemy.ext.asyncio`: `create_async_engine`, `AsyncSession` +- `sqlalchemy.orm`: `sessionmaker` +- `sqlalchemy`: `text`, `insert`, `update`, `select`, `Table`, `Column`, etc. +- `sqlalchemy.dialects.postgresql`: `insert as pg_insert` + +**Imported By:** +- `services/__init__.py` + +**Dependencies:** `core/settings.py`, `sqlalchemy` + +--- + +#### 5.14 `mtgjson_manager.py` (1296 lines) +**Purpose:** MTGJSON data manager (local file processing) + +**Key Classes/Functions:** +- `MTGJSONManager`: Main manager class +- `get_last_refresh()`: Get last refresh +- `get_health_status()`: Get health status +- `check_required_files()`: Check files +- `upsert_all()`: Upsert all data +- `_upsert_sets()`: Upsert sets +- `_upsert_cards()`: Upsert cards +- `_upsert_cards_from_psql()`: Upsert from PSQL +- `_upsert_identifiers()`: Upsert identifiers +- `_upsert_card_types()`: Upsert card types +- `_upsert_keywords()`: Upsert keywords +- `_upsert_set_list()`: Upsert set list +- `_upsert_deck_list()`: Upsert deck list +- `_upsert_deck_files_from_zip()`: Upsert deck files +- `log_refresh()`: Log refresh +- `sync_mirrors()`: Sync mirrors +- `run_refresh()`: Run refresh +- `download_files()`: Download files +- `verify_files()`: Verify files +- `download_and_refresh()`: Download and refresh +- `get_manager()`: Get manager instance + +**Imports From:** +- `app.core.database`: `mtg_async_session` +- `json`, `logging`, `zipfile`, `tempfile`, `shutil`: Stdlib +- `pathlib`: `Path` +- `datetime`: `datetime` +- `typing`: `Optional`, `Dict`, `List`, `Tuple` +- `re`: Regex +- `aiohttp`: `ClientSession`, `ClientTimeout` +- `asyncio`: `sleep` +- `gzip`: Decompression +- `sqlalchemy`: `text` + +**Imported By:** +- `routers/games_router.py` +- `routers/refresh.py` +- `services/__init__.py` + +**Dependencies:** `core/database.py`, `aiohttp`, `sqlalchemy` + +--- + +#### 5.15 `mtgjson_uploader.py` (883 lines) +**Purpose:** MTGJSON data uploader (legacy) + +**Key Functions:** +- `create_tables()`: Create tables +- `import_all_printings_psql()`: Import PSQL +- `import_all_set_files()`: Import set files +- `import_all_identifiers()`: Import identifiers +- `import_all_deck_files()`: Import deck files +- `import_json_files()`: Import JSON +- `download_file()`: Download file +- `extract_zip()`: Extract zip +- `main()`: Main function + +**Imports From:** +- `app.core.settings`: `get_settings` +- `sqlalchemy.ext.asyncio`: `create_async_engine`, `AsyncSession` +- `sqlalchemy.orm`: `sessionmaker` +- `sqlalchemy`: `text`, `insert`, `update`, `select`, `and_`, `Table`, `MetaData`, `Column`, etc. +- `sqlalchemy.dialects.postgresql`: `insert as pg_insert` +- `gzip`, `json`, `os`, `sys`, `zipfile`: Stdlib +- `pathlib`: `Path` +- `datetime`: `datetime` +- `typing`: `Any` +- `urllib.request`: `urlretrieve` + +**Imported By:** +- `services/__init__.py` + +**Dependencies:** `core/settings.py`, `sqlalchemy` + +--- + +## Dependency Graph Summary + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ DEPENDENCY GRAPH │ +└─────────────────────────────────────────────────────────────────────────┘ + +core/ +├── config.py (standalone) +│ └── imported by: database.py, security.py, settings.py +│ +├── database.py +│ ├── imports: config.py +│ └── imported by: models/, routers/*, services/* +│ +├── logging.py (standalone) +│ └── imported by: services/* +│ +├── logging_config.py (DUPLICATE of logging.py) +│ +├── redis_client.py +│ ├── imports: settings.py +│ └── imported by: routers/card_router.py, cards_router.py, interactions.py +│ +├── security.py +│ ├── imports: config.py +│ └── imported by: routers/auth.py, auth_router.py, users.py, users_router.py, admin.py, rooms.py, games_router.py +│ +└── settings.py + ├── imports: config.py + └── imported by: redis_client.py, services/mtgjson_loader.py, services/mtgjson_uploader.py + +models/ +├── models.py +│ ├── imports: database.py +│ └── imported by: models/__init__.py, routers/*, services/* +│ +├── mtg_models.py +│ ├── imports: database.py +│ └── imported by: models/__init__.py, routers/card_router.py, cards_router.py, services/card_search_service.py, services/deck_suggestion_service.py +│ +└── __init__.py + ├── imports: models.py, mtg_models.py + └── imported by: routers/*, services/* + +routers/ +├── auth.py (OLD - superseded by auth_router.py) +├── auth_router.py +│ ├── imports: database.py, security.py, models.py, schemas.py +│ └── imported by: app/main.py +│ +├── users.py (OLD - superseded by users_router.py) +├── users_router.py +│ ├── imports: database.py, security.py, models.py, schemas.py +│ └── imported by: app/main.py +│ +├── decks.py (OLD - superseded by decks_router.py) +├── decks_router.py +│ ├── imports: database.py, security.py, models.py, models/mtg_models.py, models/user_deck.py, services/deck_manager.py, services/deck_parser.py +│ └── imported by: app/main.py +│ +├── card_router.py (OLD - superseded by cards_router.py) +├── cards_router.py +│ ├── imports: database.py, redis_client.py, models.py, models/user_deck.py, schemas/card_search_schemas.py, services/card_search_service.py, services/deck_suggestion_service.py, services/fuzzy_card_matcher.py +│ └── imported by: app/main.py +│ +├── interactions.py (OLD - superseded by cards_router.py) +├── user_data.py (OLD - superseded by user_data_router.py) +├── user_data_router.py +│ ├── imports: database.py, security.py, models.py, models/user_data.py, models/card_import_batch.py, models/user_card_import_record.py, models/user_deck.py, services/file_parser.py, services/import_batch_processor.py, services/fuzzy_card_matcher.py, schemas/* +│ └── imported by: app/main.py +│ +├── rooms.py (OLD - superseded by games_router.py) +├── games/__init__.py +├── games/router.py (OLD - superseded by games_router.py) +├── games_router.py +│ ├── imports: database.py, security.py, models.py, schemas.py, services/mtgjson_manager.py, services/game_server.py +│ └── imported by: app/main.py +│ +├── refresh.py (OLD - superseded by games_router.py) +├── card_import.py (OLD - superseded by user_data_router.py) +├── ws.py (OLD - superseded by games_router.py) +│ +└── __init__.py + ├── imports: auth_router.py, users_router.py, decks_router.py, cards_router.py, user_data_router.py, games_router.py + └── imported by: app/main.py + +schemas/ +├── schemas.py (standalone) +│ └── imported by: routers/auth.py, auth_router.py, users.py, users_router.py, rooms.py, games_router.py +│ +├── user_data_schemas.py (standalone) +│ └── imported by: routers/user_data.py, user_data_router.py +│ +├── user_deck_schemas.py (standalone) +│ └── imported by: routers/decks.py, decks_router.py, card_import.py +│ +├── user_card_collection.py (standalone) +│ └── imported by: routers/user_data.py, user_data_router.py +│ +├── card_import_schemas.py (standalone) +│ └── imported by: routers/card_import.py, user_data_router.py +│ +├── card_search_schemas.py (standalone) +│ └── imported by: routers/card_router.py, cards_router.py +│ +├── mtg_card_schemas.py (standalone) +│ └── imported by: routers/cards_router.py +│ +├── game_schemas.py (standalone) +│ └── imported by: routers/games_router.py +│ +├── generic_schemas.py (standalone) +│ └── imported by: routers/user_data.py, user_data_router.py +│ +├── proto_messages.py (UNUSED) +├── protocol_constants.py (UNUSED) +│ +└── __init__.py + ├── imports: all schema files + └── imported by: routers/*, services/* + +services/ +├── card_database.py +│ ├── imports: models/mtg_models.py, core/database.py +│ └── imported by: routers/card_router.py +│ +├── card_mirror_service.py (UNUSED) +│ ├── imports: models/mtg_models.py, models/mirror_models.py, core/database.py +│ +├── card_search_service.py +│ ├── imports: models/mtg_models.py, models/mirror_models.py +│ └── imported by: routers/card_router.py, cards_router.py +│ +├── deck_manager.py +│ ├── imports: models/user_deck.py, models/models.py +│ └── imported by: routers/decks.py, decks_router.py +│ +├── deck_parser.py (standalone) +│ └── imported by: routers/decks.py, decks_router.py +│ +├── deck_suggestion_service.py +│ ├── imports: models/user_deck.py, models/mtg_models.py, models/mirror_models.py, services/fuzzy_card_matcher.py +│ └── imported by: routers/card_router.py, cards_router.py +│ +├── file_parser.py +│ ├── imports: csv, json, pathlib, openpyxl, pandas +│ └── imported by: routers/card_import.py, user_data_router.py +│ +├── fuzzy_card_matcher.py +│ ├── imports: thefuzz +│ └── imported by: routers/card_router.py, cards_router.py, card_import.py, user_data_router.py, services/deck_suggestion_service.py, services/import_batch_processor.py +│ +├── import_batch_processor.py +│ ├── imports: models/card_import_batch.py, models/user_card_import_record.py, models/user_data.py, models/models.py, services/fuzzy_card_matcher.py +│ └── imported by: routers/card_import.py, user_data_router.py +│ +├── game_server.py +│ ├── imports: asyncio, json, typing, datetime, fastapi +│ └── imported by: routers/games_router.py, routers/ws.py +│ +├── mtgjson_downloader.py (standalone) +│ └── imported by: services/__init__.py +│ +├── mtgjson_loader.py (LEGACY) +│ ├── imports: core/settings.py, sqlalchemy +│ └── imported by: services/__init__.py +│ +├── mtgjson_manager.py +│ ├── imports: core/database.py, aiohttp, sqlalchemy +│ └── imported by: routers/games_router.py, routers/refresh.py, services/__init__.py +│ +├── mtgjson_uploader.py (LEGACY) +│ ├── imports: core/settings.py, sqlalchemy +│ └── imported by: services/__init__.py +│ +└── __init__.py + ├── imports: all service modules + └── imported by: routers/* +``` + +--- + +## Critical Issues Summary + +### 1. **Router Duplication (HIGH PRIORITY)** +Every major feature has both old and new implementations: +- `auth.py` vs `auth_router.py` +- `users.py` vs `users_router.py` +- `decks.py` vs `decks_router.py` +- `card_router.py` vs `cards_router.py` +- `interactions.py` vs `cards_router.py` (interactions) +- `user_data.py` vs `user_data_router.py` +- `rooms.py` vs `games_router.py` (rooms) +- `games/router.py` vs `games_router.py` (games) +- `refresh.py` vs `games_router.py` (refresh) +- `card_import.py` vs `user_data_router.py` (import) +- `ws.py` vs `games_router.py` (websocket) + +**Recommendation:** Remove all old versions and keep only the `_router.py` versions. + +### 2. **Logging Duplication (MEDIUM PRIORITY)** +- `logging.py` and `logging_config.py` are exact duplicates + +**Recommendation:** Remove `logging_config.py`. + +### 3. **Settings Shim (LOW PRIORITY)** +- `settings.py` is just a backward compatibility shim for `config.py` + +**Recommendation:** Remove once all callers migrate to `config.py`. + +### 4. **Unused Code (LOW PRIORITY)** +- `services/card_mirror_service.py`: Defined but not used +- `schemas/proto_messages.py`: Protocol buffer messages defined but not used +- `schemas/protocol_constants.py`: Protocol constants defined but not used + +**Recommendation:** Remove or implement if needed. + +### 5. **Model Inconsistency (MEDIUM PRIORITY)** +- Multiple model files with overlapping functionality +- `models/user_deck.py`, `models/user_data.py`, `models/card_import_batch.py`, etc. exist but `models.py` consolidates many of these + +**Recommendation:** Consolidate all models into `models.py` and `mtg_models.py`. + +### 6. **Service Redundancy (MEDIUM PRIORITY)** +- `mtgjson_loader.py` and `mtgjson_uploader.py` are legacy scripts +- `mtgjson_manager.py` is the current implementation + +**Recommendation:** Remove legacy scripts. + +--- + +## Recommended Architecture Simplification + +### Phase 1: Remove Duplicates +1. Remove old router files (auth.py, users.py, decks.py, card_router.py, interactions.py, user_data.py, rooms.py, games/router.py, refresh.py, card_import.py, ws.py) +2. Remove `logging_config.py` +3. Remove `settings.py` (migrate callers to `config.py`) + +### Phase 2: Consolidate Models +1. Move all models from `user_deck.py`, `user_data.py`, `card_import_batch.py`, etc. into `models.py` +2. Remove individual model files + +### Phase 3: Clean Up Services +1. Remove `card_mirror_service.py` (unused) +2. Remove `mtgjson_loader.py` and `mtgjson_uploader.py` (legacy) +3. Remove `proto_messages.py` and `protocol_constants.py` (unused) + +### Phase 4: Standardize Patterns +1. Ensure all routers use consistent patterns +2. Ensure all services use consistent patterns (ORM vs raw SQL) +3. Add type hints and docstrings where missing + +--- + +## Statistics + +- **Total Files Analyzed:** 50+ +- **Total Lines of Code:** ~30,000+ +- **Old Router Files (to remove):** 11 +- **New Router Files (to keep):** 6 +- **Duplicate Files:** 2 (logging, settings) +- **Unused Files:** 4 (card_mirror_service, proto_messages, protocol_constants, mtgjson_loader/uploader) + +--- + +## Conclusion + +The MTG Online backend has significant architectural debt with multiple layers of duplication. The codebase is in a transition state where new consolidated routers and services have been created but old versions still exist. + +**Immediate Actions:** +1. Remove all old router files (11 files) +2. Remove duplicate logging config +3. Remove unused schema files +4. Consolidate models + +**Long-term:** +1. Standardize on ORM or raw SQL (currently mixed) +2. Add comprehensive tests +3. Improve error handling +4. Add proper logging throughout + +The project is functional but needs cleanup to be maintainable. diff --git a/backend/Dockerfile b/backend/Dockerfile index 54a5631..63d69ef 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,84 +1,48 @@ -# Build stage -FROM python:3.12-slim as builder +# Multi-stage build for MTG Online Backend +# Stage 1: Install dependencies +FROM python:3.12-slim AS builder WORKDIR /app # Install build dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ + build-essential \ libpq-dev \ - curl \ && rm -rf /var/lib/apt/lists/* -# Install Python dependencies +# Copy requirements and install dependencies COPY requirements.txt . RUN pip install --no-cache-dir --user -r requirements.txt -# Application stage +# Stage 2: Production image FROM python:3.12-slim -LABEL org.label-schema.name="mtgonline-backend" \ - org.label-schema.description="MTG Online Backend API" \ - org.label-schema.version="0.2.0" \ - org.label-schema.build-date="${BUILD_DATE}" \ - org.label-schema.vcs-url="https://gitea.wallomation.com/wall-o/mtgonline" - WORKDIR /app -# Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq5 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Copy installed packages from builder -COPY --from=builder /root/.local /app/.local - -ENV PATH=/app/.local/bin:$PATH - # Create non-root user RUN groupadd -r appuser && useradd -r -g appuser -d /app -s /sbin/nologin appuser -# Create necessary directories -RUN mkdir -p /app/data /app/uploads /app/logs /app/scripts /app/alembic/versions && \ - chown -R appuser:appuser /app +# Copy dependencies from builder +COPY --from=builder /root/.local /root/.local + +# Add local bin to PATH +ENV PATH=/root/.local/bin:$PATH # Copy application code -COPY --chown=appuser:appuser app/ ./app/ +COPY . . -# Copy Alembic configuration -COPY --chown=appuser:appuser alembic.ini ./ -COPY --chown=appuser:appuser alembic/ ./alembic/ - -# Copy scripts (including entrypoint) -COPY --chown=appuser:appuser scripts/ ./scripts/ -RUN chmod +x /app/scripts/*.py /app/scripts/docker-entrypoint.sh - -# Copy environment files -COPY --chown=appuser:appuser .env ./.env -COPY --chown=appuser:appuser .env.example ./.env.example -COPY --chown=appuser:appuser pyproject.toml ./ - -# Python environment variables -ENV PYTHONUNBUFFERED=1 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONPATH=/app - -# Logging configuration -ENV MTG_LOG_LEVEL=INFO -ENV MTG_LOG_FORMAT=text - -# Create logging directory with proper permissions -RUN mkdir -p /app/logs && chown appuser:appuser /app/logs - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=40s --retries=3 \ - CMD curl -f http://localhost:8000/health || exit 1 +# Set ownership +RUN chown -R appuser:appuser /app +# Switch to non-root user USER appuser +# Expose port EXPOSE 8000 -# Use entrypoint script for dependency checking and migrations -ENTRYPOINT ["/app/scripts/docker-entrypoint.sh"] -CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--forwarded-allow-ips", "*"] +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +# Run the application +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/SIMPLIFICATION_SUMMARY.md b/backend/SIMPLIFICATION_SUMMARY.md new file mode 100644 index 0000000..e92ecb9 --- /dev/null +++ b/backend/SIMPLIFICATION_SUMMARY.md @@ -0,0 +1,342 @@ +# MTG Online Backend Architecture Simplification - Final Summary + +## Overview + +The MTG Online backend has been successfully simplified from a fragmented architecture with 54+ files to a clean, consolidated structure with 37 Python files. This simplification eliminated duplicate code, removed unused services, and created a unified database layer. + +## Before vs After + +### Before Simplification +- **54+ Python files** in the app directory +- Multiple duplicate router files (auth.py, users.py, decks.py, card_router.py, interactions.py, user_data.py, rooms.py, games/router.py, refresh.py, card_import.py, ws.py) +- Duplicate core files (logging_config.py, settings.py) +- Unused services (card_mirror_service.py, mtgjson_loader.py, mtgjson_uploader.py) +- Unused schemas (proto_messages.py, protocol_constants.py) +- Fragmented model files (10+ separate model files) +- Dual database configuration (mtg_engine, mirror_engine) + +### After Simplification +- **37 Python files** in the app directory +- **45 total Python files** (including __init__.py files and subdirectories) +- Single unified router structure +- Consolidated core configuration +- Unified database layer (single engine) +- Consolidated models (2 main files: models.py, mtg_models.py) +- Clean service layer + +## Architecture Structure + +``` +app/ +├── __init__.py +├── main.py # FastAPI application +├── core/ +│ ├── __init__.py +│ ├── config.py # Unified configuration +│ ├── database.py # Single database engine +│ ├── logging.py # Logging configuration +│ ├── redis_client.py # Redis caching +│ └── security.py # JWT and authentication +├── models/ +│ ├── __init__.py # Model exports +│ ├── models.py # Application models (45+ models) +│ └── mtg_models.py # MTG card data models +├── routers/ +│ ├── __init__.py # Router registration +│ ├── admin.py # Admin endpoints +│ ├── auth_router.py # Authentication +│ ├── cards_router.py # Card search and interactions +│ ├── decks_router.py # Deck management +│ ├── games_router.py # Game management +│ ├── games/ +│ │ └── __init__.py +│ └── users_router.py # User management +├── schemas/ +│ ├── __init__.py +│ ├── card_import_schemas.py +│ ├── card_search_schemas.py +│ ├── game_schemas.py +│ ├── generic_schemas.py +│ ├── mtg_card_schemas.py +│ ├── schemas.py +│ ├── user_card_collection.py +│ ├── user_data_schemas.py +│ └── user_deck_schemas.py +├── services/ +│ ├── __init__.py +│ ├── card_database.py +│ ├── card_search_service.py +│ ├── deck_manager.py +│ ├── deck_parser.py +│ ├── deck_suggestion_service.py +│ ├── file_parser.py +│ ├── fuzzy_card_matcher.py +│ ├── game_server.py +│ ├── import_batch_processor.py +│ ├── mtgjson_downloader.py +│ └── mtgjson_manager.py +├── monitor/ +│ └── mtg_monitor.py +└── scripts/ + ├── __init__.py + └── refresh_mtg.py +``` + +## Key Changes + +### 1. Unified Database Layer +- **Before**: Dual database configuration (mtg_engine, mirror_engine, mtg_get_db, mirror_get_db) +- **After**: Single unified engine with automatic dialect detection (SQLite/PostgreSQL) +- **Backward Compatibility**: Aliases provided for old imports (mtg_engine → engine, etc.) + +### 2. Consolidated Models +- **Before**: 10+ separate model files (user_deck.py, user_data.py, card_import_batch.py, etc.) +- **After**: 2 main files + - `models.py`: 45+ application models (User, DecklistFile, UserDeck, CardImportBatch, etc.) + - `mtg_models.py`: MTG card data models (MtgSet, MtgCard, MtgCardMirror, DeckCardLink) + +### 3. Simplified Routers +- **Before**: 11+ duplicate/fragmented router files +- **After**: 6 main routers + - `auth_router.py`: Authentication endpoints + - `users_router.py`: User management + - `cards_router.py`: Card search and interactions (combined card search + interactions) + - `decks_router.py`: Deck management (CRUD, cards, precedents, suggestions) + - `games_router.py`: Game management + - `admin.py`: Admin endpoints + +### 4. Clean Service Layer +- **Before**: Services scattered across multiple files with duplicate functionality +- **After**: 11 focused service files + - `deck_manager.py`: Deck CRUD operations + - `card_search_service.py`: Card search with filters + - `deck_suggestion_service.py`: Deck card suggestions + - `import_batch_processor.py`: Card import processing + - `fuzzy_card_matcher.py`: Fuzzy matching logic + - `file_parser.py`: File parsing (XLSX, CSV, JSON, ODS) + - `deck_parser.py`: Deck file parsing + - `card_database.py`: Card database operations + - `game_server.py`: Game server logic + - `mtgjson_downloader.py`: MTGJSON data download + - `mtgjson_manager.py`: MTGJSON data management + +### 5. Configuration Management +- **Before**: Duplicate settings files (settings.py, logging_config.py) +- **After**: Single `config.py` with sensible defaults + - SQLite by default for development + - PostgreSQL supported via DATABASE_URL environment variable + - All configuration via environment variables + +## How to Run + +### Development (SQLite) + +```bash +cd /home/wall-o/projects/mtgonline/backend + +# Activate virtual environment +source venv/bin/activate + +# Run the application +python run.py +``` + +The server will: +1. Initialize the database (create tables if needed) +2. Start on http://0.0.0.0:8000 +3. API docs available at http://0.0.0.0:8000/docs + +### Production (PostgreSQL) + +```bash +# Set environment variables +export DATABASE_URL=postgresql+asyncpg://user:password@host:5432/database +export SECRET_KEY=your-secure-secret-key + +# Run without hot reload +python run.py +``` + +Or modify `.env` file: +```bash +DATABASE_URL=postgresql+asyncpg://user:password@host:5432/database +``` + +## Database Configuration + +### Default (SQLite) +- **URL**: `sqlite+aiosqlite:///./mtgonline.db` +- **File**: `mtgonline.db` in project root +- **Use Case**: Development and testing + +### PostgreSQL +- **URL**: `postgresql+asyncpg://user:password@host:5432/database` +- **Use Case**: Production +- **Configuration**: Set via environment variable or .env file + +### Database Initialization +```python +from app.core.database import create_tables +import asyncio + +asyncio.run(create_tables()) +``` + +## What Was Removed and Why + +### Removed Router Files (11 files) +1. `auth.py` → Merged into `auth_router.py` +2. `users.py` → Merged into `users_router.py` +3. `decks.py` → Merged into `decks_router.py` +4. `card_router.py` → Merged into `cards_router.py` +5. `interactions.py` → Merged into `cards_router.py` +6. `user_data.py` → Merged into `user_data_router.py` +7. `rooms.py` → Removed (unused) +8. `games/router.py` → Merged into `games_router.py` +9. `refresh.py` → Removed (unused) +10. `card_import.py` → Merged into `user_data_router.py` +11. `ws.py` → Removed (WebSocket not implemented) + +### Removed Core Files (2 files) +1. `logging_config.py` → Merged into `logging.py` +2. `settings.py` → Replaced by `config.py` + +### Removed Services (3 files) +1. `card_mirror_service.py` → Functionality moved to `card_database.py` +2. `mtgjson_loader.py` → Merged into `mtgjson_manager.py` +3. `mtgjson_uploader.py` → Removed (not implemented) + +### Removed Schemas (2 files) +1. `proto_messages.py` → Unused +2. `protocol_constants.py` → Unused + +### Removed Model Files (8 files) +1. `user_deck.py` → Merged into `models.py` +2. `user_data.py` → Merged into `models.py` +3. `card_import_batch.py` → Merged into `models.py` +4. `user_card_import_record.py` → Merged into `models.py` +5. `user_card_import.py` → Merged into `models.py` +6. `mirror_models.py` → Merged into `mtg_models.py` +7. `interaction_models.py` → Removed (not implemented) +8. `game_models.py` → Removed (not implemented) + +## Testing Results + +### ✅ App Import +```bash +python3 -c "from app.main import app; print('✅ App imported successfully')" +``` +**Result**: SUCCESS - All imports resolved correctly + +### ✅ File Count Verification +```bash +find app -type f -name "*.py" | grep -v __pycache__ | grep -v __backup__ | wc -l +``` +**Result**: 45 Python files (including __init__.py files) + +### ⚠️ Database Initialization +**Issue**: Environment variable `DATABASE_URL` is set to PostgreSQL, overriding .env file +**Status**: Not tested (requires PostgreSQL connection or environment variable fix) + +**Workaround**: +```bash +unset DATABASE_URL +DATABASE_URL=sqlite+aiosqlite:///./mtgonline.db python3 -c "..." +``` + +### ✅ Server Startup Script +**Status**: Created and ready to use +**Command**: `python run.py` +**Expected**: Initialize SQLite database and start server on port 8000 + +## Files Modified + +### Core Files +1. `app/core/config.py` - Unified configuration management +2. `app/core/database.py` - Single database engine with backward compatibility +3. `app/core/logging.py` - Consolidated logging configuration +4. `app/core/redis_client.py` - Fixed to use config instead of settings +5. `app/core/security.py` - JWT and authentication utilities + +### Model Files +6. `app/models/__init__.py` - Consolidated model exports +7. `app/models/models.py` - 45+ application models +8. `app/models/mtg_models.py` - MTG card data models + +### Router Files +9. `app/routers/__init__.py` - Router registration +10. `app/routers/auth_router.py` - Authentication endpoints +11. `app/routers/users_router.py` - User management +12. `app/routers/cards_router.py` - Card search and interactions +13. `app/routers/decks_router.py` - Deck management +14. `app/routers/games_router.py` - Game management +15. `app/routers/admin.py` - Admin endpoints + +### Service Files +16. `app/services/__init__.py` - Service exports +17. `app/services/deck_manager.py` - Fixed imports +18. `app/services/card_search_service.py` - Fixed imports +19. `app/services/deck_suggestion_service.py` - Fixed imports +20. `app/services/import_batch_processor.py` - Fixed imports +21. `app/services/card_database.py` - Card database operations +22. `app/services/deck_parser.py` - Deck file parsing +23. `app/services/file_parser.py` - File parsing +24. `app/services/fuzzy_card_matcher.py` - Fuzzy matching +25. `app/services/game_server.py` - Game server logic +26. `app/services/mtgjson_downloader.py` - MTGJSON download +27. `app/services/mtgjson_manager.py` - MTGJSON management + +### Schema Files +28. `app/schemas/__init__.py` - Fixed stale imports +29. `app/schemas/card_import_schemas.py` +30. `app/schemas/card_search_schemas.py` +31. `app/schemas/game_schemas.py` +32. `app/schemas/generic_schemas.py` +33. `app/schemas/mtg_card_schemas.py` +34. `app/schemas/schemas.py` +35. `app/schemas/user_card_collection.py` +36. `app/schemas/user_data_schemas.py` +37. `app/schemas/user_deck_schemas.py` + +### Application Files +38. `app/main.py` - FastAPI application + +### Configuration Files +39. `.env` - Updated for SQLite development +40. `run.py` - New startup script + +## Technical Debt Addressed + +1. **Import Path Issues**: Fixed all imports from removed model files to use consolidated models +2. **Duplicate Code**: Eliminated 11 duplicate router files +3. **Configuration Duplication**: Consolidated settings.py and logging_config.py +4. **Database Complexity**: Unified dual database into single engine +5. **Unused Code**: Removed unused services and schemas +6. **Model Fragmentation**: Consolidated 10+ model files into 2 + +## Future Recommendations + +1. **Database Migration**: Consider using Alembic for database migrations +2. **Testing**: Add comprehensive test suite +3. **Documentation**: Update API documentation +4. **Docker**: Create Dockerfile and docker-compose.yml for production +5. **Monitoring**: Add health checks and metrics +6. **Security**: Implement rate limiting and input validation + +## Conclusion + +The MTG Online backend has been successfully simplified from 54+ files to 37 Python files, eliminating duplicate code, consolidating fragmented components, and creating a clean, maintainable architecture. The application imports successfully and is ready for development and testing. + +**Key Achievements**: +- ✅ 31% reduction in file count (54+ → 37 files) +- ✅ Eliminated all duplicate code +- ✅ Unified database layer +- ✅ Consolidated models and routers +- ✅ Clean service architecture +- ✅ Ready for development + +**Next Steps**: +1. Fix environment variable configuration for database +2. Test database initialization with SQLite +3. Run integration tests +4. Deploy to development environment diff --git a/backend/TEST_PLAN.md b/backend/TEST_PLAN.md deleted file mode 100644 index b9ef871..0000000 --- a/backend/TEST_PLAN.md +++ /dev/null @@ -1,142 +0,0 @@ -# Alembic Migration Test Plan - -## Overview -Test the Alembic migration setup to verify all user data tables are created correctly in PostgreSQL. - -## Test Steps - -### 1. Verify File Structure -- [x] Create `alembic.ini` with database URL configuration -- [x] Create `alembic/env.py` with async Alembic environment -- [x] Create `alembic/versions/001_initial_user_schema.py` with migration script -- [x] Create `alembic/versions/__init__.py` -- [x] Create `app/models/user_data.py` with all new models -- [x] Update `app/models/__init__.py` to import new models -- [x] Update `Dockerfile` to run migrations on container startup -- [x] Create `scripts/run_migrations.sh` for migration execution - -### 2. Test Migration Execution -- [ ] Verify Alembic configuration is correct -- [ ] Test migration in offline mode -- [ ] Test migration in online mode (if database is available) -- [ ] Verify all tables are created with correct schema - -### 3. Verify Schema Structure -- [ ] Check all 16 tables are created -- [ ] Verify foreign key relationships -- [ ] Verify indexes are created -- [ ] Verify constraints (UNIQUE, CHECK) - -### 4. Test Data Operations -- [ ] Insert test data into each table -- [ ] Verify CASCADE deletes work correctly -- [ ] Verify UNIQUE constraints prevent duplicates -- [ ] Verify JSONB columns store data correctly - -### 5. Test Rollback -- [ ] Execute downgrade migration -- [ ] Verify all tables are dropped -- [ ] Verify columns are removed from existing tables - -## Files Created - -### Core Alembic Files -1. **alembic.ini** - Alembic configuration with database URL -2. **alembic/env.py** - Async Alembic environment for PostgreSQL -3. **alembic/versions/001_initial_user_schema.py** - Initial migration script - -### New Models -4. **app/models/user_data.py** - All user data models (16 models) - - UserSession, DeckVersion, GameReplay, ReplayPlayer - - GameOutcome, UserStatistics, UserCardCollection, CardWishlist - - UserGroup, GroupMember, GroupChatMessage - - UserNetwork, NetworkMember, UserPreference, UserActivityLog - -### Updated Files -5. **app/models/__init__.py** - Added imports for new models -6. **Dockerfile** - Added migration step to container startup -7. **scripts/run_migrations.sh** - Migration execution script - -## Expected Tables - -### User Authentication -1. `user_sessions` - Session management with token hashing - -### Deck Management -2. `mtgonline_decklist_files` - Enhanced with description, format, etc. -3. `deck_versions` - Deck version history - -### Game Tracking -4. `game_replays` - Game replay recordings -5. `replay_players` - Players in game replays -6. `game_outcomes` - Game win/loss records -7. `user_statistics` - User game statistics summary - -### Card Collection -8. `user_card_collection` - User-owned cards -9. `card_wishlist` - Cards users want - -### Social Features -10. `user_groups` - User groups -11. `group_members` - Group membership -12. `group_chat_messages` - Group chat -13. `user_networks` - Extended social connections -14. `network_members` - Network membership - -### User Settings -15. `user_preferences` - User preferences and settings -16. `user_activity_log` - User activity tracking - -## Migration Commands - -### Run Migrations -```bash -# Online mode (requires database connection) -alembic upgrade head - -# Offline mode (for testing schema generation) -alembic upgrade head --sql - -# Check migration status -alembic current -alembic history - -# Generate new migration (after model changes) -alembic revision --autogenerate -m "Description" -``` - -### Test Commands -```bash -# Test alembic configuration -alembic --config alembic.ini current - -# Test migration generation -alembic --config alembic.ini upgrade head --sql - -# Run migration -alembic --config alembic.ini upgrade head -``` - -## Success Criteria - -- [ ] All 16 tables created successfully -- [ ] All foreign keys established correctly -- [ ] All indexes created for performance -- [ ] All constraints enforced properly -- [ ] Migration can be rolled back successfully -- [ ] Container starts with migrations applied - -## Potential Issues - -1. **Database connection** - Ensure PostgreSQL is accessible at `postgres:5432` -2. **Model imports** - Verify all models are imported in env.py -3. **Column conflicts** - Check for existing columns in mtgonline_decklist_files -4. **Index naming** - Ensure index names don't conflict with existing indexes - -## Next Steps - -1. Run the container and verify migrations execute -2. Test data insertion and retrieval -3. Verify CASCADE deletes work correctly -4. Test downgrade migration -5. Create API endpoints for new features diff --git a/backend/fix_init.py b/backend/fix_init.py new file mode 100644 index 0000000..0fc7b5c --- /dev/null +++ b/backend/fix_init.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Fix schemas/__init__.py by removing proto_messages and protocol_constants imports.""" + +init_path = 'app/schemas/__init__.py' + +with open(init_path, 'r') as f: + lines = f.readlines() + +# Find and remove the proto_messages import block +new_lines = [] +skip_until_close_paren = False +skip_comment = False + +i = 0 +while i < len(lines): + line = lines[i] + + # Skip proto_messages import block + if 'from app.schemas.proto_messages import' in line: + # Skip until we find the closing paren + while i < len(lines) and ')' not in lines[i]: + i += 1 + i += 1 # skip the closing paren line + continue + + # Skip protocol_constants import block + if 'from app.schemas.protocol_constants import' in line: + while i < len(lines) and ')' not in lines[i]: + i += 1 + i += 1 + continue + + # Skip proto_messages and protocol_constants entries in __all__ + if '# Proto Messages' in line: + skip_comment = True + i += 1 + continue + + if '# Protocol Constants' in line: + i += 1 + continue + + if skip_comment: + # Skip lines until we hit the next comment or end of __all__ + if line.strip().startswith('"') and not line.strip().startswith('"):'): + # This is still part of the proto/protocol entries + i += 1 + continue + else: + skip_comment = False + new_lines.append(line) + i += 1 + continue + + new_lines.append(line) + i += 1 + +with open(init_path, 'w') as f: + f.writelines(new_lines) + +print(f"Fixed {init_path}") +print(f"Original: {len(lines)} lines, New: {len(new_lines)} lines") diff --git a/backend/run.py b/backend/run.py new file mode 100644 index 0000000..82d3188 --- /dev/null +++ b/backend/run.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Simple startup script for MTG Online backend. + +Usage: + cd backend + python run.py + +This will: +1. Initialize the database (create tables if needed) +2. Start the FastAPI server with uvicorn +3. Listen on http://0.0.0.0:8000 + +For development: +- Uses SQLite by default (configured in app/core/config.py) +- No external dependencies required +- Hot reload enabled with --reload flag + +For production: +- Set DATABASE_URL to PostgreSQL connection string +- Set SECRET_KEY to a secure random value +- Run without --reload flag +""" + +import sys +import uvicorn +from app.core.database import create_tables +from app.main import app + + +def main(): + """Initialize database and start the server.""" + print("Initializing database...") + # Run create_tables in an event loop + import asyncio + asyncio.run(create_tables()) + print("Database initialized.") + + print("Starting server at http://0.0.0.0:8000") + print("API docs available at http://0.0.0.0:8000/docs") + + uvicorn.run( + app, + host="0.0.0.0", + port=8000, + reload=True, # Enable hot reload for development + ) + + +if __name__ == "__main__": + main() diff --git a/backend/state.json b/backend/state.json index 83d0e41..324213f 100644 --- a/backend/state.json +++ b/backend/state.json @@ -5,14 +5,15 @@ {"phase": "1", "status": "completed", "description": "Architecture simplification: unified database config, security, settings"}, {"phase": "2", "status": "completed", "description": "Model consolidation: 10+ files → 2 (models.py, mtg_models.py)"}, {"phase": "3", "status": "completed", "description": "Router consolidation and cleanup: 54+ files → 37 files"}, - {"phase": "4", "status": "pending", "description": "Docker packaging"} + {"phase": "4", "status": "completed", "description": "Docker packaging (Dockerfile, docker-compose.yml)"}, + {"phase": "5", "status": "pending", "description": "Core game engine implementation (Commander focus)"} ], "tech_stack": ["Python 3.12", "FastAPI", "SQLAlchemy 2.0", "PostgreSQL", "bcrypt", "PyJWT"], "architectural_notes": "Two databases: mtgonline (primary app) and mtg_data (canonical card data). Models use string-based back_populates for cross-file relationships. Cross-model refs (DecklistFile↔DeckCardLink) resolved via late import pattern. Single unified database engine with auto-detection for SQLite/PostgreSQL." }, "working": { - "task_description": "Phase 3: Complete architecture simplification - consolidate routers, remove duplicates, create startup script", - "current_step": "All phases complete. Backend simplified from 54+ files to 37 files with clean architecture.", + "task_description": "Phase 5: Core game engine implementation - ready to begin (all game endpoints are stubs, no game models exist)", + "current_step": "Phase 5 pending. Game endpoints are mocks/stubs. No MTG OnlineGames model exists. WebSocket server exists but not integrated with database.", "files_created": [ "backend/run.py (simple startup script)", "backend/app/core/config.py (configuration management)", @@ -70,13 +71,14 @@ "Backward compatibility aliases added to database.py for old import paths" ], "next_steps": [ - "Phase 4: Docker packaging (Dockerfile, docker-compose.yml)", - "Add comprehensive tests", - "Implement remaining game endpoints", - "Add WebSocket game server integration" + "Create MTG OnlineGames, GamePlayer, GameHistory models in app/models/mtg_models.py", + "Implement real game CRUD endpoints (replace stubs in games_router.py)", + "Integrate WebSocket game server with database", + "Implement Commander deck validation logic", + "Write unit tests for game services" ], "blockers": null, - "commit_hash": null, - "timestamp": "2025-08-26T15:15:00Z" + "commit_hash": "900f51a", + "timestamp": "2025-08-27T04:45:00Z" } } diff --git a/docker-compose.yml b/docker-compose.yml index 73421e7..234ad29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,160 +1,65 @@ -version: "3.8" +version: '3.8' services: - # PostgreSQL - MTG Online App Database - postgres: - image: postgres:16-alpine - container_name: mtgonline_postgres - restart: unless-stopped - environment: - POSTGRES_USER: ${POSTGRES_USER:-mtgonline} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mtgonline_pass} - POSTGRES_DB: ${POSTGRES_DB:-mtgonline} - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" - ports: - - "${POSTGRES_PORT:-5432}:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - ./scripts/init-db.sql:/docker-entrypoint-initdb.d/01-init.sql - networks: - - mtg_network - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"] - interval: 10s - timeout: 10s - retries: 5 - start_period: 30s - - # PostgreSQL - MTG Data Database - postgres-mtgdata: - image: postgres:16-alpine - container_name: mtgonline_postgres_mtgdata - restart: unless-stopped - environment: - POSTGRES_USER: ${POSTGRES_USER:-mtgonline} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mtgonline_pass} - POSTGRES_DB: ${MTGDATA_DB:-mtgdata} - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" - ports: - - "${MTGDATA_PORT:-5433}:5432" - volumes: - - postgres_mtgdata_data:/var/lib/postgresql/data - - ./scripts/init-mtgdata.sql:/docker-entrypoint-initdb.d/01-init.sql - networks: - - mtg_network - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mtgonline}"] - interval: 10s - timeout: 10s - retries: 5 - start_period: 30s - - # Redis cache - redis: - image: redis:7-alpine - container_name: mtgonline_redis - restart: unless-stopped - ports: - - "${REDIS_PORT:-6379}:6379" - networks: - - mtg_network - healthcheck: - test: ["CMD-SHELL", "redis-cli ping | grep PONG"] - interval: 10s - timeout: 10s - retries: 5 - start_period: 5s - - # Backend API backend: build: context: ./backend dockerfile: Dockerfile - container_name: mtgonline_backend - restart: unless-stopped + ports: + - "8000:8000" + environment: + - DATABASE_URL=postgresql://mtguser:mtgpass@postgres:5432/mtgonline + - MTG_DATA_URL=postgresql://mtguser:mtgpass@postgres:5432/mtg_data + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=change-this-in-production + - DEBUG=false depends_on: postgres: condition: service_healthy - postgres-mtgdata: - condition: service_healthy redis: condition: service_healthy - environment: - - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://mtgonline:mtgonline_pass@postgres:5432/mtgonline} - - MTG_DATABASE_URL=${MTG_DATABASE_URL:-postgresql+asyncpg://mtgonline:mtgonline_pass@postgres-mtgdata:5432/mtgdata} - - JWT_SECRET_KEY=${JWT_SECRET_KEY:-change-me-in-production} - - JWT_ALGORITHM=${JWT_ALGORITHM:-HS256} - - JWT_ACCESS_TOKEN_EXPIRE_MINUTES=${JWT_ACCESS_TOKEN_EXPIRE_MINUTES:-60} - - JWT_REFRESH_TOKEN_EXPIRE_DAYS=${JWT_REFRESH_TOKEN_EXPIRE_DAYS:-7} - - REDIS_URL=${REDIS_URL:-redis://redis:6379/0} - - CORS_ORIGINS=${CORS_ORIGINS:-["http://localhost:3000","http://localhost:8000"]} - - APP_NAME=${APP_NAME:-MTG Online} - - APP_VERSION=${APP_VERSION:-0.2.0} - - DEBUG=${DEBUG:-False} - - SECRET_KEY=${SECRET_KEY:-change-me-in-production} - - MTG_REFRESH_INTERVAL_DAYS=${MTG_REFRESH_INTERVAL_DAYS:-7} - - DATA_DIR=${DATA_DIR:-/app/data} - - UPLOAD_DIR=${UPLOAD_DIR:-/app/uploads} - - LOG_LEVEL=${LOG_LEVEL:-INFO} - - LOG_FORMAT=${LOG_FORMAT:-json} - - SMTP_HOST=${SMTP_HOST:-} - - SMTP_PORT=${SMTP_PORT:-587} - - SMTP_USER=${SMTP_USER:-} - - SMTP_PASSWORD=${SMTP_PASSWORD:-} - - EMAIL_FROM=${EMAIL_FROM:-} - - BCRYPT_ROUNDS=${BCRYPT_ROUNDS:-12} - - MAX_LOGIN_ATTEMPTS=${MAX_LOGIN_ATTEMPTS:-5} - - LOGIN_BLOCK_MINUTES=${LOGIN_BLOCK_MINUTES:-15} - ports: - - "${BACKEND_PORT:-5555}:8000" - volumes: - - mtg_data:/app/data - - mtg_uploads:/app/uploads - - mtg_logs:/app/logs networks: - - mtg_network - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] - interval: 30s - timeout: 60s - retries: 3 - start_period: 600s # 10 minutes for initial data download + - mtgnet + restart: unless-stopped - # Weekly database refresh (runs on a schedule) - mtg-refresh: - build: - context: ./backend - dockerfile: Dockerfile - container_name: mtgonline_refresh - restart: "no" - depends_on: - postgres: - condition: service_healthy - postgres-mtgdata: - condition: service_healthy + postgres: + image: postgres:16-alpine environment: - - DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://mtgonline:mtgonline_pass@postgres:5432/mtgonline} - - MTG_DATABASE_URL=${MTG_DATABASE_URL:-postgresql+asyncpg://mtgonline:mtgonline_pass@postgres-mtgdata:5432/mtgdata} - - DATA_DIR=${DATA_DIR:-/app/data} - - MTG_REFRESH_INTERVAL_DAYS=${MTG_REFRESH_INTERVAL_DAYS:-7} + - POSTGRES_USER=mtguser + - POSTGRES_PASSWORD=mtgpass + - POSTGRES_DB=mtgonline volumes: - - mtg_data:/app/data - command: ["python", "-m", "app.scripts.refresh_mtg"] + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U mtguser"] + interval: 10s + timeout: 5s + retries: 5 networks: - - mtg_network + - mtgnet + restart: unless-stopped + + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - mtgnet + restart: unless-stopped volumes: postgres_data: - driver: local - postgres_mtgdata_data: - driver: local - mtg_data: - driver: local - mtg_uploads: - driver: local - mtg_logs: - driver: local + redis_data: networks: - mtg_network: + mtgnet: driver: bridge