diff --git a/C++/ARCHITECTURE_ANALYSIS.md b/C++/ARCHITECTURE_ANALYSIS.md new file mode 100644 index 0000000..b85f930 --- /dev/null +++ b/C++/ARCHITECTURE_ANALYSIS.md @@ -0,0 +1,488 @@ +# Cockatrice Architecture Analysis +## Target: /home/wall-o/projects/mtgonline/C++ +## Date: 2026-07-20 + +--- + +## 1. Project Overview + +**Cockatrice** (v3.1.0 "Graduation Day") is a mature open-source MTG online client/server +application written in C++20 using Qt 5/6. It has been maintained for many years and +represents a battle-tested implementation of an online card game. + +### Key Architectural Decisions: +- **Server-Client separation**: The server (`servatrice`) is authoritative; the client + (`cockatrice`) is a thin visualizer that sends commands through the server. +- **Protocol Buffers**: All network communication uses `.proto` files — no custom + serialization. This is the canonical contract between components. +- **Event-driven game loop**: Qt signals/slots drive game state transitions. +- **Plugin-like library structure**: Core logic lives in `libcockatrice_*` shared libraries. + +--- + +## 2. Top-Level Directory Structure + +``` +Cockatrice/ +├── CMakeLists.txt # Root build system (C++20, Qt5/6, Protobuf) +├── servatrice/ # SERVER application +│ ├── src/ +│ │ ├── servatrice.cpp/h # Main server class +│ │ ├── serversocketinterface.cpp/h # Per-client socket handler (~106KB — single massive file) +│ │ ├── servatrice_database_interface.cpp/h # Database operations +│ │ ├── isl_interface.cpp/h # Inter-server-link (cluster) support +│ │ ├── smtp/ # Email support +│ │ └── main.cpp/h +│ ├── migrations/ # DB schema migrations +│ └── resources/ # Config, SQL init +├── cockatrice/ # CLIENT application +│ ├── src/ +│ │ ├── client/ # Client network layer +│ │ │ ├── network/ +│ │ │ │ ├── connection_controller/ # Connection lifecycle +│ │ │ │ ├── interfaces/ # Client-server interface abstractions +│ │ │ │ ├── parsers/ # Protocol message parsers +│ │ │ │ └── update/ # Auto-update checks +│ │ │ ├── sound_engine.cpp/h # Audio feedback +│ │ │ └── settings/ +│ │ ├── game/ # GAME ENGINE +│ │ │ ├── abstract_game.h/cpp # Game abstraction +│ │ │ ├── game.cpp/h # Concrete game instance +│ │ │ ├── game_event_handler.cpp/h # Central event dispatch +│ │ │ ├── game_meta_info.h/cpp # Game metadata (wraps protobuf) +│ │ │ ├── game_state.h/cpp # Board state tracking +│ │ │ ├── phase.h/cpp # Turn phase definitions +│ │ │ ├── player/ # Player logic +│ │ │ │ ├── player_actions.cpp/h # Player commands +│ │ │ │ ├── player_event_handler.cpp/h +│ │ │ │ ├── player_logic.cpp/h # Player AI/logic +│ │ │ │ ├── player_manager.cpp/h # Multiplayer coordination +│ │ │ │ └── event_processing_options.h +│ │ │ ├── board/ # Board visualization state +│ │ │ │ ├── card_list.cpp/h +│ │ │ │ ├── card_state.cpp/h +│ │ │ │ └── counter_state.cpp/h +│ │ │ ├── zones/ # Zone logic +│ │ │ │ ├── card_zone_algorithms.h +│ │ │ │ ├── card_zone_logic.cpp/h +│ │ │ │ ├── hand_zone_logic.cpp/h +│ │ │ │ ├── pile_zone_logic.cpp/h +│ │ │ │ ├── stack_zone_logic.cpp/h +│ │ │ │ ├── table_zone_logic.cpp/h +│ │ │ │ └── view_zone_logic.cpp/h +│ │ │ ├── replay.cpp/h # Game replay system +│ │ │ └── arrow_registry.cpp/h # Card targeting arrows +│ │ ├── game_graphics/ # Visual rendering layer +│ │ │ ├── game_scene.cpp/h # QGraphicsScene for the board +│ │ │ ├── game_view.cpp/h # View controller +│ │ │ ├── board/ +│ │ │ ├── deckview/ +│ │ │ ├── dialogs/ +│ │ │ ├── phases_toolbar.cpp/h # Phase navigation UI +│ │ │ ├── player/ +│ │ │ ├── tally/ # Life total displays +│ │ │ ├── z_value_layer_manager.h +│ │ │ └── z_values.h +│ │ ├── database/ +│ │ │ └── interface/ +│ │ ├── filters/ # Deck/card filter system +│ │ │ ├── deck_filter_string.cpp/h +│ │ │ ├── filter_builder.cpp/h +│ │ │ └── filter_tree_model.cpp/h +│ │ ├── interface/ # UI/UX layer +│ │ │ ├── window_main.cpp/h # Main application window +│ │ │ ├── layouts/ +│ │ │ ├── widgets/ +│ │ │ ├── theme_manager.cpp/h # Theming system +│ │ │ ├── card_picture_loader/ +│ │ │ ├── deck_loader/ +│ │ │ ├── pixel_map_generator.cpp/h +│ │ │ ├── logger.cpp/h +│ │ │ └── palette_editor/ +│ │ └── main.cpp/h # Client entry point +├── oracle/ # CARD DATABASE TOOL +│ └── (downloads card data from MTGJSON) +├── libcockatrice_card/ # Card data model library +│ └── libcockatrice/card/ +│ ├── card_info.cpp/h # Card information model +│ ├── database/ # Card database access +│ ├── format/ # Deck format support +│ ├── import/ # Deck importers +│ ├── printing/ # Printing history +│ ├── relation/ # Card relationships +│ └── set/ # Set data +├── libcockatrice_deck_list/ # Deck list management library +│ └── libcockatrice/deck_list/ +│ ├── deck_list.cpp/h # Core deck list class +│ ├── deck_list_node_tree.cpp/h # Tree structure for decks +│ ├── sideboard_plan.cpp/h # Sideboarding +│ └── tree/ +├── libcockatrice_network/ # Network protocol library +│ └── libcockatrice/network/ +│ └── network/ # TCP client/server +├── libcockatrice_protocol/ # Protocol definition library +│ └── libcockatrice/protocol/ # .proto file translations to C++ +├── libcockatrice_rng/ # RNG library (SFMT) +├── libcockatrice_settings/ # Settings/preferences +├── libcockatrice_utility/ # Utility functions +├── docker-compose.yml # Dev environment (MySQL + Servatrice) +└── doc/ # Documentation (Doxygen) +``` + +--- + +## 3. Gameplay Engine Architecture + +### 3.1 Core Game Loop — Event-Driven Architecture + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ GAME ENGINE │ +│ │ +│ ┌──────────────┐ ┌──────────────────┐ ┌───────────────────┐ │ +│ │ Client │ │ Game │ │ Event Handler │ │ +│ │ (User Input)│──▶│ (AbstractGame) │◀──│ (Central Dispatch)│ │ +│ └──────────────┘ └──────────────────┘ └───────────────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────┐ ┌──────────────────┐ │ +│ │ GameState │ │ PlayerManager │ │ +│ │ (Board State)│ │ (Multiplayer │ │ +│ │ │ │ Coordination) │ │ +│ └─────────────┘ └──────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────┐ ┌──────────────────┐ │ +│ │ CardZone │ │ PlayerLogic │ │ +│ │ (Hand/ │ │ (Actions, Rules │ │ +│ │ Stack/ │ │ Processing) │ │ +│ │ Table/ │ └──────────────────┘ │ +│ │ Graveyard) │ │ +│ └─────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### 3.2 Game Class Hierarchy + +| Class | Role | Key Data | +|-------|------|----------| +| `AbstractGame` | Base game abstraction | `gameMetaInfo`, `gameState`, `gameEventHandler`, `playerManager` | +| `Game` | Concrete online/offline game | Extends AbstractGame with client list | +| `GameMetaInfo` | Wraps protobuf `ServerInfo_Game` | gameId, maxPlayers, description, started, spectators settings | +| `GameState` | Board state tracking | currentPhase, activePlayer, hostId, gameTimer, clients | +| `GameEventHandler` | **Central event dispatch** (~21KB handler) | Processes all game events, prepares commands | +| `Phase` | Turn phase definitions | 11 phases with sub-phases, colors, sounds | + +### 3.3 Phase System (MTG Turn Structure) + +``` +Phases::phases[] array contains 11 phases: +1. Untap +2. Upkeep +3. Draw +4. Main 1 +5. Combat (with sub-phases) + - Beginning of Combat + - Declare Attackers + - Declare Blockers + - Combat Damage + - End of Combat +6. Main 2 +7. End of Turn +8. Cleanup +(plus unknownPhase as sentinel) +``` + +Sub-phases are tracked via `Phases::subPhasesEnd` — the combat phase has 5 sub-phases. + +### 3.4 Player System + +| Class | Role | File Size | +|-------|------|-----------| +| `PlayerManager` | Coordinates all players in a game | ~2.5KB header | +| `PlayerLogic` | Per-player game logic (actions, rules) | ~10.7KB | +| `PlayerActions` | Concrete player commands (play, tap, draw, etc.) | **~64KB** — single massive file | +| `PlayerEventHandler` | Per-player event processing | ~24KB | +| `EventProcessingOptions` | Flags controlling event processing | ~575B | + +### 3.5 Zone System + +| Zone | File | Description | +|------|------|-------------| +| `CardZone` | `card_zone_logic.cpp/h` | Base class for all zones | +| `HandZone` | `hand_zone_logic.cpp/h` | Player's hand (secret zone) | +| `StackZone` | `stack_zone_logic.cpp/h` | The stack ( spells, abilities) | +| `TableZone` | `table_zone_logic.cpp/h` | Battlefield (permanent zone) | +| `PileZone` | `pile_zone_logic.cpp/h` | Graveyard, exile, library | +| `ViewZone` | `view_zone_logic.cpp/h` | Shared view zones (for effects) | +| `CardZoneAlgorithms` | `card_zone_algorithms.h` | Search, shuffle, sort algorithms | + +Each zone type implements its own `CardZoneLogic` subclass. + +### 3.6 Command/Event Architecture + +``` +User Action (GUI) + │ + ▼ +GameCommand (protobuf message) + │ + ▼ +GameEventHandler.processGameEventContainer() + │ + ▼ +PlayerLogic.handleCommand() + │ + ▼ +CardZoneLogic (state change) + │ + ▼ +GameEvent (sent to all clients) + │ + ▼ +Client receives & displays +``` + +### 3.7 Replay System + +`replay.cpp/h` — Games are recorded as a stream of protobuf events and can be +replayed identically. This is a direct serialization of `GameReplay` protobuf messages. + +### 3.8 Card State on Board + +| Class | Role | +|-------|------| +| `CardState` | Card face-up/face-down, tapped, counters | +| `CardList` | Ordered/unordered collection of cards in a zone | +| `CounterState` | Life total, poison counters, etc. | +| `ArrowData` | Targeting arrow visual | +| `ArrowRegistry` | Manages targeting arrows | + +--- + +## 4. Network Protocol Architecture + +### 4.1 Communication Stack + +``` +┌─────────────────────────────────────────────────────────┐ +│ Application Layer │ +│ - GameCommands (play, draw, tap, attack, etc.) │ +│ - GameEvents (state changes broadcast to all clients) │ +│ - Chat messages, user management │ +│ - Deck management │ +│ - Room management │ +│ - Admin commands │ +├─────────────────────────────────────────────────────────┤ +│ Protocol Layer │ +│ - Protocol Buffers (.proto files) │ +│ - Message types: ServerInfo_Game, ServerInfo_Player, │ +│ Event_Join, Command_PlayCard, etc. │ +├─────────────────────────────────────────────────────────┤ +│ Network Layer │ +│ - TCP connections │ +│ - Connection pooling │ +│ - ISL (Inter-Server Link) for clustering │ +└─────────────────────────────────────────────────────────┘ +``` + +### 4.2 Key Protocol Messages (from .pb files) + +| Category | Example Messages | +|----------|-----------------| +| **Game State** | `ServerInfo_Game`, `ServerInfo_Player`, `Event_GameStateChanged` | +| **Player Actions** | `Command_PlayCard`, `Command_Attack`, `Command_Damage`, `Command_DrawCard` | +| **Events** | `Event_Join`, `Event_Leave`, `Event_SetActivePlayer`, `Event_SetActivePhase` | +| **User Management** | `ServerInfo_User`, `Command_Acknowledge`, `Command_SpectateGame` | +| **Chat** | `Event_GameSay`, `ServerInfo_Chat` | +| **Decks** | Deck list serialization, sideboard plans | + +### 4.3 Authority Model + +- **Server is authoritative**: All game state changes go through the server. +- **Client sends commands, server validates**: The client cannot manipulate the game state directly. +- **Events are broadcast**: Server sends `GameEvent` containers to all connected clients. +- **Replays are deterministic**: Same command sequence produces identical game states. + +--- + +## 5. Server Architecture (Servatrice) + +### 5.1 Server Class Hierarchy + +| Class | Role | +|-------|------| +| `Servatrice` | Main server class (~42KB) — manages connections, rooms, games | +| `ServatriceDatabaseInterface` | Database operations (~58KB) — user accounts, decks, logs | +| `ServerSocketInterface` | Per-client handler (~106KB) — handles all client messages | +| `IslInterface` | Inter-server-link for clustering | +| `ServerLogger` | Centralized logging | + +### 5.2 Server Features + +- **Room management**: Games are organized in rooms; rooms have game types. +- **User accounts**: MySQL-backed with authentication, password reset, email. +- **Deck storage**: Decks saved to database with versioning. +- **Game logging**: Full game event logs for disputes. +- **Spectator system**: Spectators can watch (with optional omniscient mode). +- **Admin commands**: Kicking, banning, game management. +- **Clustering**: ISL protocol for running multiple servers in a cluster. +- **Email**: SMTP support for registration/password reset. + +### 5.3 Database Schema (MySQL) + +The `servatrice.sql` file defines the schema including: +- User accounts, groups, bans +- Deck lists with versioning +- Game logs +- Room/game state tables +- Admin logs + +--- + +## 6. Card Database (Oracle) + +The `oracle` module downloads and maintains the card database: +- Sources: MTGJSON data +- Local storage: SQLite or file-based +- Features: Card searching, printing history, set filtering +- Updates: Periodic refresh capability + +--- + +## 7. Deck Management + +| Component | Role | +|-----------|------| +| `DeckList` | Core deck data structure (cards, sideboard, categories) | +| `DeckListNodeTree` | Tree structure for organizing decks | +| `SideboardPlan` | Pre/post-sideboard plan management | +| `DeckListHistoryManager` | Deck version history | +| `DeckFilterString` | Card name filtering | + +Deck formats supported: Commander, Standard, Modern, Legacy, Vintage, etc. + +--- + +## 8. UI/Rendering Architecture + +### 8.1 Qt Graphics Framework + +``` +┌─────────────────────────────────────────────────────────┐ +│ Window Main (window_main.cpp) — ~44KB │ +│ ┌──────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ Game │ │ Chat/Log │ │ Deck/Filter │ │ +│ │ Scene │ │ Panel │ │ Panel │ │ +│ │ (QScene) │ │ │ │ │ │ +│ └──────────┘ └──────────────┘ └──────────────────┘ │ +│ │ │ +│ ▼ │ +│ GameScene (game_scene.cpp) — ~23KB │ +│ GameView (game_view.cpp) — ~10KB │ +│ PhasesToolbar — Phase navigation │ +│ ZValueLayerManager — 2D depth ordering │ +└─────────────────────────────────────────────────────────┘ +``` + +### 8.2 Theming System + +- SVG-based card back designs +- Configurable themes (backgrounds, colors, layouts) +- Card picture loading from external URLs +- Custom counters and tally displays + +--- + +## 9. Technology Stack Summary + +| Layer | Technology | +|-------|-----------| +| Language | C++20 | +| GUI Framework | Qt 5 or Qt 6 (QtWidgets) | +| Network Protocol | Protocol Buffers (3.21+) | +| Build System | CMake 3.10+ | +| Server Database | MySQL | +| Card Database | SQLite / file-based (Oracle) | +| Packaging | CPack (DEB, RPM, NSIS, DMG) | +| Networking | Qt TCP | +| Serialization | protobuf | +| RNG | SFMT (Sobol/Smirnov) | +| Translations | Qt .ts files | +| Documentation | Doxygen | + +--- + +## 10. Key Architectural Patterns + +| Pattern | Where Used | +|---------|-----------| +| **MVC (via Qt)** | `game_state` (model), `game_scene` (view), `GameEventHandler` (controller) | +| **Event Bus** | Qt signals/slots for all state transitions | +| **Command Pattern** | `Command_PlayCard`, `Command_Attack`, etc. — all game actions are commands | +| **Observer** | `GameEventHandler` emits signals to update UI on every state change | +| **Strategy** | `CardZoneLogic` subclasses (Hand, Stack, Table, Pile, View) | +| **Facade** | `AbstractGame` wraps all game subsystems behind a single interface | +| **Adapter** | `GameMetaInfo` wraps protobuf messages with Qt-friendly getters | +| **Singleton** | Settings cache, database interface | +| **Memento** | `DeckListMemento` for undo/redo in deck editing | +| **Repository** | `ServatriceDatabaseInterface` abstracts MySQL access | +| **Plugin (debatable)** | ISL interface for adding server nodes | + +--- + +## 11. Strengths & Weaknesses (for modern web adaptation) + +### Strengths +- **Clear authority model**: Server is always right — ideal for web multiplayer. +- **Deterministic replay**: Game events are serialized, enabling perfect replays. +- **Zone abstraction**: Each zone type is independently implementable. +- **Phase system**: Well-defined MTG turn structure. +- **Command/Event separation**: Clean separation between what a player *wants* to do and what *happens*. + +### Weaknesses (for web adaptation) +- **Desktop-first**: Qt Widgets, not designed for browser deployment. +- **Massive files**: `serversocketinterface.cpp` is 106KB — monolithic. +- **No web protocol**: Only TCP, no WebSocket, no REST. +- **No mobile**: No responsive design, no mobile clients. +- **Complex build**: Qt + CMake + protobuf + MySQL — heavy dev environment. +- **No real-time scaling**: Single-server architecture (ISL is clustering, not load balancing). +- **No card images built-in**: External URL loading only. +- **No API**: No REST/GraphQL for external integrations. + +--- + +## 12. Relevance to Modern Web MTG App + +### What to Reuse (Architecture Patterns) +1. **Event-driven game state**: Server authoritative, client event-driven UI. +2. **Zone abstraction**: Hand/Stack/Table/Graveyard/Exile/Library as separate zone types. +3. **Command pattern**: Player actions as discrete commands, validated server-side. +4. **Phase system**: 11-phase MTG turn structure with sub-phases. +5. **Replay system**: Serialize game events for replay capability. +6. **Card database**: Structure for card info, sets, printing history. +7. **Deck management**: Tree structure, sideboard plans, format support. + +### What to Modernize +1. **Protocol**: Replace protobuf over TCP with **WebSocket** (or gRPC-Web) for browser. +2. **API**: Add **REST/GraphQL** layer for external integrations. +3. **Frontend**: Replace Qt Widgets with **React/Next.js** or **Vue 3**. +4. **State management**: Replace Qt signals/slots with **state machines** (XState) or **Zustand**. +5. **Backend**: Replace C++/MySQL with **TypeScript/Node.js** or **Python/FastAPI** + **PostgreSQL**. +6. **Card images**: Bundle card images with the application or use CDN. +7. **Real-time**: Use **WebSocket** for live game state sync. +8. **Mobile**: Responsive design from the start. +9. **Testing**: Unit test the game engine in isolation (no GUI dependency). + +### Suggested Tech Stack for Modern Web MTG +| Component | Recommendation | +|-----------|---------------| +| Frontend | Next.js 14+ (React, TypeScript) | +| Game State | XState (state machines) or Zustand | +| UI Rendering | React Three Fiber (3D) or PixiJS (2D) | +| Backend | FastAPI (Python) or Hono (TypeScript) | +| Database | PostgreSQL (user data) + Redis (game state cache) | +| Real-time | WebSocket (via FastAPI or Socket.IO) | +| Card Data | MTGJSON v5 (already available in project) | +| Auth | JWT + refresh tokens | +| Deployment | Docker Compose (already used in project) | diff --git a/HANDOFF.md b/HANDOFF.md index 1c846c2..1b300ca 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -255,44 +255,123 @@ docker exec psql -U mtgonline_user mtgdata -c "SELECT * F - Check CORS_ORIGINS setting in app/core/settings.py - Ensure frontend URL matches allowed origins -## Next Phase: Backend Expansion for Frontend Support +## Next Phase: V1 Backend — Deck Building & Card Management (NEW SCOPE) -**Primary Focus**: Expand the PostgreSQL database schema and API endpoints to support frontend deckbuilding and gameplay features on a per-user basis. +The v1 app function focuses on three core capabilities: +1. **Per-user deck building** with card search, deck precedents, and card suggestions +2. **Card list import** from spreadsheet/text files with fuzzy matching +3. **Multiplayer gameplay** (handled by a separate backend) -### Database Schema Expansion -- Per-user deck storage (decks, folders, custom card sets) -- Game state persistence (saved games, match history, game logs) -- User card collection tracking (owned cards, favorites) -- Game room state management (active games, waiting lists) -- Tournament and custom rule support +### 1. Per-User Deck Building -### API Endpoints to Implement -- Deck CRUD with user ownership and sharing -- Game room creation, joining, and state management -- Real-time WebSocket endpoints for multiplayer gameplay -- Card collection APIs (search, filter, organize) -- Game history and replay APIs -- Admin tools for game monitoring and moderation +#### Database Schema +- **`user_decks` table** (per-user storage in primary `mtgonline` database): + - `deck_id` (PK, auto-increment) + - `user_id` (FK → users) + - `name` (text) + - `status` (ENUM: `DRAFT`, `FINAL`) + - `DRAFT` = works in progress, can be edited freely + - `FINAL` = user considers it complete, no further changes expected + - `cards` (JSONB or separate junction table with `card_id`, `quantity`) + - `created_at`, `updated_at` (timestamps) + - `folder_id` (FK → user folders, optional) -### Frontend Features to Support (from ROADMAP.md Phase 2) -- **Deck Builder**: Card search with filters (name, color, type, set), drag-and-drop editor, import/export formats -- **Game Interface**: Game board visualization, player zones (hand, library, graveyard, exile, command), real-time updates -- **Chat System**: Room chat, game chat, player list, moderator tools -- **Admin Dashboard**: User management, ban/unban controls, game logs, system statistics +#### API Endpoints to Implement +- `POST /decks/` — Create new draft deck (auto status: DRAFT) +- `GET /decks/` — List user's decks, filtered by status +- `GET /decks/{deck_id}` — Get full deck details +- `PATCH /decks/{deck_id}` — Update deck (name, status, card list) +- `POST /decks/{deck_id}/finalize` — Transition DRAFT → FINAL +- `DELETE /decks/{deck_id}` — Delete deck (only if FINAL, or admin override) +- `GET /decks/{deck_id}/cards` — Get cards in deck with quantity counts -### Integration Requirements (from ROADMAP.md Phase 3) -- WebSocket client with reconnection logic -- Card database caching and search functionality -- Game logic implementation (turn-based state, mana tracking, stack resolution) -- Performance optimization (virtual scrolling, memoization, code splitting) +#### Deck Building Features +- **Card Search**: Search the MTG card database by name, type, set, color, etc. Returns matching cards with full details. +- **Deck Precedents**: Preset/starting deck templates that users can use as a basis. Could be built-in (e.g., "Starter Deck") or user-saved as FINAL decks to be reused. +- **Card Suggestion**: Given a card already in the deck, suggest similar cards (same type, same color, same set, same mana cost, or cards often paired with the input card in existing decks). -### Deployment & Production (from ROADMAP.md Phase 4) -- Docker Compose for development and production -- CI/CD pipeline with GitHub Actions -- Security hardening (rate limiting, input validation, HTTPS) -- Monitoring and alerting (structured logging, error tracking) +### 2. Card Import from Files -See **ROADMAP.md** for complete feature specifications and timeline. +#### Supported Formats +- XLSX (Excel) +- CSV +- JSON +- ODS (OpenDocument Spreadsheet) + +#### Import Flow +1. User uploads a file (XLSX, CSV, JSON, or ODS) +2. Backend parses the file — each row is treated as one card entry +3. Duplicates within the file are allowed (each row → one card instance) +4. For each card name in the file, backend performs **fuzzy matching** against the `Cards` PSQL table +5. Matched cards are stored in a **user-owned card table** with metadata: + - `user_id` (FK → users) + - `card_id` (FK → mtg_cards from mtgdata, matched via fuzzy search) + - `raw_name` (original name from file, for traceability) + - `confidence` (match score from fuzzy search) + - `imported_at` (timestamp) + +#### Fuzzy Search Requirements +- Must handle **spelling errors** (e.g., "Wondrrland" → "Wonderland") +- Must handle **American vs British English** differences (e.g., "color" vs "colour", "armor" vs "armour") +- Use a fuzzy string matching library (e.g., `python-Levenshtein`, `thefuzz`/`fuzzymatch`) +- Confidence threshold to auto-accept vs flag for manual review +- Bulk matching: process all cards in the file in a single batch operation + +#### API Endpoints to Implement +- `POST /cards/import` — Upload file for import +- `GET /cards/import/{import_id}/status` — Check import progress/status +- `GET /cards/import/{import_id}/results` — Get match results with confidence scores +- `POST /cards/import/{import_id}/confirm` — Confirm import (save to user card table) +- `GET /user/cards` — List user's imported/owned cards +- `DELETE /user/cards/{card_import_id}` — Remove from user card table + +### 3. Multiplayer Play Feature (Separate Backend) + +#### Architecture Decision +The multiplayer gameplay feature will live in a **separate backend codebase** to ensure smooth, independent development. This backend will communicate with the card backend via: + +- **API Calls**: For authentication, user data, deck retrieval, card lookups +- **Direct PSQL Queries**: For card data and user deck data + +#### Integration Points +- **Card Backend API** (`http://backend:8000`): + - `GET /api/cards/{card_id}` — Get card details for in-game display + - `GET /api/cards/search?q=...` — Search cards during gameplay + - `GET /api/sets/` — List available sets for game formatting + +- **PSQL Direct Access** (via shared connection string): + - `mtgdata` database — Read card data (cards, sets, etc.) + - `mtgonline` database — Read user decks (for deck validation, game setup) + +#### API Endpoints for Play Backend +- `POST /play/decks/{deck_id}/validate` — Validate deck against card database +- `GET /play/users/{user_id}/decks` — Get user's FINAL decks for selection +- `GET /play/cards/{card_id}` — Get card details for game board display + +**Note**: The play backend is out of scope for this document. See separate codebase/repository when ready. + +### Summary of Work Required in This Backend + +| Feature | Database | API | Notes | +|---------|----------|-----|-------| +| User deck CRUD | `mtgonline` (new tables) | Full CRUD + finalize | DRAFT/FINAL status | +| Card search | `mtgdata` (existing) | Search endpoint | Leverages existing card DB | +| Card suggestions | `mtgonline` + `mtgdata` | Suggestion endpoint | Based on similar cards | +| File import (XLSX/CSV/JSON/ODS) | `mtgonline` (new user cards table) | Upload + confirm | Fuzzy match required | +| Fuzzy matching service | N/A | Internal service | Handles spelling + EN variants | +| Play backend integration | Read-only access | API consumer | Separate codebase | + +### Files to Create/Modify +- New models: `models/user_deck.py`, `models/user_card.py` +- New migrations: Alembic migrations for new tables +- New router: `routers/decks.py`, `routers/card_import.py` +- New service: `services/fuzzy_card_matcher.py` +- New service: `services/deck_suggestion.py` +- New schema: `schemas/deck.py`, `schemas/card_import.py` +- Update `core/database.py` if new engine needed +- Update `requirements.txt` with fuzzy matching libraries + +See **ROADMAP.md Phase 2** for detailed task breakdown. ## State File diff --git a/HANDOFF_PROMPT.md b/HANDOFF_PROMPT.md deleted file mode 100644 index 17c16b4..0000000 --- a/HANDOFF_PROMPT.md +++ /dev/null @@ -1,83 +0,0 @@ -# MTG Online Backend - Handoff Prompt - -## Context -You are taking over the MTG Online Backend project. The project is a Python FastAPI application that integrates with MTGJSON API to provide Magic: The Gathering card data. The system downloads, processes, and stores MTGJSON datasets in PostgreSQL with a weekly refresh cycle. - -## Current Status -- ✅ **All containers destroyed** and Docker pruned -- ✅ **Code is working** - last tested successfully with 5.4M+ cards loaded -- ⏸️ **Project paused** - ready for handoff - -## Quick Start for New Thread - -```bash -# 1. Read the handoff documentation -cat /home/wall-o/projects/mtgonline/HANDOFF.md - -# 2. Check current state -cat /home/wall-o/projects/mtgonline/state.json - -# 3. Rebuild and deploy -cd /home/wall-o/projects/mtgonline -docker build -t mtgonline_backend backend/ -docker compose -p mtgonline up -d - -# 4. Monitor startup -docker logs -f mtgonline_backend -``` - -## Key Information - -**Location**: `/home/wall-o/projects/mtgonline` -**Last Commit**: `ad2742c` - "Fix MTGJSON download: use .gz URLs and handle pre-uncompressed files" - -**Core Service**: `backend/app/services/mtgjson_manager.py` -- Downloads MTGJSON data from `https://mtgjson.com/api/v5` -- Handles both gzip-compressed and pre-uncompressed JSON files -- Upserts to PostgreSQL with `ON CONFLICT DO UPDATE` -- Weekly refresh cycle (7 days) -- Container marked unhealthy if data corrupted - -**Known Issues Fixed**: -1. MTGJSON URL scheme changed from `.json` to `.json.gz` -2. Some files are pre-uncompressed (not actually gzipped) -3. Large files need 60-minute download timeout - -## What to Do Next - -1. **Rebuild and test** the deployment -2. **Verify MTGJSON data loads** correctly -3. **Review the codebase** and make any needed improvements -4. **Consider**: - - Incremental updates (vs full refresh) - - Better error handling - - Database optimization - - Monitoring and alerting - -## Important Files -- `HANOFOFF.md` - Complete documentation -- `state.json` - Project state -- `backend/app/services/mtgjson_manager.py` - Core MTGJSON integration -- `docker-compose.yml` - Container orchestration -- `.env` - Configuration - -## State Tracking -Update `state.json` after each significant change: -```json -{ - "task_description": "...", - "current_step": "...", - "files_created": [...], - "files_modified": [...], - "decisions": [...], - "next_steps": [...], - "blockers": [...], - "commit_hash": "...", - "timestamp": ... -} -``` - -## Git -- Credentials: `/home/wall-o/projects/gitea_credentials.txt` -- Commit after each milestone -- Push changes regularly diff --git a/ROADMAP.md b/ROADMAP.md index 3ab98a6..19ccce2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -51,100 +51,406 @@ A modern web-based implementation of the MTG Online multiplayer Magic: The Gathe - [x] Project Roadmap - [x] State tracking -## Phase 2: Frontend Development (TODO) +## Phase 2: V1 Backend — Deck Building & Card Management (IN PROGRESS) -### 2.1 Project Setup +### 2.0 Project Setup +- [x] Initialize Python project structure +- [x] Create requirements.txt with pinned dependencies +- [x] Set up pydantic-settings configuration +- [x] Configure async SQLAlchemy with PostgreSQL +- [x] Create JWT authentication system with bcrypt +- [x] Set up FastAPI application with CORS +- [x] Fuzzy matching library setup (python-Levenshtein / thefuzz) + +### 2.1 Database Models +- [x] User model (accounts, profiles, VIP status) +- [x] Ban model (moderation, history) +- [ ] **NEW: User Deck model** (`user_decks` table) + - `deck_id` (PK, auto-increment) + - `user_id` (FK → users) + - `name` (text) + - `status` (ENUM: DRAFT, FINAL) + - `cards` (JSONB or junction table with card_id, quantity) + - `created_at`, `updated_at` (timestamps) + - `folder_id` (FK → user folders, optional) +- [ ] **NEW: User Card model** (`user_cards` table) + - `user_card_id` (PK, auto-increment) + - `user_id` (FK → users) + - `card_id` (FK → mtg_cards from mtgdata) + - `raw_name` (original name from import file) + - `confidence` (match score from fuzzy search) + - `imported_at` (timestamp) + - `import_id` (FK → import batch) +- [ ] **NEW: Card Import Batch model** + - `import_id` (PK) + - `user_id` (FK → users) + - `file_name` (text) + - `status` (ENUM: PENDING, PROCESSING, COMPLETED, FAILED) + - `total_cards` (int) + - `matched_cards` (int) + - `created_at` (timestamp) + +### 2.2 API Endpoints + +#### User Deck CRUD +- [ ] `POST /decks/` — Create new draft deck (auto status: DRAFT) +- [ ] `GET /decks/` — List user's decks, filterable by status +- [ ] `GET /decks/{deck_id}` — Get full deck details +- [ ] `PATCH /decks/{deck_id}` — Update deck (name, status, card list) +- [ ] `POST /decks/{deck_id}/finalize` — Transition DRAFT → FINAL +- [ ] `DELETE /decks/{deck_id}` — Delete deck (only if FINAL, or admin override) +- [ ] `GET /decks/{deck_id}/cards` — Get cards in deck with quantity counts + +#### Card Search & Suggestions +- [ ] `GET /api/cards/search?q={query}&type={type}&set={set}&color={color}` — Search MTG cards +- [ ] `GET /api/cards/{card_id}` — Get card details +- [ ] `GET /api/sets/` — List available sets +- [ ] `GET /api/cards/suggest?deck_id={deck_id}&limit={n}` — Suggest similar cards + +#### Card Import +- [ ] `POST /cards/import` — Upload file (XLSX, CSV, JSON, ODS) +- [ ] `GET /cards/import/{import_id}/status` — Check import progress +- [ ] `GET /cards/import/{import_id}/results` — Get match results with confidence +- [ ] `POST /cards/import/{import_id}/confirm` — Confirm import +- [ ] `GET /user/cards` — List user's imported cards +- [ ] `DELETE /user/cards/{card_import_id}` — Remove from user cards + +### 2.3 Services + +#### Deck Building Services +- [ ] `services/deck_manager.py` — Deck CRUD operations, status transitions +- [ ] `services/card_search.py` — Card search with filters (name, type, set, color) +- [ ] `services/deck_suggestion.py` — Similar card suggestions based on existing deck + - Match by: same type, same color, same set, same mana cost + - Match by: cards often paired in existing user decks + +#### Card Import Services +- [ ] `services/file_parser.py` — Parse XLSX, CSV, JSON, ODS files + - Each row = one card entry (duplicates allowed) +- [ ] `services/fuzzy_card_matcher.py` — Fuzzy string matching service + - Handle spelling errors (e.g., "Wondrrland" → "Wonderland") + - Handle American vs British English (e.g., "color" vs "colour") + - Use python-Levenshtein or thefuzz for matching + - Return confidence scores for each match +- [ ] `services/import_batch_processor.py` — Process import batches + - Batch fuzzy matching for all cards in file + - Update import batch status + - Save matched cards to user_cards table + +### 2.4 Fuzzy Matching Implementation + +#### Requirements +- Must handle **spelling errors** in card names +- Must handle **American vs British English** differences +- Must return **confidence scores** for match quality +- Must process **bulk imports** efficiently + +#### Approach +1. Use `thefuzz` (Python) or `python-Levenshtein` for string matching +2. Implement two-stage matching: + - Stage 1: Exact match (if card name exists exactly) + - Stage 2: Fuzzy match (if no exact match, find best match) +3. Configure similarity threshold (e.g., 85% for auto-accept, 70-84% for manual review) +4. Pre-process card names to normalize: + - Remove extra whitespace + - Normalize punctuation + - Handle known spelling variants + +#### Example Match Results +``` +Raw: "Wondrrland Explorer" +Match: "Wonderland Explorer" (confidence: 92%) + +Raw: "Armour Plated" +Match: "Armor Plated" (confidence: 88%) + +Raw: "Colorful Burst" +Match: "Color Burst" (confidence: 85%) +``` + +### 2.5 Multiplayer Play Backend — Architecture Blueprint (Derived from Cockatrice Analysis) + +A detailed architecture analysis of **Cockatrice** (v3.1.0 "Graduation Day") — the mature open-source MTG online client/server — has been completed at `/home/wall-o/projects/mtgonline/C++/ARCHITECTURE_ANALYSIS.md`. + +The play backend will follow the same authoritative server model, adapted for a modern web stack: + +#### Authority Model + +``` +┌─────────────────┐ WebSocket (JSON) ┌─────────────────┐ +│ CLIENT │◄────────────────────────────────────►│ SERVER │ +│ (React/TS) │ │ (FastAPI/Py) │ +│ │ │ │ +│ • Game Scene │ GameCommands (play, attack, etc.) │ • PostgreSQL │ +│ • Hand View │◄────────────────────────────────────►│ • Game State │ +│ • Chat Panel │ GameEvents (state changes) │ • Room/Player │ +│ • Deck Panel │ │ Management │ +│ • Phase Toolbar │ Chat, Admin, Spectator │ • Replay Log │ +└─────────────────┘ └─────────────────┘ +``` + +#### Game Engine Architecture + +| Cockatrice Component | Modern Web Equivalent | Role | +|---|---|---| +| `AbstractGame` | `Game` (server-side) | Core game instance holding state, players, event handler | +| `GameMetaInfo` | `GameMetadata` | gameId, maxPlayers, description, started, spectators | +| `GameState` | `GameBoardState` | currentPhase, activePlayer, hostId, gameTimer | +| `GameEventHandler` | `GameEventDispatcher` | Central dispatch — processes events, prepares commands (~21KB in Cockatrice) | +| `PlayerManager` | `PlayerRegistry` | Coordinates all players in a game | +| `PlayerLogic` | `Player` | Per-player game logic (~10.7KB in Cockatrice) | +| `PlayerActions` | `PlayerCommands` | Concrete commands: play, attack, tap, draw (~64KB in Cockatrice) | +| `CardZone` | `Zone` (base class) | Abstract zone — Hand/Stack/Table/Graveyard/Exile/Library | +| `HandZone` | `Hand` | Player's hand (secret/hidden zone) | +| `StackZone` | `Stack` | Spells/abilities on the stack | +| `TableZone` | `Battlefield` | Permanents on the battlefield | +| `PileZone` | `Pile` (Graveyard, Exile, Library) | Discard/exile/draw piles | +| `Replay` | `GameReplay` | Serialized event stream for replay | +| `Phase` | `TurnPhase` | 11-phase MTG turn structure | + +#### Turn Phase System (11 Phases with Sub-Phases) + +``` +Untap → Upkeep → Draw → Main 1 → Combat → Main 2 → End → Cleanup + │ + └── Sub-phases: + Beginning of Combat + Declare Attackers + Declare Blockers + Combat Damage + End of Combat +``` + +#### Command/Event Flow + +``` +User Action (React component) + │ + ▼ +GameCommand (JSON message) + │ + ▼ +GameEventDispatcher.process() + │ + ▼ +Player.handleCommand() + │ + ▼ +ZoneLogic (state mutation) + │ + ▼ +GameEvent (broadcast to all clients via WebSocket) + │ + ▼ +Client receives & updates UI via Zustand/XState +``` + +#### Network Protocol Design (Adapted from Cockatrice's protobuf) + +Cockatrice uses Protocol Buffers over TCP. The modern web equivalent replaces protobuf with JSON over WebSocket: + +| Cockatrice Proto Message | JSON WebSocket Message | +|---|---| +| `ServerInfo_Game` | `{ "type": "game_info", "game_id": 1, "max_players": 2, ... }` | +| `ServerInfo_Player` | `{ "type": "player_info", "player_id": 1, "name": "...", ... }` | +| `Command_PlayCard` | `{ "type": "cmd_play_card", "card_id": 42, "zone": "hand", ... }` | +| `Command_Attack` | `{ "type": "cmd_attack", "attacker_id": 7, "targets": [3, 5], ... }` | +| `Event_Join` | `{ "type": "event_join", "player_id": 2, "properties": {...} }` | +| `Event_Leave` | `{ "type": "event_leave", "player_id": 1, "reason": "..." }` | +| `Event_SetActivePlayer` | `{ "type": "event_active_player", "player_id": 1 }` | +| `Event_SetActivePhase` | `{ "type": "event_active_phase", "phase": 5 }` | +| `Event_GameSay` | `{ "type": "event_chat", "player_id": 1, "message": "..." }` | +| `GameReplay` | `{ "type": "replay", "events": [...] }` | + +#### Architecture Patterns to Reuse + +| Pattern | Cockatrice Usage | Modern Equivalent | +|---|---|---| +| **Event Bus** | Qt signals/slots | WebSocket broadcast + Zustand stores | +| **Command Pattern** | `Command_PlayCard`, `Command_Attack` | JSON command messages, validated server-side | +| **Observer** | `GameEventHandler` emits signals | WebSocket events trigger UI updates | +| **Strategy** | `CardZoneLogic` subclasses | Zone classes with strategy pattern (Hand, Stack, Table, Pile, View) | +| **Facade** | `AbstractGame` | Single `Game` object wrapping all subsystems | +| **Memento** | `DeckListMemento` for undo | Immutable state snapshots for undo/redo | +| **Repository** | `ServatriceDatabaseInterface` | SQLAlchemy repositories for game state, decks, logs | + +#### Key Decisions (Informed by Cockatrice Analysis) + +1. **Server is authoritative** — clients send commands, server validates and broadcasts events. No client-side state manipulation. +2. **Deterministic replay** — serialize all game events; same command sequence produces identical state. +3. **Zone abstraction** — each zone type (Hand, Stack, Battlefield, Pile) is independently implementable. +4. **Phase system** — 11-phase MTG turn with sub-phases, tracked server-side. +5. **Command/Event separation** — what a player *wants* to do vs. what *happens*. + +#### What Modernizes Cockatrice + +| Cockatrice Limitation | Modern Web Solution | +|---|---| +| TCP only, no web protocol | WebSocket (JSON) | +| Qt Widgets desktop UI | React/Next.js with PixiJS for game board | +| C++/MySQL backend | Python/FastAPI + PostgreSQL | +| Single-server clustering | Stateless game servers + Redis state cache | +| No mobile support | Responsive design from start | +| No REST API | FastAPI REST + WebSocket hybrid | +| Protobuf serialization | JSON over WebSocket | +| 106KB monolithic server handler | Modular service architecture | + +### 2.6 Play Backend Responsibilities + +The play backend will implement: +- [ ] Real-time game state management (server-authoritative) +- [ ] Multiplayer WebSocket communication +- [ ] Game rule enforcement (combat, stack resolution, priority) +- [ ] Deck validation during gameplay +- [ ] Game history and replay (serialized event log) +- [ ] Room and game lobbies +- [ ] Spectator mode +- [ ] Admin commands (kick, ban, game control) + +**Note**: The play backend lives in a separate codebase. Integration points with the card backend: +- **Card Backend API** (`http://backend:8000`): + - Authentication via JWT + - Card lookups: `GET /api/cards/{card_id}` + - Card search: `GET /api/cards/search?q=...` + - Set lists: `GET /api/sets/` + - User decks: `GET /api/users/{user_id}/decks` +- **Direct PSQL Access**: + - `mtgdata` database: Read card data, sets, etc. + - `mtgonline` database: Read user decks, validate deck legality + - Normalize punctuation + - Handle known spelling variants + +#### Example Match Results +``` +Raw: "Wondrrland Explorer" +Match: "Wonderland Explorer" (confidence: 92%) + +Raw: "Armour Plated" +Match: "Armor Plated" (confidence: 88%) + +Raw: "Colorful Burst" +Match: "Color Burst" (confidence: 85%) +``` + +### 2.5 Multiplayer Play Backend (SEPARATE CODEBASE) + +#### Architecture +The multiplayer gameplay feature will live in a **separate backend codebase** to ensure smooth, independent development. + +#### Integration Points +- **Card Backend API** (`http://backend:8000`): + - Authentication via JWT + - Card lookups: `GET /api/cards/{card_id}` + - Card search: `GET /api/cards/search?q=...` + - Set lists: `GET /api/sets/` + - User decks: `GET /api/users/{user_id}/decks` + +- **Direct PSQL Access**: + - `mtgdata` database: Read card data, sets, etc. + - `mtgonline` database: Read user decks, validate deck legality + +#### Play Backend Responsibilities +- Real-time game state management +- Multiplayer WebSocket communication +- Game rule enforcement +- Deck validation during gameplay +- Game history and replay + +**Note**: This backend is out of scope for the current codebase. See separate repository when ready. + +### 2.6 Testing +- [ ] Unit tests for deck CRUD operations +- [ ] Unit tests for card search functionality +- [ ] Unit tests for card suggestion algorithm +- [ ] Unit tests for file parsers (XLSX, CSV, JSON, ODS) +- [ ] Unit tests for fuzzy matching service +- [ ] Integration tests for import workflow +- [ ] Load tests for bulk import processing + +### 2.7 Documentation +- [ ] API documentation (FastAPI auto-generated) +- [ ] Database schema documentation +- [ ] Fuzzy matching algorithm documentation +- [ ] Import workflow documentation +- [ ] Play backend integration guide + +## Phase 3: Frontend Development (TODO) + +### 3.1 Project Setup - [ ] Initialize React + TypeScript project with Vite - [ ] Configure ESLint, Prettier, TypeScript strict mode - [ ] Set up Zustand for state management - [ ] Configure Tailwind CSS for styling - [ ] Set up Vitest + React Testing Library -### 2.2 Authentication +### 3.2 Authentication - [ ] Login form with JWT token storage - [ ] Registration form with validation - [ ] Protected routes and auth context - [ ] Session management and token refresh -### 2.3 Deck Builder +### 3.3 Deck Builder - [ ] Card search with filters (name, color, type, set) - [ ] Deck list editor with drag-and-drop - [ ] Import/export deck formats (plain text, native XML) - [ ] Folder management UI - [ ] Real-time deck statistics (card count, mana curve) +- [ ] **NEW: Deck status indicator** (DRAFT vs FINAL) +- [ ] **NEW: Card suggestion panel** (shows similar cards) -### 2.4 Game Interface +### 3.4 Card Import Interface +- [ ] File upload component (XLSX, CSV, JSON, ODS) +- [ ] Import progress indicator +- [ ] Match results display with confidence scores +- [ ] Manual override for low-confidence matches +- [ ] Import history and re-import capability + +### 3.5 Game Interface (TODO - Dependent on Play Backend) - [ ] Game board visualization (zones, cards) - [ ] Player hand (private zone) - [ ] Library, graveyard, exile, command zones - [ ] Card interaction (click, drag, hover) - [ ] Real-time WebSocket updates -### 2.5 Chat System +### 3.6 Chat System - [ ] Room chat interface - [ ] Game chat (in-game messaging) - [ ] Player list display - [ ] Moderator tools (kick, ban) -### 2.6 Admin Dashboard +### 3.7 Admin Dashboard - [ ] User management interface - [ ] Ban/unban controls - [ ] Game logs viewer - [ ] System statistics -## Phase 3: Integration & Polish (TODO) +## Phase 4: Integration & Polish (TODO) -### 3.1 WebSocket Client +### 4.1 WebSocket Client - [ ] WebSocket connection management - [ ] Reconnection logic with exponential backoff - [ ] Message serialization/deserialization - [ ] Protocol buffer message handling -### 3.2 Card Database +### 4.2 Card Database - [ ] Import MTJSON card data - [ ] Cache card images locally - [ ] Search and filter functionality - [ ] Card tooltips with oracle text -### 3.3 Game Logic +### 4.3 Game Logic (TODO - Dependent on Play Backend) - [ ] Turn-based state management - [ ] Priority system implementation - [ ] Stack resolution - [ ] Mana payment tracking - [ ] Life totals and counters -### 3.4 Performance +### 4.4 Performance - [ ] Virtual scrolling for card lists - [ ] Memoization and React.memo - [ ] Code splitting and lazy loading - [ ] WebSocket message batching -## Phase 4: Deployment & Production (TODO) - -### 4.1 DevOps -- [ ] Docker Compose for development -- [ ] Production Docker images -- [ ] CI/CD pipeline (GitHub Actions) -- [ ] Environment configuration management - -### 4.2 Security -- [ ] Rate limiting -- [ ] Input validation and sanitization -- [ ] CORS configuration -- [ ] HTTPS/SSL configuration - -### 4.3 Monitoring -- [ ] Logging with structured formats -- [ ] Error tracking (Sentry) -- [ ] Performance monitoring -- [ ] Uptime monitoring - -### 4.4 Documentation -- [ ] User documentation -- [ ] Developer documentation -- [ ] API documentation -- [ ] Architecture documentation - ## Phase 5: Advanced Features (FUTURE) ### 5.1 Multiplayer Enhancements diff --git a/backend/BACKEND_TESTING_SUMMARY.md b/backend/BACKEND_TESTING_SUMMARY.md deleted file mode 100644 index e2837e8..0000000 --- a/backend/BACKEND_TESTING_SUMMARY.md +++ /dev/null @@ -1,115 +0,0 @@ -# Backend Testing Summary - -## Overview -Successfully fixed and completed the backend test suite for the mtgonline project. All **20 tests** are now passing. - -## Test Results -``` -======================= 20 passed, 57 warnings in 5.20s ======================== -``` - -### Test Breakdown -- **Admin Tests**: 5/5 passing - - `test_list_users_admin` - - `test_list_users_non_admin` - - `test_create_ban` - - `test_list_bans` - - `test_unban_user` - -- **Authentication Tests**: 7/7 passing - - `test_login_success` - - `test_login_invalid_password` - - `test_login_nonexistent_user` - - `test_register_success` - - `test_register_duplicate_username` - - `test_get_current_user` - - `test_refresh_token` - -- **Deck Management Tests**: 8/8 passing - - `test_create_deck` - - `test_list_decks` - - `test_get_deck` - - `test_update_deck` - - `test_delete_deck` - - `test_create_folder` - - `test_list_folders` - - `test_delete_folder` - -## Key Changes Made - -### 1. JWT Token Updates (`app/core/security.py`) -- Added `privlevel` field to JWT access tokens -- Updated `get_current_user()` to extract `privlevel` from token -- Updated `create_access_token()` to accept `privlevel` parameter - -### 2. Test Fixtures (`tests/conftest.py`) -- Fixed `client` fixture to share database session with test fixtures -- Used `hash_password()` for proper bcrypt password hashing -- Updated fixture scope from `session` to `function` for isolation -- Properly cleaned up dependency overrides - -### 3. Router Fixes (`app/routers/decks.py`) -- Reordered routes to prevent `/folders` from matching `/{deck_id}` -- Routes now checked in correct order: specific routes first, then parameterized - -### 4. Database Model Updates (`app/models/models.py`) -- Made `folder_id` in `DecklistFile` nullable (optional at creation) - -### 5. Schema Updates (`app/schemas/schemas.py`) -- Removed relationship fields from `FolderResponse` to avoid async context issues -- Simplified schema to only include direct fields - -## Architecture Notes - -### Authentication Flow -``` -Login → JWT Token (includes privlevel) → Authorization checks -``` - -### Test Database Setup -- Uses in-memory SQLite (`sqlite+aiosqlite:///:memory:`) -- Each test function gets isolated database state -- Shared session via dependency override - -### Security Features Tested -- Password hashing with bcrypt -- JWT token validation -- Admin privilege checks (privlevel-based authorization) -- Duplicate username/email prevention -- Token refresh mechanism - -## Repository Information -- **Repository**: `https://git.optimex.systems/admin/mtgonline.git` -- **Branch**: `main` -- **Latest Commit**: `167a352` -- **Status**: Backend test suite complete and ready for frontend development - -## Next Steps -1. ✅ Backend test suite complete -2. ⏳ Frontend development (Next.js) -3. ⏳ API integration testing -4. ⏳ Deployment setup - -## Files Modified -- `app/core/security.py` -- `app/models/models.py` -- `app/routers/auth.py` -- `app/routers/decks.py` -- `app/schemas/schemas.py` -- `tests/conftest.py` -- `tests/test_admin.py` -- `tests/test_auth.py` -- `tests/test_decks.py` - -## Commands -```bash -# Run all tests -cd /home/wall-o/projects/mtgonline/backend -/home/wall-o/workspace/venv/bin/python -m pytest tests/ -v - -# Run specific test file -/home/wall-o/workspace/venv/bin/python -m pytest tests/test_auth.py -v - -# Run with verbose output -/home/wall-o/workspace/venv/bin/python -m pytest tests/ -v --tb=short -``` diff --git a/backend/CHAT_PROMPT_TEST.md b/backend/CHAT_PROMPT_TEST.md deleted file mode 100644 index ebdc340..0000000 --- a/backend/CHAT_PROMPT_TEST.md +++ /dev/null @@ -1,103 +0,0 @@ -# Backend System Test Prompt - -## Project Context -You are working on the **mtgonline** backend project located at `/home/wall-o/projects/mtgonline/backend/`. - -The backend is a FastAPI application with: -- PostgreSQL database (MTG Online + MTG data) -- Redis caching -- JWT authentication -- MTG card search and statistics endpoints - -## Current State -Read the `state.json` file at `/home/wall-o/projects/mtgonline/backend/state.json` to understand: -- What has been completed -- What the current focus is -- Any blockers or pending tasks - -## Task: Full Backend System Test - -Your goal is to perform a comprehensive system test of the backend to verify all components work correctly together. - -### Steps to Follow: - -1. **Read State File** - - Read `/home/wall-o/projects/mtgonline/backend/state.json` - - Understand current progress and what's been tested - -2. **Check Docker Containers** - - Run: `cd /home/wall-o/projects/mtgonline && docker compose ps` - - If containers aren't running, start them: `docker compose up -d` - - Wait for services to be healthy (~40 seconds) - -3. **Run System Test** - - Execute: `cd /home/wall-o/projects/mtgonline/backend && python test_system.py` - - This tests: - - Database connections (PostgreSQL + Redis) - - Cache operations - - Database tables - - API endpoints (health, card search, statistics, auth) - -4. **Analyze Results** - - If tests fail, investigate and fix issues - - Common issues: - - Database not running - - Redis connection failed - - API routes not registered - - Environment variables not set - -5. **Update State** - - Update `state.json` with test results - - Note any bugs found and fixed - - Document what's working and what needs attention - -6. **Report Findings** - - Summarize test results (pass/fail rates) - - List any issues found - - Recommend next steps - -## Expected Test Coverage - -The system test (`test_system.py`) should verify: -- ✅ MTG Online PostgreSQL connection -- ✅ MTG PostgreSQL connection -- ✅ Redis connection and operations -- ✅ MTG Online database tables (users, decks) -- ✅ MTG database tables (sets, cards) -- ✅ MTG card search functionality -- ✅ Health endpoint -- ✅ Card search API endpoint -- ✅ Statistics API endpoint -- ✅ Authentication endpoint - -## Success Criteria - -All tests should pass (100% success rate) before considering the backend ready for frontend development. - -## Commands Reference - -```bash -# Check Docker status -cd /home/wall-o/projects/mtgonline && docker compose ps - -# Start containers -cd /home/wall-o/projects/mtgonline && docker compose up -d - -# Wait for health (40 seconds) -sleep 40 - -# Run system test -cd /home/wall-o/projects/mtgonline/backend && python test_system.py - -# Check logs if issues -cd /home/wall-o/projects/mtgonline && docker compose logs backend -``` - -## Important Notes - -- All code execution must be as user `wall-o` (not root) -- Use sudo -u wall-o when running commands -- Pin versions in requirements.txt -- Follow PEP 8 for Python code -- Log errors with context -- One change per commit, descriptive messages diff --git a/backend/CONTINUATION_PROMPT.md b/backend/CONTINUATION_PROMPT.md deleted file mode 100644 index 179d9ae..0000000 --- a/backend/CONTINUATION_PROMPT.md +++ /dev/null @@ -1,120 +0,0 @@ -# MTG Online Backend - Context for Continuation - -## Current State -I just completed fixing 10 issues with the MTG Online backend codebase. All fixes are in place but **have not yet been deployed or tested**. - -## What Was Fixed - -### Files Modified/Created: -1. **`app/routers/interactions.py`** - NEW - Comprehensive interaction search endpoints -2. **`app/main.py`** - REWRITTEN - Fixed router mounting and logging -3. **`app/routers/__init__.py`** - REWRITTEN - Added all router exports -4. **`Dockerfile`** - REWRITTEN - Added scripts directory and permissions - -### Key Changes: -- All 8 routers now properly mounted in FastAPI -- Interaction endpoints: synergies, counters, evolutions, recommendations, search, stats -- Verbose logging with debug support -- Consistent async database usage -- Redis caching for performance -- Proper error handling throughout - -## Next Steps - Execute These Commands - -### Step 1: Stop all containers -```bash -cd /home/wall-o/projects/mtgonline -docker compose down -v -``` - -### Step 2: Build the backend -```bash -cd /home/wall-o/projects/mtgonline -docker compose build backend -``` - -### Step 3: Deploy the stack -```bash -cd /home/wall-o/projects/mtgonline -docker compose up -d -``` - -### Step 4: Wait for containers to start -```bash -# Watch logs in real-time -docker compose logs -f backend -``` - -### Step 5: Verify health -```bash -# Check all containers -docker compose ps - -# Test health endpoint -curl -s http://localhost:5555/health | python -m json.tool - -# Check API docs are accessible -curl -s http://localhost:5555/docs | head -20 -``` - -### Step 6: Verify interaction endpoints -```bash -# List all registered routes -curl -s http://localhost:5555/openapi.json | python -m json.tool | grep -E '"path":|"/(interactions|cards|auth|users|decks|rooms|games|admin)"' - -# Test interaction search endpoint -curl -s "http://localhost:5555/interactions/search/synergies?limit=5" | python -m json.tool -``` - -## Expected Success Indicators -- All containers show `healthy` status -- `/health` returns `{"status": "healthy", "version": "0.2.0"}` -- `/docs` returns OpenAPI JSON -- All routers appear in `/openapi.json` -- No Python import errors in logs -- Database connections successful -- Redis connection successful - -## Troubleshooting - -### If container fails to start: -```bash -# Check exit codes -docker compose ps - -# View recent logs -docker compose logs --tail=100 backend - -# Check if port 5555 is in use -sudo lsof -i :5555 -``` - -### Common Issues: -1. **Import errors**: Check that all router files exist and have proper syntax -2. **Database connection**: Verify `.env` has correct database URLs -3. **Port conflicts**: Ensure no other service uses port 5555 -4. **Permission issues**: Dockerfile should run as `appuser` not root - -### To view database state: -```bash -# Connect to card database -docker exec -it mtgonline_db_card psql -U mtgonline -d mtgdata -c "\dt" - -# Check tables -docker exec -it mtgonline_db_card psql -U mtgonline -d mtgdata -c "SELECT count(*) FROM mtg_cards;" -``` - -## Important Context -- Backend port: **5555** (not 8000!) -- Two PostgreSQL containers: `mtgdata` and `users` -- Redis for caching -- All code runs as `wall-o` user (UID 1001) -- Python venv at `/home/wall-o/workspace/venv` - -## Files to Review if Issues Arise -- `/home/wall-o/projects/mtgonline/backend/app/main.py` - Router mounting -- `/home/wall-o/projects/mtgonline/backend/app/routers/interactions.py` - New endpoints -- `/home/wall-o/projects/mtgonline/backend/app/routers/__init__.py` - Exports -- `/home/wall-o/projects/mtgonline/backend/Dockerfile` - Container setup -- `/home/wall-o/projects/mtgonline/.env` - Configuration -- `/home/wall-o/projects/mtgonline/docker-compose.yml` - Stack definition diff --git a/backend/PORTED_STATE.md b/backend/PORTED_STATE.md deleted file mode 100644 index 9eb90ab..0000000 --- a/backend/PORTED_STATE.md +++ /dev/null @@ -1,132 +0,0 @@ -# MTG Online Backend - Ported State and Next Steps - -## Project Overview -The `mtgonline` project is a Magic: The Gathering online application with a Docker-based stack: -- Two PostgreSQL containers (card data + user data) -- Backend application on port 5555 -- MTGJSON data loading pipeline - -## Recent Work Summary - -### What Was Done -1. **Created comprehensive interaction router** (`backend/app/routers/interactions.py`) - - Synergies search with filters (type, strength, confidence, pagination) - - Counters search with filters - - Evolutions search with filters - - Card recommendations (synergy, counter, evolution types) - - Card interaction statistics - - Redis caching for performance - -2. **Fixed main.py** to properly mount all routers - - Removed duplicate search implementation - - Added all routers: auth, users, decks, rooms, games, admin, card_router, interactions - - Added verbose logging configuration - - Added lifespan events for startup/shutdown - -3. **Updated __init__.py** to export all routers - - Centralized router imports - - Proper package structure - -4. **Updated Dockerfile** to include interaction scripts - - Added scripts directory to container - - Made scripts executable - - Proper permissions for appuser - -### Files Modified/Created -- `backend/app/routers/interactions.py` - NEW -- `backend/app/main.py` - REWRITTEN -- `backend/app/routers/__init__.py` - REWRITTEN -- `backend/Dockerfile` - REWRITTEN - -### Key Technical Decisions -- All interactions use async SQLAlchemy with mtg_get_db dependency -- Redis caching with 10-30 minute TTLs -- Proper error handling with HTTPException -- Consistent database connection pattern across all endpoints -- Logging setup with debug/verbose support - -## Next Steps (Execute in Order) - -### 1. Stop and Destroy All Docker Containers -```bash -cd /home/wall-o/projects/mtgonline -docker compose down -v -``` - -### 2. Build the Backend Docker Container -```bash -cd /home/wall-o/projects/mtgonline -docker compose build backend -``` - -### 3. Deploy the Stack as a Test Instance -```bash -cd /home/wall-o/projects/mtgonline -docker compose up -d -``` - -### 4. Verify Stack Health -```bash -# Check all containers are running -docker compose ps - -# Check backend health endpoint -curl http://localhost:5555/health -``` - -### 5. Check Logs for Issues -```bash -# View backend logs -docker compose logs backend - -# View PostgreSQL logs if needed -docker compose logs db_card -docker compose logs db_user -``` - -### 6. Troubleshoot Issues -If errors are found: -- **Import errors**: Check that all router modules exist and are properly imported -- **Database connection errors**: Verify `.env` file has correct database URLs -- **Port conflicts**: Ensure port 5555 is available -- **Permission errors**: Check Dockerfile has proper user/permissions setup - -## Commands to Monitor Progress -```bash -# Watch logs in real-time -docker compose logs -f backend - -# Check container status -docker compose ps - -# Restart specific container -docker compose restart backend - -# View specific container logs -docker compose logs --tail=50 backend -``` - -## Key Configuration -- Backend port: 5555 -- Database URLs in `.env` file -- Two PostgreSQL databases: `mtgdata` (card data) and `users` (user data) -- Redis for caching -- All routers mounted in `app/main.py` - -## Expected Behavior -Once healthy, the backend should: -- Serve API documentation at `/docs` -- Respond to health checks at `/health` -- Have all interaction endpoints available at `/interactions/*` -- Show proper logging output indicating successful startup - -## Error Resolution Strategy -1. **Simple errors**: Fix directly (typos, import paths, missing dependencies) -2. **Complex issues**: Document the problem, check Docker logs for stack traces, and consult with user -3. **Database issues**: Verify connection strings, check PostgreSQL logs, ensure databases exist - -## Important Notes -- All code execution must be as wall-o user (not root) -- Use `/home/wall-o/workspace/venv` for Python dependencies -- Docker commands should be run from `/home/wall-o/projects/mtgonline` -- The `.env` file is separate from application config diff --git a/backend/README.md b/backend/README.md index d6078ba..6aba3e8 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,271 +1,104 @@ # MTG Online Backend -Python FastAPI application for processing MTGJSON card data and managing the MTG Online platform backend. +A Python FastAPI application that processes Magic: The Gathering card data from [MTGJSON](https://mtgjson.com/) and stores it in PostgreSQL, with Redis for caching. + +## Overview + +This project provides a backend API for a Magic: The Gathering Online platform. It downloads and processes MTGJSON v5 dataset dumps, loads them into a PostgreSQL database, and exposes REST endpoints for card data, user authentication, deck management, and game state. + +### Key Features + +- **MTGJSON Data Pipeline** — Downloads `AllPrintings.psql`, `AllIdentifiers.json`, `Keywords.json`, `CardTypes.json`, and `AllDeckFiles.zip` from MTGJSON v5 on startup or via a `POST /refresh` endpoint. +- **Dual PostgreSQL** — Two databases: `mtgonline` for the application (users, decks, auth) and `mtgdata` for MTG card data. +- **Redis Caching** — Used for card lookup caching and interaction pipeline state. +- **REST API** — `/docs` (Swagger) available at runtime. ## Architecture -### Core Components - -The backend consists of several key layers: - -**Core Layer** (`app/core/`) -- `settings.py` — Application configuration using pydantic-settings with environment variable overrides -- `database.py` — Dual PostgreSQL engine setup (mtgonline app DB + mtgdata MTG cards DB) -- `redis_client.py` — Redis connection and caching utilities - -**Models** (`app/models/`) -- `models.py` — SQLAlchemy ORM models for application data (users, decks, auth) -- `mtg_models.py` — ORM models for MTG card data - -**Services** (`app/services/`) -- `mtgjson_manager.py` — Primary MTGJSON data pipeline (download, unzip, upsert) -- `mtgjson_downloader.py` — HTTP client for MTGJSON API -- `mtgjson_loader.py` — Data loading and transformation -- `mtgjson_uploader.py` — Database upsert operations -- `card_database.py` — Card data access layer -- `game_server.py` — Game state management -- `deck_parser.py` — Deck list parsing and validation - -**Routers** (`app/routers/`) -- `auth.py` — JWT authentication endpoints -- `users.py` — User management -- `decks.py` — Deck CRUD operations -- `rooms.py` — Game room management -- `games.py` — Game state endpoints -- `admin.py` — Admin tools -- `card_router.py` — MTG card data API -- `interactions.py` — Card interaction engine -- `refresh.py` — MTGJSON data refresh endpoint -- `ws.py` — WebSocket support - -**Schemas** (`app/schemas/`) -- `schemas.py` — Pydantic models for request/response validation -- `proto_messages.py` — Protocol buffer message definitions -- `protocol_constants.py` — MTG protocol constants +``` +mtgonline/ +├── backend/ # FastAPI application +│ ├── app/ +│ │ ├── core/ # Settings, database engines, Redis client +│ │ ├── models/ # SQLAlchemy ORM models (app + MTG) +│ │ ├── routers/ # API route modules +│ │ ├── schemas/ # Pydantic request/response schemas +│ │ ├── services/ # Business logic (MTGJSON manager, card DB, game server) +│ │ └── main.py # FastAPI app entry point +│ ├── scripts/ # Utility scripts (downloads, migrations, checks) +│ ├── Dockerfile +│ └── requirements.txt +├── docker-compose.dev.yml # Development stack (Postgres x2, Redis, Backend) +├── docker-compose.yml # Production stack +├── scripts/ # Shared utility scripts +└── README.md +``` ### Data Flow -``` -MTGJSON API (https://mtgjson.com/api/v5/) - ↓ download -MTGJSON files (AllPrintings.psql, AllIdentifiers.json, etc.) - ↓ load/transform -PostgreSQL (mtgdata database) - ↓ query -REST API / Swagger Docs +1. **On startup**, the backend connects to PostgreSQL (both instances) and Redis. +2. It checks `mtg_refresh_log` in the `mtgdata` database for existing data. +3. If no data exists, it downloads MTGJSON files from `https://mtgjson.com/api/v5/`, unzips if needed, and upserts them into `mtgdata` tables (`mtg_sets`, `mtg_cards`, etc.). +4. The data is then available via REST endpoints. + +## Quick Start + +### Prerequisites + +- Docker and Docker Compose + +### Run the Stack + +```bash +cd /home/wall-o/projects/mtgonline + +# Start all services (Postgres x2, Redis, Backend) +docker compose -f docker-compose.dev.yml up -d + +# View logs +docker compose -f docker-compose.dev.yml logs -f backend ``` -## Database Setup +The backend will automatically download MTGJSON data on first startup (this may take several minutes). -### Primary Database (mtgonline) -- **Purpose**: Application data (users, decks, auth tokens) -- **Connection**: `postgresql+asyncpg://mtgonline_user:mtgonline_password@postgres:5432/mtgonline` -- **Port**: 5432 (internal), 5432:5432 (host) +### Access -### MTG Data Database (mtgdata) -- **Purpose**: MTG card data, sets, refresh logs -- **Connection**: `postgresql+asyncpg://mtgonline_user:mtgonline_password@mtgdata:5432/mtgdata` -- **Port**: 5432 (internal), 5433:5432 (host) -- **Tables**: - - `mtg_sets` — Card sets metadata - - `mtg_cards` — Individual card data - - `mtg_refresh_log` — Refresh history and status +| Service | Address | +|---------------|---------------------| +| Backend API | `http://localhost:5555` | +| Swagger Docs | `http://localhost:5555/docs` | +| Health Check | `http://localhost:5555/health` | +| PostgreSQL (app) | `localhost:5432` | +| PostgreSQL (MTG) | `localhost:5433` | +| Redis | `localhost:6379` | -### Redis -- **Purpose**: Caching, session management -- **Connection**: `redis://redis:6379` -- **Port**: 6379 (internal), 6379:6379 (host) +### Manual Data Refresh -## MTGJSON Data Pipeline - -### Downloaded Files - -The backend downloads these files from MTGJSON v5: -- `AllPrintings.psql` — Main card data (PostgreSQL format) -- `AllIdentifiers.json` — Card identifiers (Multiverse, Scryfall, etc.) -- `Keywords.json` — Card keywords -- `CardTypes.json` — Card type definitions -- `AllDeckFiles.zip` — Deck files (must be unzipped) - -### Refresh Logic - -1. **On Startup**: Checks `mtg_refresh_log` for existing data -2. **If No Data**: Downloads and loads all MTGJSON files (may take minutes) -3. **Manual Refresh**: `POST /refresh` endpoint triggers immediate reload -4. **Logging**: All refreshes logged to `mtg_refresh_log` with status, timing, and counts +```bash +# Trigger a manual MTGJSON refresh +curl -X POST http://localhost:5555/refresh +``` ## Environment Variables | Variable | Default | Description | |----------|---------|-------------| -| `DATABASE_URL` | `postgresql+asyncpg://mtgonline_user:mtgonline_password@postgres:5432/mtgonline` | Primary database | -| `MTG_DATABASE_URL` | `postgresql+asyncpg://mtgonline_user:mtgonline_password@mtgdata:5432/mtgdata` | MTG data database | +| `DATABASE_URL` | `postgresql+asyncpg://mtgonline_user:mtgonline_password@postgres:5432/mtgonline` | Primary database connection | +| `MTG_DATABASE_URL` | `postgresql+asyncpg://mtgonline_user:mtgonline_password@mtgdata:5432/mtgdata` | MTG data database connection | | `REDIS_URL` | `redis://redis:6379` | Redis connection | -| `DATA_DIR` | `/app/data` | MTGJSON files directory | -| `UPLOAD_DIR` | `/app/uploads` | User uploads directory | +| `DATA_DIR` | `/app/data` | Directory for MTGJSON files | | `DEBUG` | `False` | Enable debug logging | -| `LOG_LEVEL` | `INFO` | Logging level | -| `SECRET_KEY` | `change-me-in-production` | JWT secret | -| `JWT_SECRET_KEY` | `change-me-in-production` | JWT signing key | -## Running the Backend - -### Docker (Recommended) +## Docker Cleanup ```bash -# Build image -cd backend -docker build -t mtgonline-backend:latest . +# Stop and remove all containers +docker compose -f docker-compose.dev.yml down -# Run with dependencies -docker compose -f ../docker-compose.dev.yml up -d backend +# Remove images and prune +docker system prune -a --volumes ``` -### Local Development - -```bash -cd backend - -# Create venv -python -m venv venv -source venv/bin/activate - -# Install dependencies -pip install -r requirements.txt - -# Set environment variables -export DATABASE_URL="postgresql+asyncpg://mtgonline_user:mtgonline_password@localhost:5432/mtgonline" -export MTG_DATABASE_URL="postgresql+asyncpg://mtgonline_user:mtgonline_password@localhost:5433/mtgdata" -export REDIS_URL="redis://localhost:6379" - -# Run server -python -m uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 -``` - -## API Endpoints - -### Health & Status -- `GET /health` — Health check with MTGJSON status -- `GET /` — API info - -### Authentication -- `POST /auth/login` — User login -- `POST /auth/register` — User registration -- `POST /auth/refresh` — Refresh JWT -- `GET /auth/me` — Current user - -### Users -- `GET /users/{user_id}` — Get user -- `PATCH /users/{user_id}` — Update user -- `POST /users/{user_id}/ban` — Ban user (admin) - -### Decks -- `GET /decks/` — List decks -- `POST /decks/` — Create deck -- `GET /decks/{deck_id}` — Get deck -- `PATCH /decks/{deck_id}` — Update deck -- `DELETE /decks/{deck_id}` — Delete deck - -### MTG Cards -- `GET /api/cards/` — Search cards -- `GET /api/cards/{card_id}` — Get card -- `GET /api/sets/` — List sets - -### Admin -- `GET /admin/users` — List all users -- `GET /admin/bans` — List bans -- `POST /admin/bans` — Create ban - -### Data Management -- `POST /refresh` — Trigger MTGJSON refresh - -### WebSocket -- `WS /ws/{room_id}` — Real-time game communication - -## Utility Scripts - -Located in `scripts/`: -- `download_mtgjson.py` — Manual MTGJSON download -- `check_mtgjson_status.py` — Verify data freshness -- `verify_mtgjson_data.py` — Data validation -- `inspect_db.py` — Database inspection -- `load_mtgjson_data.py` — Data loading - -## Testing - -```bash -cd backend - -# Run tests -pytest - -# Run with coverage -pytest --cov=app --cov-report=html -``` - -## Project Structure - -``` -backend/ -├── app/ -│ ├── __init__.py -│ ├── main.py # FastAPI app entry -│ ├── core/ -│ │ ├── settings.py # Configuration -│ │ ├── database.py # Database engines -│ │ └── redis_client.py # Redis utilities -│ ├── models/ -│ │ ├── models.py # App models -│ │ └── mtg_models.py # MTG models -│ ├── routers/ -│ │ ├── auth.py # Auth endpoints -│ │ ├── users.py # User endpoints -│ │ ├── decks.py # Deck endpoints -│ │ ├── rooms.py # Room endpoints -│ │ ├── games.py # Game endpoints -│ │ ├── admin.py # Admin endpoints -│ │ ├── card_router.py # Card API -│ │ ├── interactions.py # Card interactions -│ │ ├── refresh.py # Data refresh -│ │ └── ws.py # WebSocket -│ ├── schemas/ -│ │ ├── schemas.py # Pydantic models -│ │ ├── proto_messages.py # Protocol messages -│ │ └── protocol_constants.py -│ └── services/ -│ ├── mtgjson_manager.py # MTGJSON pipeline -│ ├── mtgjson_downloader.py -│ ├── mtgjson_loader.py -│ ├── mtgjson_uploader.py -│ ├── card_database.py # Card data access -│ ├── game_server.py # Game logic -│ └── deck_parser.py # Deck parsing -├── scripts/ # Utility scripts -├── tests/ # Test suite -├── Dockerfile # Container build -├── requirements.txt # Python dependencies -├── pyproject.toml # Ruff config -├── .env.example # Environment template -└── setup_db.py # Database setup script -``` - -## Troubleshooting - -### Backend can't connect to databases -- Verify all services are running: `docker compose -f ../docker-compose.dev.yml ps` -- Check logs: `docker compose -f ../docker-compose.dev.yml logs backend` -- Ensure environment variables match docker-compose.dev.yml - -### MTGJSON download fails -- Check network connectivity to mtgjson.com -- Verify DATA_DIR has write permissions -- Check disk space: `df -h` -- Manual download: `python scripts/download_mtgjson.py` - -### Database tables missing -- Run initialization: `docker exec -i mtgdata psql -U mtgonline_user mtgdata < /path/to/scripts/init-mtgdata.sql` -- Check tables: `docker exec mtgdata psql -U mtgonline_user mtgdata -c "\dt"` - ## License MIT diff --git a/backend/SPEC_synergy-mapping-engine.md b/backend/SPEC_synergy-mapping-engine.md deleted file mode 100644 index 5abf4c1..0000000 --- a/backend/SPEC_synergy-mapping-engine.md +++ /dev/null @@ -1,82 +0,0 @@ -# Technical Specification: MTG Synergy Mapping Engine - -## 1. Project Overview -The goal is to create a Python-based data pipeline that processes MTG card data from MTGJSON, identifies synergistic relationships between cards, and stores these relationships in a PostgreSQL database. This "Data Map" will power a deck-building assistant that suggests cards based on mechanical and strategic complementarity. - -## 2. Tech Stack -- **Language:** Python 3.12+ -- **Libraries:** `pandas` (data manipulation), `SQLAlchemy` (ORM), `psycopg2` (DB driver), `re` (regex for text processing). -- **Database:** PostgreSQL. -- **Data Source:** MTGJSON (`AllPrintings.json`, `AllSets.json`). - -## 3. Phase 1: Data Ingestion & Normalization -The script must flatten the nested MTGJSON structure into a relational format. - -### 3.1 Extraction -Extract the following fields from `AllPrintings.json`: -- `name`, `manaCost`, `types`, `text` (oracle text), `colorIdentity`, `set`. - -### 3.2 Text Processing (`TextProcessor` Class) -Implement a class to convert raw oracle text into "Functional Tokens." -- **Regex Mapping:** Use a dictionary of regex patterns to identify key actions. - - *Example:* `"draw a card"` $\rightarrow$ `TOKEN_DRAW_1` - - *Example:* `"destroy all creatures"` $\rightarrow$ `TOKEN_BOARD_WIPE_CREATURE` -- **Tagging:** Extract subtypes (Tribes) from the `types` field (e.g., "Elf", "Zombie"). - -## 4. Phase 2: The Synergy Engine (Logic) -The engine must evaluate every card pair and assign a weighted connection based on three tiers of synergy. - -### Tier A: Hard Synergies (Weight: 1.0) -**Logic:** Direct mechanical triggers. -- **Tribal Link:** If `Card_A.tags` (Tribe) $\cap$ `Card_B.text` (contains Tribe name) $\neq \emptyset$. -- **Trigger-Response:** Identify "Providers" (e.g., "Whenever you gain life") and "Payoffs" (e.g., "When you gain life, [Effect]"). Link Provider $\rightarrow$ Payoff. - -### Tier B: Functional Similarity (Weight: 0.6) -**Logic:** Substitution/Role mapping. -- **Role Dictionary:** Define roles (e.g., `RAMP`, `CARD_DRAW`, `REMOVAL`). -- **Mapping:** If both cards share the same `Role_ID` based on their Functional Tokens, create a link. - -### Tier C: Strategic Archetypes (Weight: 0.3) -**Logic:** Thematic co-occurrence. -- **Archetype Buckets:** Define keyword groups (e.g., `GRAVEYARD_STRAT` = ["mill", "graveyard", "reanimate"]). -- **Density Check:** If both cards have a high overlap of keywords from the same bucket, create a link. - -## 5. Phase 3: Database Schema (PSQL) -Implement the following schema: - -### Table: `cards` -- `card_id`: UUID (Primary Key) -- `name`: VARCHAR -- `oracle_text`: TEXT -- `mana_cost`: VARCHAR -- `color_identity`: ARRAY[VARCHAR] -- `tags`: ARRAY[VARCHAR] (Stored functional tokens and tribes) - -### Table: `synergy_types` -- `type_id`: INT (Primary Key) -- `label`: VARCHAR (e.g., 'Tribal', 'Mechanical', 'Substitute') - -### Table: `card_connections` -- `card_id_a`: UUID (FK $\rightarrow$ cards) -- `card_id_b`: UUID (FK $\rightarrow$ cards) -- `type_id`: INT (FK $\rightarrow$ synergy_types) -- `weight`: FLOAT -- **Constraint:** `CHECK (card_id_a < card_id_b)` to prevent bidirectional duplicates. - -## 6. Phase 4: Execution Pipeline -The script must execute in the following order: -1. **Ingest:** Parse JSON $\rightarrow$ Bulk load into `cards` table. -2. **Analyze:** Run `TextProcessor` $\rightarrow$ Update `cards.tags`. -3. **Map:** - - Iterate through card pairs. - - Evaluate Tiers A, B, and C. - - Insert identified synergies into `card_connections`. -4. **Index:** Create B-Tree indices on `card_id_a` and `card_id_b`. - -## 7. Phase 5: Recommendation Logic (API Level) -The resulting database must support the following query logic for the API: -1. **Input:** A list of `card_ids` currently in a deck. -2. **Query:** Find all `card_id_b` linked to any of the input IDs in `card_connections`. -3. **Aggregate:** Sum the `weight` for each suggested card. -4. **Filter:** Remove suggestions that do not match the `color_identity` of the deck. -5. **Output:** Return the top $N$ cards sorted by aggregate weight. \ No newline at end of file diff --git a/backend/STATE.md b/backend/STATE.md deleted file mode 100644 index fd0a3e6..0000000 --- a/backend/STATE.md +++ /dev/null @@ -1,32 +0,0 @@ -# MTG Online Backend - State Management - -## Overview -This project manages state for the MTG Online Backend application. -State is persisted in `state.json` and updated after every response. - -## State Schema - -```json -{ - "task_description": "Brief description of the current task", - "current_step": "What step we're on (e.g., 'Fixing backend issues')", - "files_created": ["list of files created"], - "files_modified": ["list of files modified"], - "decisions": ["list of key decisions made"], - "next_steps": ["list of next steps"], - "blockers": null | "description of blocker", - "commit_hash": null | "last commit hash", - "timestamp": "ISO 8601 timestamp" -} -``` - -## Commands - -- **Default**: On new chat, show summary, ask to resume or start fresh -- **Reset**: Clear all state and start fresh - -## Current State - -- **Last updated**: 2026-05-27T04:00:00Z -- **Status**: Active development -- **Next action**: Await user instructions diff --git a/backend/SUPPORTED_FILE_TYPES.md b/backend/SUPPORTED_FILE_TYPES.md deleted file mode 100644 index a25409d..0000000 --- a/backend/SUPPORTED_FILE_TYPES.md +++ /dev/null @@ -1,101 +0,0 @@ -# MTGJSON Data Manager - File Type Support - -## Supported File Types - -### 1. AllPrintings (Primary Card Database) -- **Accepts:** `AllPrintings.json` OR `AllPrintings.psql` -- **Processing:** - - `.json`: Parses JSON structure, extracts card data from `cards` array - - `.psql`: Parses SQL INSERT statements to extract card data -- **Database:** `mtg_cards` table -- **Required:** Yes (one of the two formats) - -### 2. AllIdentifiers (Stable Card Referencing) -- **Accepts:** `AllIdentifiers.json` -- **Processing:** Parses JSON structure, extracts identifiers -- **Database:** `mtg_identifiers` table -- **Required:** Yes - -### 3. Keywords & CardTypes (Game Logic/Mechanics) -- **Accepts:** `Keywords.json` and `CardTypes.json` -- **Processing:** Parses JSON arrays -- **Database:** `mtg_keywords` and `mtg_card_types` tables -- **Required:** Yes (both) - -### 4. AllDeckFiles (Deck Format Testing) -- **Accepts:** `AllDeckFiles.zip` -- **Processing:** - 1. Unzips the archive - 2. Finds all `.json` files recursively - 3. Parses each JSON file - 4. Upserts deck data -- **Database:** `mtg_deck_list` table -- **Required:** Yes - -## File Validation - -The health check endpoint now validates that all required files are present: - -```json -{ - "status": "healthy", - "data": { - "required_files": { - "all_present": true, - "found": ["AllPrintings.json", "AllIdentifiers.json", "Keywords.json", "CardTypes.json", "AllDeckFiles.zip"], - "missing": [], - "details": { - "AllPrintings": "OK (.json)", - "AllIdentifiers": "OK", - "Keywords": "OK", - "CardTypes": "OK", - "AllDeckFiles": "OK" - } - } - } -} -``` - -## PSQL File Parsing - -For `AllPrintings.psql`, the manager: -1. Reads the SQL file -2. Extracts `INSERT INTO mtg_cards (...) VALUES (...)` statements using regex -3. Parses column names and values -4. Handles NULL values, quoted strings, and JSON arrays -5. Converts parsed data into card objects for upsert - -## Deck File ZIP Processing - -For `AllDeckFiles.zip`: -1. Extracts to temporary directory -2. Recursively finds all `.json` files -3. Parses each file expecting deck format: - ```json - { - "listId": "...", - "name": "...", - "year": "...", - "date": "...", - "format": "..." - } - ``` -4. Upserts each deck into the database -5. Cleans up temporary files - -## Error Handling - -- Validates file existence before processing -- Logs warnings for missing files -- Handles malformed JSON/PSQL gracefully -- Rolls back transactions on individual card failures -- Continues processing remaining files on errors - -## Health Check - -The app is considered "healthy" when: -1. Cards and sets are loaded in database (>0) -2. All required files are present in the mounted volume -3. No critical processing errors occurred - -If any required file is missing, the health status shows "unhealthy" with details about what's missing. diff --git a/backend/scripts/card_interaction_rule_engine.py b/backend/scripts/card_interaction_rule_engine.py deleted file mode 100644 index dcaed63..0000000 --- a/backend/scripts/card_interaction_rule_engine.py +++ /dev/null @@ -1,959 +0,0 @@ -""" -MTG Card Interaction Rule Engine - -Extracts card interactions using structured rules instead of NLP. -Designed for rolling updates when new MTGJSON data is loaded. -""" -import re -from typing import Dict, List, Tuple, Optional, Any -from dataclasses import dataclass -from enum import Enum - - -class InteractionType(Enum): - """Types of card interactions.""" - MECHANIC = "mechanic" - ARCHETYPE = "archetype" - SYNERGY = "synergy" - COUNTER = "counter" - EVOLUTION = "evolution" - MANA = "mana" - SET_THEME = "set_theme" - - -class SynergyType(Enum): - """Types of synergies between cards.""" - ARCHETYPE_SUPPORT = "archetype_support" - MECHANIC_SUPPORT = "mechanic_support" - MANA_BASE = "mana_base" - COMBO_PARTNER = "combo_partner" - COUNTER_PARTNER = "counter_partner" - EVOLUTION_CHAIN = "evolution_chain" - - -class CounterType(Enum): - """Types of counter relationships.""" - DIRECT_COUNTER = "direct_counter" - MANA_DISADVANTAGE = "mana_disadvantage" - OUTCLASS = "outclass" - COUNTER_ROLE = "counter_role" - - -class EvolutionType(Enum): - """Types of evolution relationships.""" - TRANSFORM = "transform" - EVOLVE = "evolve" - DOUBLE_SIDED = "double_sided" - MODAL_DFC = "modal_dfc" - REPRINTED = "reprinted" - - -@dataclass -class CardProfile: - """Structured profile of a card for interaction extraction.""" - name: str - mana_cost: Optional[str] - type_line: Optional[str] - oracle_text: Optional[str] - subtypes: Optional[str] - supertypes: Optional[str] - colors: Optional[str] - color_identity: Optional[str] - power: Optional[str] - toughness: Optional[str] - loyalty: Optional[str] - set_code: Optional[str] - set_id: int - card_id: int - - # Extracted fields - mechanics: List[str] = None - archetypes: List[str] = None - targets: List[str] = None - triggers: List[str] = None - effects: List[str] = None - themes: List[str] = None - - def __post_init__(self): - if self.mechanics is None: - self.mechanics = [] - if self.archetypes is None: - self.archetypes = [] - if self.targets is None: - self.targets = [] - if self.triggers is None: - self.triggers = [] - if self.effects is None: - self.effects = [] - if self.themes is None: - self.themes = [] - - -class MTGRuleEngine: - """ - Extracts card interactions using structured rules. - - This is NOT NLP. It uses: - - Regex patterns for known game language - - Curated dictionaries for mechanics/archetypes - - Game rule logic for determining interactions - """ - - def __init__(self): - # Define mechanics and their extraction patterns - self.mechanics_patterns = { - 'flying': r'Flying', - 'first_strike': r'First strike', - 'double_strike': r'Double strike', - 'deathtouch': r'Death touch', - 'lifelink': r'Lifelink', - 'haste': r'Haste', - 'trample': r'Trample', - 'menace': r'Menace', - 'vigilance': r'Vegilance', - 'reach': r'Reach', - 'indestructible': r'Indestructible', - 'hexproof': r'Hexproof', - 'shroud': r'Shroud', - 'defender': r'Defender', - 'landfall': r'Landfall', - 'delve': r'Delve', - 'soulshift': r'Soulshift', - 'suspend': r'Suspend', - 'convoke': r'Convoke', - 'rampage': r'Rampage', - 'toxic': r'Toxic', - 'crew': r'Crew', - 'equip': r'Equip', - 'annihilator': r'Annihilator', - 'spectacle': r'Spectacle', - 'prowess': r'Prowess', - 'aftermath': r'Aftermath', - 'adapt': r'Adapt', - 'amplify': r'Amplify', - 'awaken': r'Awaken', - 'banding': r'Band with', - 'bestow': r'Bestow', - 'burst': r'Burst', - 'channel': r'Channel', - 'clash': r'Clash', - 'curse': r'Curse', - 'day_night': r'Day|Night', - 'decay': r'Decay', - 'defiant': r'Defiant', - 'demolish': r'Demolish', - 'detain': r'Detain', - 'detect': r'Detect', - 'devour': r'Devour', - 'disguise': r'Disguise', - 'disturb': r'Disturb', - 'dome': r'Dome', - 'dredge': r'Dredge', - 'emerge': r'Emerge', - 'encore': r'Encore', - 'endure': r'Endure', - 'evoke': r'Evoke', - 'evolve': r'Evolve', - 'exalted': r'Exalted', - 'exile': r'Exile', - 'exploit': r'Exploit', - 'extort': r'Extort', - 'fairy': r'Fairy', - 'fanatic': r'Fanatic', - 'fathom': r'Fathom', - 'fear': r'Fear', - 'feline': r'Feline', - 'flash': r'Flash', - 'flight': r'Flight', - 'foretell': r'Foretell', - 'frenzy': r'Frenzy', - 'fumble': r'Fumble', - 'galvanize': r'Galvanize', - 'gateway': r'Gateway', - 'genesis': r'Genesis', - 'graft': r'Graft', - 'grave': r'Grave', - 'grit': r'Grit', - 'guardian': r'Guardian', - 'harvest': r'Harvest', - 'healer': r'Healer', - 'heroic': r'Heroic', - 'hideaway': r'Hideaway', - 'hinterland': r'Hinterland', - 'hoard': r'Hoard', - 'hour': r'Hour', - 'illusion': r'Illusion', - 'immortal': r'Immortal', - 'impulse': r'Impulse', - 'inspiration': r'Inspiration', - 'instill': r'Instill', - 'iron': r'Iron', - 'junk': r'Junk', - 'kicker': r'Kicker', - 'knight': r'Knight', - 'land': r'Land', - 'leech': r'Leech', - 'lich': r'Lich', - 'lifespan': r'Lifespan', - 'lightning': r'Lightning', - 'living': r'Living', - 'lurk': r'Lurk', - 'madness': r'Madness', - 'manifest': r'Manifest', - 'map': r'Map', - 'meld': r'Meld', - 'miracle': r'Miracle', - 'mitosis': r'Mitosis', - 'modular': r'Modular', - 'moon': r'Moon', - 'mother': r'Mother', - 'morph': r'Morph', - 'mutate': r'Mutate', - 'ninja': r'Ninja', - 'night': r'Night', - 'nightmare': r'Nightmare', - 'pact': r'Pact', - 'paradox': r'Paradox', - 'persist': r'Persist', - 'pillage': r'Pillage', - 'pivot': r'Pivot', - 'planar': r'Planar', - 'polar': r'Polar', - 'pour': r'Pour', - 'prey': r'Prey', - 'priest': r'Priest', - 'primer': r'Primer', - 'probe': r'Probe', - 'prosperity': r'Prosperity', - 'psychic': r'Psychic', - 'puppet': r'Puppet', - 'quest': r'Quest', - 'quote': r'Quote', - 'rage': r'Rage', - 'raid': r'Raid', - 'raise': r'Raise', - 'rally': r'Rally', - 'rapid': r'Rapid', - 'rat': r'Rat', - 'rebound': r'Rebound', - 'reckless': r'Reckless', - 'recoup': r'Recoup', - 'reflect': r'Reflect', - 'refresh': r'Refresh', - 'replicate': r'Replicate', - 'reverberate': r'Reverberate', - 'reviviant': r'Reviviant', - 'rift': r'Rift', - 'rip': r'Rip', - 'ritual': r'Ritual', - 'rite': r'Rite', - 'rogue': r'Rogue', - 'savant': r'Savant', - 'scavenge': r'Scavenge', - 'seek': r'Seek', - 'shadow': r'Shadow', - 'shards': r'Shards', - 'skulk': r'Skulk', - 'smelt': r'Smelt', - 'snap': r'Snap', - 'snow': r'Snow', - 'spectacle': r'Spectacle', - 'splice': r'Splice', - 'spore': r'Spore', - 'sprawl': r'Sprawl', - 'stabilize': r'Stabilize', - 'stasis': r'Stasis', - 'storm': r'Storm', - 'story': r'Story', - 'substitute': r'Substitute', - 'sunder': r'Sunder', - 'surge': r'Surge', - 'survive': r'Survive', - 'swarm': r'Swarm', - 'symbiosis': r'Symbiosis', - 'synchronized': r'Synchronized', - 'synth': r'Synth', - 'table': r'Table', - 'taint': r'Taint', - 'tank': r'Tank', - 'thorn': r'Thorn', - 'thwart': r'Thwart', - 'time': r'Time', - 'tinker': r'Tinker', - 'toxin': r'Toxin', - 'trail': r'Trail', - 'transfigure': r'Transfigure', - 'transform': r'Transform', - 'transport': r'Transport', - 'trouble': r'Trouble', - 'tunnel': r'Tunnel', - 'unearth': r'Unearth', - 'unleash': r'Unleash', - 'unmask': r'Unmask', - 'unstoppable': r'Unstoppable', - 'urborg': r'Urborg', - 'urgent': r'Urgent', - 'utility': r'Utility', - 'vengeful': r'Vengeful', - 'vanish': r'Vanish', - 'venom': r'Venom', - 'victory': r'Victory', - 'villainous': r'Villainous', - 'vitalize': r'Vitalize', - 'void': r'Void', - 'voyage': r'Veoyage', - 'ward': r'Ward', - 'watch': r'Watch', - 'weave': r'Weave', - 'wed': r'Wed', - 'whammy': r'Whammy', - 'wild': r'Wild', - 'will': r'Will', - 'wisp': r'Wisp', - 'witch': r'Witch', - 'woe': r'Woe', - 'wounded': r'Wounded', - 'wrap': r'Wrap', - 'wrought': r'Wrought', - 'wurm': r'Wurm', - 'wythe': r'Wythe', - } - - # Define archetype patterns - self.archetype_patterns = { - 'goblin': r'Goblin', - 'elf': r'Elf', - 'vampire': r'Veampire', - 'angel': r'Angel', - 'dragon': r'Dragon', - 'human': r'Human', - 'zombie': r'Zombie', - 'soldier': r'Soldier', - 'knight': r'Knight', - 'wizard': r'Wizard', - 'spirit': r'Spirit', - 'demon': r'Demon', - 'snake': r'Snake', - 'cat': r'Cat', - 'wolf': r'Wolf', - 'bear': r'Bear', - 'bird': r'Bird', - 'insect': r'Insect', - 'horror': r'Horror', - 'goat': r'Goat', - 'ox': r'Ox', - 'elephant': r'Elephant', - 'whale': r'Whale', - 'shark': r'Shark', - 'fish': r'Fish', - 'serpent': r'Serpent', - 'lizard': r'Lizard', - 'scorpion': r'Scorpion', - 'spider': r'Spider', - 'rat': r'Rat', - 'drake': r'Drake', - 'wyvern': r'Wyvern', - 'phoenix': r'Phoenix', - 'lynx': r'Lynx', - 'jaguar': r'Jaguar', - 'hydra': r'Hydra', - 'leviathan': r'Leviathan', - 'kraken': r'Kraken', - 'cyclops': r'Cyclops', - 'golem': r'Golem', - 'homunculus': r'Homunculus', - 'clay': r'Clay', - 'construct': r'Construct', - 'myr': r'Myr', - 'aether': r'Aether', - 'pumpkin': r'Pumpkin', - 'pirate': r'Pirate', - 'pegasus': r'Pegasus', - 'unicorn': r'Unicorn', - 'centaur': r'Centaur', - 'merfolk': r'Merfolk', - 'mermaid': r'Mermaid', - 'naga': r'Naga', - 'satyr': r'Satyr', - 'dryad': r'Dryad', - 'treant': r'Treant', - 'elemental': r'Elemental', - 'fiend': r'Fiend', - 'imp': r'Imp', - 'faerie': r'Faerie', - 'minion': r'Minion', - 'abomination': r'Abomination', - 'beast': r'Beast', - 'demigod': r'Demigod', - 'god': r'God', - 'avatar': r'Avatar', - 'guardian': r'Guardian', - 'warrior': r'Warrior', - 'rogue': r'Rogue', - 'artificer': r'Artificer', - 'bard': r'Bard', - 'monk': r'Monk', - 'ninja': r'Ninja', - 'samurai': r'Samurai', - 'assassin': r'Assassin', - 'thief': r'Thief', - 'acrobat': r'Acrobat', - 'explorer': r'Explorer', - 'farmer': r'Farmer', - 'myth': r'Myth', - 'illusion': r'Illusion', - 'mirror': r'Mirror', - 'phantom': r'Phantom', - 'shapeshifter': r'Shapeshifter', - 'shaman': r'Shaman', - 'skeleton': r'Skeleton', - 'slime': r'Slime', - 'squirrel': r'Squirrel', - 'troll': r'Troll', - 'tyrannosaur': r'Tyrannosaur', - 'wraith': r'Wraith', - 'wurm': r'Wurm', - } - - # Target types for counter interactions - self.target_types = { - 'creature': r'creature', - 'artifact': r'artifact', - 'enchantment': r'enchantment', - 'instant': r'instant', - 'sorcery': r'sorcery', - 'planeswalker': r'planeswalker', - 'land': r'land', - 'player': r'player', - } - - # Trigger patterns - self.trigger_patterns = { - 'enters_battlefield': r'when [~|this] enters the battlefield', - 'leaves_battlefield': r'when [~|this] leaves the battlefield', - 'attacks': r'whenever [~|this] attacks', - 'blocks': r'whenever [~|this] blocks', - 'dies': r'when [~|this] dies', - 'damage': r'deals [0-9]+ damage', - 'draws_card': r'draw a card|draw two cards', - 'gains_life': r'gain [0-9]+ life', - 'creates_token': r'create a token', - 'taps': r'tap: add', - 'untaps': r'untap: add', - 'destroys': r'destroy target', - 'exiles': r'exile target', - 'counters_spell': r'counter target spell', - } - - # Effect patterns - self.effect_patterns = { - 'gain_flying': r'gain flying', - 'gain_first_strike': r'gain first strike', - 'gain_double_strike': r'gain double strike', - 'gain_deathtouch': r'gain deathtouch', - 'gain_lifelink': r'gain lifelink', - 'gain_haste': r'gain haste', - 'gain_trample': r'gain trample', - 'gain_vigilance': r'gain vigilance', - 'gain_indestructible': r'gain indestructible', - 'gain_hexproof': r'gain hexproof', - 'until_end_of_turn': r'until end of turn', - 'until_next_turn': r'until your next turn', - } - - def extract_mechanics(self, card: CardProfile) -> List[str]: - """Extract mechanics from card type line and oracle text.""" - mechanics = [] - - # Check type line for mechanics - if card.type_line: - for mechanic, pattern in self.mechanics_patterns.items(): - if re.search(pattern, card.type_line, re.IGNORECASE): - mechanics.append(mechanic) - - # Check oracle text for mechanics - if card.oracle_text: - for mechanic, pattern in self.mechanics_patterns.items(): - if re.search(pattern, card.oracle_text, re.IGNORECASE): - if mechanic not in mechanics: - mechanics.append(mechanic) - - return mechanics - - def extract_archetypes(self, card: CardProfile) -> List[str]: - """Extract archetypes from card subtypes.""" - archetypes = [] - - if card.subtypes: - for archetype, pattern in self.archetype_patterns.items(): - if re.search(pattern, card.subtypes, re.IGNORECASE): - archetypes.append(archetype) - - return archetypes - - def extract_targets(self, card: CardProfile) -> List[str]: - """Extract target types from oracle text.""" - targets = [] - - if card.oracle_text: - for target, pattern in self.target_types.items(): - if re.search(pattern, card.oracle_text, re.IGNORECASE): - targets.append(target) - - return targets - - def extract_triggers(self, card: CardProfile) -> List[str]: - """Extract trigger conditions from oracle text.""" - triggers = [] - - if card.oracle_text: - for trigger, pattern in self.trigger_patterns.items(): - if re.search(pattern, card.oracle_text, re.IGNORECASE): - triggers.append(trigger) - - return triggers - - def extract_effects(self, card: CardProfile) -> List[str]: - """Extract game effects from oracle text.""" - effects = [] - - if card.oracle_text: - for effect, pattern in self.effect_patterns.items(): - if re.search(pattern, card.oracle_text, re.IGNORECASE): - effects.append(effect) - - return effects - - def extract_themes(self, card: CardProfile) -> List[str]: - """Extract set themes based on card characteristics.""" - themes = [] - - # Storm theme - if 'storm' in card.mechanics or 'storm' in card.oracle_text.lower(): - themes.append('storm') - - # Token theme - if any(e in card.effects for e in ['creates_token']): - themes.append('tokens') - - # Mill theme - if any(t in card.triggers for t in ['draws_card']): - themes.append('draw') - - # Life gain theme - if any(e in card.effects for e in ['gains_life']): - themes.append('life_gain') - - # Board wipe theme - if any(t in card.triggers for t in ['dies']): - themes.append('board_wipe') - - # Reanimate theme - if any(t in card.triggers for t in ['leaves_battlefield']): - themes.append('reanimate') - - # Countermagic theme - if any(e in card.effects for e in ['counters_spell']): - themes.append('countermagic') - - # Card advantage theme - if any(t in card.triggers for t in ['draws_card']): - themes.append('card_advantage') - - # Mana acceleration theme - if any(t in card.triggers for t in ['taps', 'untaps']): - themes.append('mana_acceleration') - - # Combat tricks theme - if any(e in card.effects for e in ['gain_flying', 'gain_first_strike', - 'gain_double_strike', 'gain_deathtouch', - 'gain_lifelink', 'gain_vigilance']): - themes.append('combat_tricks') - - # ETB effects theme - if any(t in card.triggers for t in ['enters_battlefield']): - themes.append('etb_effects') - - # LTB effects theme - if any(t in card.triggers for t in ['leaves_battlefield']): - themes.append('ltb_effects') - - return themes - - def profile_card(self, card_data: Dict[str, Any]) -> CardProfile: - """Convert raw MTGJSON card data to CardProfile.""" - # Parse subtypes - subtypes = None - if card_data.get('subtypes'): - subtypes = ', '.join(card_data['subtypes']) - - # Parse supertypes - supertypes = None - if card_data.get('supertypes'): - supertypes = ', '.join(card_data['supertypes']) - - # Parse colors - colors = None - if card_data.get('colors'): - colors = ', '.join(card_data['colors']) - - # Parse color identity - color_identity = None - if card_data.get('colorIdentity'): - color_identity = ', '.join(card_data['colorIdentity']) - - # Extract interactions - profile = CardProfile( - name=card_data.get('name', ''), - mana_cost=card_data.get('manaCost'), - type_line=card_data.get('typeLine'), - oracle_text=card_data.get('oracleText'), - subtypes=subtypes, - supertypes=supertypes, - colors=colors, - color_identity=color_identity, - power=card_data.get('power'), - toughness=card_data.get('toughness'), - loyalty=card_data.get('loyalty'), - set_code=card_data.get('set', {}).get('code') if card_data.get('set') else None, - set_id=card_data.get('setId', 0), - card_id=card_data.get('id', 0), - ) - - # Extract mechanics, archetypes, etc. - profile.mechanics = self.extract_mechanics(profile) - profile.archetypes = self.extract_archetypes(profile) - profile.targets = self.extract_targets(profile) - profile.triggers = self.extract_triggers(profile) - profile.effects = self.extract_effects(profile) - profile.themes = self.extract_themes(profile) - - return profile - - def find_synergies(self, card_a: CardProfile, card_b: CardProfile) -> List[Tuple[str, int, str]]: - """ - Find synergies between two cards. - - Returns list of (synergy_type, strength, notes) tuples. - """ - synergies = [] - - # Same archetype synergy - if card_a.archetypes and card_b.archetypes: - common_archetypes = set(card_a.archetypes) & set(card_b.archetypes) - if common_archetypes: - synergies.append(( - 'archetype_support', - 3, - f"Both are {', '.join(common_archetypes)}" - )) - - # Mechanic support - if card_a.mechanics and card_b.mechanics: - # If card_b has a mechanic that supports card_a's archetype - for mech in card_a.mechanics: - if mech in card_b.mechanics: - synergies.append(( - 'mechanic_support', - 2, - f"Both have {mech}" - )) - - # Mana base synergy - if card_a.colors and card_b.colors: - # Check for color compatibility - colors_a = set(card_a.colors.split(',')) - colors_b = set(card_b.colors.split(',')) - - if colors_a == colors_b: - synergies.append(( - 'mana_base', - 4, - "Same color identity" - )) - - # Combo partner - if card_a.targets and card_b.triggers: - # If card_a targets creatures and card_b triggers on creatures - if 'creature' in card_a.targets and any(t in card_b.triggers for t in ['enters_battlefield', 'dies']): - synergies.append(( - 'combo_partner', - 3, - "Card A targets creatures, Card B interacts with creature entry/death" - )) - - # Counter partner - if card_a.targets and card_b.targets: - # If they target different types, they complement each other - targets_a = set(card_a.targets) - targets_b = set(card_b.targets) - - if targets_a != targets_b and targets_a & targets_b: - synergies.append(( - 'counter_partner', - 2, - "Different target types provide coverage" - )) - - # Evolution chain - if card_a.name == card_b.name: - synergies.append(( - 'evolution_chain', - 2, - "Same card name (reprint or different version)" - )) - - return synergies - - def find_counters(self, card_a: CardProfile, card_b: CardProfile) -> List[Tuple[str, int, str]]: - """ - Find counter relationships between two cards. - - Returns list of (counter_type, strength, notes) tuples. - """ - counters = [] - - # Different color identities - if card_a.color_identity and card_b.color_identity: - colors_a = set(card_a.color_identity.split(',')) - colors_b = set(card_b.color_identity.split(',')) - - if colors_a != colors_b: - counters.append(( - 'mana_disadvantage', - 2, - "Different color identities create strategic tension" - )) - - # Outclass - if card_a.power and card_b.power: - try: - power_a = int(card_a.power) - power_b = int(card_b.power) - - if power_a > power_b + 1: - counters.append(( - 'outclass', - 3, - f"Card A has higher power ({power_a} vs {power_b})" - )) - elif power_b > power_a + 1: - counters.append(( - 'outclass', - 3, - f"Card B has higher power ({power_b} vs {power_a})" - )) - except (ValueError, TypeError): - pass - - # Counter role - if card_a.targets and 'creature' in card_a.targets: - if card_b.mechanics and any(m in card_b.mechanics for m in ['deathtouch', 'trample']): - counters.append(( - 'counter_role', - 2, - "Card A targets creatures, Card B has combat keywords" - )) - - return counters - - def find_evolution(self, card: CardProfile, all_cards: Dict[int, CardProfile]) -> List[Tuple[str, int, str]]: - """ - Find evolution relationships for a card. - - Returns list of (evolution_type, strength, notes) tuples. - """ - evolutions = [] - - # Find reprints - for other_id, other_card in all_cards.items(): - if other_id != card.card_id and card.name == other_card.name: - evolutions.append(( - 'reprinted', - 2, - f"Reprint in {other_card.set_code} (set_id: {other_card.set_id})" - )) - - # Find transform pairs (same name, different face) - # This would require checking card_faces in the database - - return evolutions - - def build_interaction_graph(self, cards: List[Dict[str, Any]]) -> Dict[str, Any]: - """ - Build interaction graph for a batch of cards. - - Returns dictionary with: - - mechanics: card_id -> mechanics list - - archetypes: card_id -> archetypes list - - synergies: (card_a, card_b) -> list of synergies - - counters: (card_a, card_b) -> list of counters - - evolutions: card_id -> list of evolutions - """ - # Profile all cards - profiles = {} - for card_data in cards: - if card_data.get('id'): - profile = self.profile_card(card_data) - profiles[profile.card_id] = profile - - # Extract interactions - graph = { - 'mechanics': {}, - 'archetypes': {}, - 'synergies': [], - 'counters': [], - 'evolutions': [], - } - - # Extract mechanics and archetypes - for card_id, profile in profiles.items(): - graph['mechanics'][card_id] = profile.mechanics - graph['archetypes'][card_id] = profile.archetypes - - # Find synergies between all card pairs - card_ids = list(profiles.keys()) - for i in range(len(card_ids)): - for j in range(i + 1, len(card_ids)): - card_a = profiles[card_ids[i]] - card_b = profiles[card_ids[j]] - - synergies = self.find_synergies(card_a, card_b) - if synergies: - graph['synergies'].append({ - 'card_a': card_a.card_id, - 'card_b': card_b.card_id, - 'synergies': synergies, - }) - - # Find counters between all card pairs - for i in range(len(card_ids)): - for j in range(i + 1, len(card_ids)): - card_a = profiles[card_ids[i]] - card_b = profiles[card_ids[j]] - - counters = self.find_counters(card_a, card_b) - if counters: - graph['counters'].append({ - 'card_a': card_a.card_id, - 'card_b': card_b.card_id, - 'counters': counters, - }) - - # Find evolutions for each card - for card_id, profile in profiles.items(): - evolutions = self.find_evolution(profile, profiles) - if evolutions: - graph['evolutions'].append({ - 'card_id': card_id, - 'evolutions': evolutions, - }) - - return graph - - -def main(): - """Test the rule engine with sample data.""" - engine = MTGRuleEngine() - - # Sample card data - sample_cards = [ - { - 'id': 1, - 'name': 'Lightning Bolt', - 'manaCost': '{R}', - 'typeLine': 'Instant', - 'oracleText': 'Lightning Bolt deals 3 damage to any target.', - 'subtypes': [], - 'supertypes': [], - 'colors': ['R'], - 'colorIdentity': ['R'], - 'set': {'code': '2X2'}, - 'setId': 100, - }, - { - 'id': 2, - 'name': 'Lightning Greaves', - 'manaCost': '{1}{R}', - 'typeLine': 'Artifact — Equipment', - 'oracleText': 'Enchanted creature has hexproof and haste.\nEquip {1}', - 'subtypes': ['Equipment'], - 'supertypes': [], - 'colors': ['R'], - 'colorIdentity': ['R'], - 'power': None, - 'toughness': None, - 'set': {'code': '10E'}, - 'setId': 200, - }, - { - 'id': 3, - 'name': 'Elvish Archers', - 'manaCost': '{G}', - 'typeLine': 'Creature — Elf Ranger', - 'oracleText': 'Elvish Archers can\'t be blocked by creatures with power 2 or less.\n{T}: Target creature gets -1/-1 until end of turn.', - 'subtypes': ['Elf', 'Ranger'], - 'supertypes': [], - 'colors': ['G'], - 'colorIdentity': ['G'], - 'power': '1', - 'toughness': '1', - 'set': {'code': '5DN'}, - 'setId': 300, - }, - { - 'id': 4, - 'name': 'Swords to Plowshares', - 'manaCost': '{W}', - 'typeLine': 'Enchantment', - 'oracleText': 'Exile target creature. Its controller gains 1 life.', - 'subtypes': [], - 'supertypes': [], - 'colors': ['W'], - 'colorIdentity': ['W'], - 'set': {'code': '2X2'}, - 'setId': 100, - }, - ] - - # Build interaction graph - graph = engine.build_interaction_graph(sample_cards) - - # Print results - print("=" * 60) - print("MTG Card Interaction Graph") - print("=" * 60) - - print("\n📊 Mechanics:") - for card_id, mechanics in graph['mechanics'].items(): - print(f" Card {card_id}: {mechanics}") - - print("\n📊 Archetypes:") - for card_id, archetypes in graph['archetypes'].items(): - print(f" Card {card_id}: {archetypes}") - - print("\n🔗 Synergies:") - for synergy in graph['synergies']: - print(f" Cards {synergy['card_a']} ↔ {synergy['card_b']}:") - for syn_type, strength, notes in synergy['synergies']: - print(f" - {syn_type} (strength: {strength}): {notes}") - - print("\n⚔️ Counters:") - for counter in graph['counters']: - print(f" Cards {counter['card_a']} ↔ {counter['card_b']}:") - for counter_type, strength, notes in counter['counters']: - print(f" - {counter_type} (strength: {strength}): {notes}") - - print("\n🔄 Evolutions:") - for evolution in graph['evolutions']: - print(f" Card {evolution['card_id']}:") - for evol_type, strength, notes in evolution['evolutions']: - print(f" - {evol_type} (strength: {strength}): {notes}") - - print("\n" + "=" * 60) - print("✅ Interaction graph built successfully!") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/card_profile_extractor.py b/backend/scripts/card_profile_extractor.py deleted file mode 100644 index 004b9e4..0000000 --- a/backend/scripts/card_profile_extractor.py +++ /dev/null @@ -1,671 +0,0 @@ -""" -MTG Card Profile Extractor - -Extracts structured profiles from MTGJSON card data. -Identifies mechanics, archetypes, mana costs, targets, and other game-relevant attributes. -""" -import re -from typing import List, Dict, Optional, Set -from dataclasses import dataclass - - -@dataclass -class CardProfile: - """ - Structured profile of a card for interaction analysis. - - Contains all relevant game attributes extracted from MTGJSON data. - """ - # Basic info - id: int - name: str - mana_cost: Optional[str] - type_line: Optional[str] - oracle_text: Optional[str] - subtypes: Optional[str] - supertypes: Optional[str] - set_code: Optional[str] - - # Extracted attributes - colors: List[str] = None # ['W', 'U', 'B', 'R', 'G'] - color_identity: List[str] = None - mechanics: List[str] = None - archetypes: List[str] = None - targets: List[str] = None # ['creature', 'artifact', 'player', etc.] - triggers: List[str] = None - effects: List[str] = None - themes: List[str] = None # ['storm', 'tokens', 'draw', etc.] - - def __post_init__(self): - """Initialize lists if None.""" - if self.colors is None: - self.colors = [] - if self.color_identity is None: - self.color_identity = [] - if self.mechanics is None: - self.mechanics = [] - if self.archetypes is None: - self.archetypes = [] - if self.targets is None: - self.targets = [] - if self.triggers is None: - self.triggers = [] - if self.effects is None: - self.effects = [] - if self.themes is None: - self.themes = [] - - def to_dict(self) -> Dict: - """Convert profile to dictionary.""" - return { - 'id': self.id, - 'name': self.name, - 'mana_cost': self.mana_cost, - 'type_line': self.type_line, - 'oracle_text': self.oracle_text, - 'subtypes': self.subtypes, - 'supertypes': self.supertypes, - 'set_code': self.set_code, - 'colors': self.colors, - 'color_identity': self.color_identity, - 'mechanics': self.mechanics, - 'archetypes': self.archetypes, - 'targets': self.targets, - 'triggers': self.triggers, - 'effects': self.effects, - 'themes': self.themes, - } - - -class CardProfileExtractor: - """ - Extracts card profiles from MTGJSON data. - - Uses regex patterns and curated dictionaries to identify: - - Mana costs and color identity - - Game mechanics (flying, first strike, etc.) - - Archetypes (goblin, elf, vampire, etc.) - - Targets (creature, artifact, player, etc.) - - Triggers (enters battlefield, dies, attacks, etc.) - - Effects (gain flying, draw card, etc.) - - Themes (storm, tokens, mill, etc.) - """ - - # Color symbols in mana costs - COLOR_SYMBOLS = { - '{W}': 'W', - '{U}': 'U', - '{B}': 'B', - '{R}': 'R', - '{G}': 'G', - } - - # Known mechanics and their patterns - MECHANICS = { - 'flying': r'Flying', - 'first_strike': r'First strike', - 'double_strike': r'Double strike', - 'deathtouch': r'Death touch', - 'lifelink': r'Lifelink', - 'haste': r'Haste', - 'trample': r'Trample', - 'menace': r'Menace', - 'vigilance': r'Vegilance', - 'reach': r'Reach', - 'indestructible': r'Indestructible', - 'hexproof': r'Hexproof', - 'shroud': r'Shroud', - 'defender': r'Defender', - 'landfall': r'Landfall', - 'delve': r'Delve', - 'suspend': r'Suspend', - 'convoke': r'Convoke', - 'rampage': r'Rampage', - 'toxic': r'Toxic', - 'crew': r'Crew', - 'equip': r'Equip', - 'annihilator': r'Annihilator', - 'spectacle': r'Spectacle', - 'prowess': r'Prowess', - 'aftermath': r'Aftermath', - 'adapt': r'Adapt', - 'amplify': r'Amplify', - 'awaken': r'Awaken', - 'kicker': r'Kicker', - 'morph': r'Morph', - 'evolve': r'Evolve', - 'exalted': r'Exalted', - 'storm': r'Storm', - 'madness': r'Madness', - 'manifest': r'Manifest', - 'modular': r'Modular', - 'mutate': r'Mutate', - 'transform': r'Transform', - 'unearth': r'Unearth', - 'persist': r'Persist', - 'rebound': r'Rebound', - 'replicate': r'Replicate', - 'soulshift': r'Soulshift', - 'dredge': r'Dredge', - 'devour': r'Devour', - 'banding': r'Band with', - 'bestow': r'Bestow', - 'channel': r'Channel', - 'clash': r'Clash', - 'curse': r'Curse', - 'dwell': r'Dwell', - 'evoke': r'Evoke', - 'exploit': r'Exploit', - 'extort': r'Extort', - 'flash': r'Flash', - 'foretell': r'Foretell', - 'frenzy': r'Frenzy', - 'grudge': r'Grudge', - 'heroic': r'Heroic', - 'hideaway': r'Hideaway', - 'horrify': r'Horrify', - 'impetus': r'Impetus', - 'infect': r'Infect', - 'journey': r'Journey', - 'kicker': r'Kicker', - 'landfall': r'Landfall', - 'meld': r'Meld', - 'miracle': r'Miracle', - 'monstrosity': r'Monstrosity', - 'morph': r'Morph', - 'mutate': r'Mutate', - 'ninja': r'Ninja', - 'pact': r'Pact', - 'persist': r'Persist', - 'provoke': r'Provoke', - 'quest': r'Quest', - 'raid': r'Raid', - 'rebound': r'Rebound', - 'replicate': r'Replicate', - 'revolt': r'Revolt', - 'shroud': r'Shroud', - 'skulk': r'Skulk', - 'snow': r'Snow', - 'splice': r'Splice', - 'staunch': r'Staunch', - 'storm': r'Storm', - 'suspend': r'Suspend', - 'surge': r'Surge', - 'swarm': r'Swarm', - 'thorn': r'Thorn', - 'toxic': r'Toxic', - 'transfigure': r'Transfigure', - 'transform': r'Transform', - 'unearth': r'Unearth', - 'unleash': r'Unleash', - 'vampiric': r'Vampiric', - 'ward': r'Ward', - 'willow': r'Willow', - 'winter': r'Winter', - 'wither': r'Wither', - 'wurm': r'Wurm', - } - - # Known archetypes and their patterns - ARCHETYPES = { - 'goblin': r'Goblin', - 'elf': r'Elf', - 'vampire': r'Veampire', - 'angel': r'Angel', - 'dragon': r'Dragon', - 'human': r'Human', - 'zombie': r'Zombie', - 'soldier': r'Soldier', - 'knight': r'Knight', - 'wizard': r'Wizard', - 'spirit': r'Spirit', - 'demon': r'Demon', - 'snake': r'Snake', - 'cat': r'Cat', - 'wolf': r'Wolf', - 'bear': r'Bear', - 'bird': r'Bird', - 'insect': r'Insect', - 'horror': r'Horror', - 'goat': r'Goat', - 'ox': r'Ox', - 'elephant': r'Elephant', - 'whale': r'Whale', - 'shark': r'Shark', - 'fish': r'Fish', - 'serpent': r'Serpent', - 'lizard': r'Lizard', - 'scorpion': r'Scorpion', - 'spider': r'Spider', - 'rat': r'Rat', - 'drake': r'Drake', - 'wyvern': r'Wyvern', - 'phoenix': r'Phoenix', - 'lynx': r'Lynx', - 'jaguar': r'Jaguar', - 'hydra': r'Hydra', - 'leviathan': r'Leviathan', - 'kraken': r'Kraken', - 'cyclops': r'Cyclops', - 'golem': r'Golem', - 'homunculus': r'Homunculus', - 'clay': r'Clay', - 'construct': r'Construct', - 'myr': r'Myr', - 'pirate': r'Pirate', - 'pegasus': r'Pegasus', - 'unicorn': r'Unicorn', - 'centaur': r'Centaur', - 'merfolk': r'Merfolk', - 'mermaid': r'Mermaid', - 'naga': r'Naga', - 'satyr': r'Satyr', - 'dryad': r'Dryad', - 'treant': r'Treant', - 'elemental': r'Elemental', - 'fiend': r'Fiend', - 'imp': r'Imp', - 'faerie': r'Faerie', - 'minion': r'Minion', - 'abomination': r'Abomination', - 'beast': r'Beast', - 'demigod': r'Demigod', - 'god': r'God', - 'avatar': r'Avatar', - 'guardian': r'Guardian', - 'warrior': r'Warrior', - 'rogue': r'Rogue', - 'artificer': r'Artificer', - 'bard': r'Bard', - 'monk': r'Monk', - 'ninja': r'Ninja', - 'samurai': r'Samurai', - 'assassin': r'Assassin', - 'thief': r'Thief', - 'acrobat': r'Acrobat', - 'explorer': r'Explorer', - 'myth': r'Myth', - 'illusion': r'Illusion', - 'mirror': r'Mirror', - 'phantom': r'Phantom', - 'shapeshifter': r'Shapeshifter', - 'shaman': r'Shaman', - 'skeleton': r'Skeleton', - 'slime': r'Slime', - 'squirrel': r'Squirrel', - 'troll': r'Troll', - 'tyrannosaur': r'Tyrannosaur', - 'wraith': r'Wraith', - 'wurm': r'Wurm', - } - - # Target types and their patterns - TARGET_TYPES = { - 'creature': r'creature', - 'artifact': r'artifact', - 'enchantment': r'enchantment', - 'instant': r'instant', - 'sorcery': r'sorcery', - 'planeswalker': r'planeswalker', - 'land': r'land', - 'player': r'player', - 'spell': r'spell', - 'permanent': r'permanent', - 'creature card': r'creature [Cc]ard', - } - - # Trigger conditions and their patterns - TRIGGERS = { - 'enters_battlefield': r'when [~|this] enters the battlefield', - 'leaves_battlefield': r'when [~|this] leaves the battlefield', - 'attacks': r'whenever [~|this] attacks', - 'blocks': r'whenever [~|this] blocks', - 'dies': r'when [~|this] dies', - 'damage': r'deals [0-9]+ damage', - 'draws_card': r'draw a card|draw two cards', - 'gains_life': r'gain [0-9]+ life', - 'creates_token': r'create a token', - 'taps': r'tap: add', - 'untaps': r'untap: add', - 'destroys': r'destroy target', - 'exiles': r'exile target', - 'counters_spell': r'counter target spell', - } - - # Game effects and their patterns - EFFECTS = { - 'gain_flying': r'gain flying', - 'gain_first_strike': r'gain first strike', - 'gain_double_strike': r'gain double strike', - 'gain_deathtouch': r'gain deathtouch', - 'gain_lifelink': r'gain lifelink', - 'gain_haste': r'gain haste', - 'gain_trample': r'gain trample', - 'gain_vigilance': r'gain vigilance', - 'gain_indestructible': r'gain indestructible', - 'gain_hexproof': r'gain hexproof', - 'until_end_of_turn': r'until end of turn', - 'until_next_turn': r'until your next turn', - 'deal_damage': r'deal [0-9]+ damage', - 'gain_life': r'gain [0-9]+ life', - 'draw_card': r'draw [0-9]+ card', - 'create_token': r'create [0-9]+ token', - 'destroy': r'destroy target', - 'exile': r'exile target', - 'counter_spell': r'counter target spell', - } - - # Theme keywords and their patterns - THEMES = { - 'storm': r'Storm', - 'tokens': r'create a token', - 'draw': r'draw a card', - 'life_gain': r'gain life', - 'board_wipe': r'destroy all', - 'reanimate': r'put from grave', - 'countermagic': r'counter target spell', - 'card_advantage': r'draw', - 'mana_acceleration': r'tap: add', - 'combat_tricks': r'gain [A-Za-z]+ until end of turn', - 'etb_effects': r'enters the battlefield', - 'ltb_effects': r'leaves the battlefield', - 'mill': r'put on bottom of library', - 'draw_go': r'draw a card', - 'aggro': r'deal [0-9]+ damage', - 'control': r'counter target spell', - 'midrange': r'creature', - } - - def __init__(self): - """Initialize the profile extractor.""" - pass - - def extract_colors(self, mana_cost: Optional[str]) -> List[str]: - """ - Extract colors from mana cost. - - Args: - mana_cost: Mana cost string (e.g., '{1}{R}') - - Returns: - List of color symbols (e.g., ['R']) - """ - if not mana_cost: - return [] - - colors = [] - for symbol, color in self.COLOR_SYMBOLS.items(): - if symbol in mana_cost: - if color not in colors: - colors.append(color) - - return colors - - def extract_mechanics(self, card_type_line: Optional[str], card_oracle: Optional[str]) -> List[str]: - """ - Extract game mechanics from card text. - - Args: - card_type_line: Card type line (e.g., 'Creature - Goblin Warrior') - card_oracle: Card oracle text - - Returns: - List of mechanic names (e.g., ['haste', 'trample']) - """ - mechanics = [] - - # Combine type line and oracle text for checking - text_to_check = f"{card_type_line or ''} {card_oracle or ''}".upper() - - for mechanic, pattern in self.MECHANICS.items(): - if re.search(pattern, text_to_check, re.IGNORECASE): - if mechanic not in mechanics: - mechanics.append(mechanic) - - return mechanics - - def extract_archetypes(self, card_subtypes: Optional[str]) -> List[str]: - """ - Extract archetypes from card subtypes. - - Args: - card_subtypes: Card subtypes (e.g., 'Goblin, Warrior') - - Returns: - List of archetype names (e.g., ['goblin']) - """ - if not card_subtypes: - return [] - - archetypes = [] - - for archetype, pattern in self.ARCHETYPES.items(): - if re.search(pattern, card_subtypes, re.IGNORECASE): - if archetype not in archetypes: - archetypes.append(archetype) - - return archetypes - - def extract_targets(self, card_oracle: Optional[str]) -> List[str]: - """ - Extract target types from oracle text. - - Args: - card_oracle: Card oracle text - - Returns: - List of target types (e.g., ['creature', 'player']) - """ - if not card_oracle: - return [] - - targets = [] - - for target, pattern in self.TARGET_TYPES.items(): - if re.search(pattern, card_oracle, re.IGNORECASE): - if target not in targets: - targets.append(target) - - return targets - - def extract_triggers(self, card_oracle: Optional[str]) -> List[str]: - """ - Extract trigger conditions from oracle text. - - Args: - card_oracle: Card oracle text - - Returns: - List of trigger names (e.g., ['enters_battlefield', 'dies']) - """ - if not card_oracle: - return [] - - triggers = [] - - for trigger, pattern in self.TRIGGERS.items(): - if re.search(pattern, card_oracle, re.IGNORECASE): - if trigger not in triggers: - triggers.append(trigger) - - return triggers - - def extract_effects(self, card_oracle: Optional[str]) -> List[str]: - """ - Extract game effects from oracle text. - - Args: - card_oracle: Card oracle text - - Returns: - List of effect names (e.g., ['gain_flying', 'draw_card']) - """ - if not card_oracle: - return [] - - effects = [] - - for effect, pattern in self.EFFECTS.items(): - if re.search(pattern, card_oracle, re.IGNORECASE): - if effect not in effects: - effects.append(effect) - - return effects - - def extract_themes(self, card_mechanics: List[str], card_triggers: List[str], - card_effects: List[str], card_targets: List[str]) -> List[str]: - """ - Extract card themes based on characteristics. - - Args: - card_mechanics: List of mechanics - card_triggers: List of triggers - card_effects: List of effects - card_targets: List of targets - - Returns: - List of theme names (e.g., ['storm', 'tokens']) - """ - themes = [] - - # Storm theme - if 'storm' in card_mechanics or 'storm' in card_targets: - themes.append('storm') - - # Token theme - if any(e in card_effects for e in ['create_token', 'draw_card']): - themes.append('tokens') - - # Mill theme - if any(t in card_triggers for t in ['draws_card']): - themes.append('mill') - - # Life gain theme - if any(e in card_effects for e in ['gain_life', 'draw_card']): - themes.append('life_gain') - - # Board wipe theme - if any(t in card_triggers for t in ['dies']): - themes.append('board_wipe') - - # Reanimate theme - if any(t in card_triggers for t in ['leaves_battlefield']): - themes.append('reanimate') - - # Countermagic theme - if any(e in card_effects for e in ['counter_spell']): - themes.append('countermagic') - - # Card advantage theme - if any(t in card_triggers for t in ['draws_card']): - themes.append('card_advantage') - - # Mana acceleration theme - if any(t in card_triggers for t in ['taps']): - themes.append('mana_acceleration') - - # Combat tricks theme - if any(e in card_effects for e in ['gain_flying', 'gain_first_strike', - 'gain_double_strike', 'gain_deathtouch', - 'gain_lifelink', 'gain_vigilance']): - themes.append('combat_tricks') - - # ETB effects theme - if any(t in card_triggers for t in ['enters_battlefield']): - themes.append('etb_effects') - - # LTB effects theme - if any(t in card_triggers for t in ['leaves_battlefield']): - themes.append('ltb_effects') - - # Aggro theme - if any(e in card_effects for e in ['deal_damage']): - themes.append('aggro') - - # Control theme - if any(e in card_effects for e in ['counter_spell']): - themes.append('control') - - # Midrange theme - if any(t in card_targets for t in ['creature']): - themes.append('midrange') - - return themes - - def extract_profile(self, card_data: Dict) -> CardProfile: - """ - Extract a complete card profile from MTGJSON data. - - Args: - card_data: MTGJSON card dictionary - - Returns: - CardProfile object with all extracted attributes - """ - # Parse subtypes - subtypes = None - if card_data.get('subtypes'): - subtypes = ', '.join(card_data['subtypes']) - - # Parse supertypes - supertypes = None - if card_data.get('supertypes'): - supertypes = ', '.join(card_data['supertypes']) - - # Extract colors from mana cost - colors = self.extract_colors(card_data.get('manaCost')) - - # Extract mechanics - mechanics = self.extract_mechanics( - card_data.get('typeLine'), - card_data.get('oracleText') - ) - - # Extract archetypes - archetypes = self.extract_archetypes(subtypes) - - # Extract targets - targets = self.extract_targets(card_data.get('oracleText')) - - # Extract triggers - triggers = self.extract_triggers(card_data.get('oracleText')) - - # Extract effects - effects = self.extract_effects(card_data.get('oracleText')) - - # Extract themes - themes = self.extract_themes(mechanics, triggers, effects, targets) - - # Create profile - profile = CardProfile( - id=card_data.get('id', 0), - name=card_data.get('name', ''), - mana_cost=card_data.get('manaCost'), - type_line=card_data.get('typeLine'), - oracle_text=card_data.get('oracleText'), - subtypes=subtypes, - supertypes=supertypes, - set_code=card_data.get('set', {}).get('code') if card_data.get('set') else None, - colors=colors, - color_identity=card_data.get('colorIdentity'), - mechanics=mechanics, - archetypes=archetypes, - targets=targets, - triggers=triggers, - effects=effects, - themes=themes, - ) - - return profile - - def extract_profiles_batch(self, cards: List[Dict]) -> List[CardProfile]: - """ - Extract profiles for a batch of cards. - - Args: - cards: List of MTGJSON card dictionaries - - Returns: - List of CardProfile objects - """ - return [self.extract_profile(card) for card in cards] diff --git a/backend/scripts/check_mtgjson_full.py b/backend/scripts/check_mtgjson_full.py deleted file mode 100644 index 849fa10..0000000 --- a/backend/scripts/check_mtgjson_full.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Check MTGJSON data status in filesystem and database.""" - -import asyncio -from pathlib import Path -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -import sys - -sys.path.append("/app") - -from app.services.mtgjson_manager import MTGJSONManager -from app.config import get_settings - -async def check_mtgjson_status(): - """Comprehensive check of MTGJSON data status.""" - - print("=" * 60) - print("MTGJSON DATA STATUS REPORT") - print("=" * 60) - - settings = get_settings() - - # 1. Check data directory - print("\n1. DATA DIRECTORY CHECK") - print("-" * 40) - data_dir = Path(settings.DATA_DIR) - print(f"Data Directory: {data_dir}") - print(f"Directory exists: {data_dir.exists()}") - - if data_dir.exists(): - files = list(data_dir.glob("*.json.gz")) + list(data_dir.glob("*.json")) - print(f"MTGJSON files found: {len(files)}") - - # Check specific files - required_files = [ - "AllPrintings.json.gz", - "AllSetFiles.json.gz", - "AllIdentifiers.json.gz", - "CardTypes.json.gz", - "Keywords.json.gz", - "MagicRoots.json.gz", - "MagicSets.json.gz", - "SetTranslations.json.gz" - ] - - missing_files = [] - existing_files = [] - - for f in required_files: - filepath = data_dir / f - if filepath.exists(): - size_mb = filepath.stat().st_size / (1024 * 1024) - existing_files.append((f, size_mb)) - print(f" ✓ {f}: {size_mb:.1f} MB") - else: - missing_files.append(f) - print(f" ✗ {f}: MISSING") - - print(f"\n Summary: {len(existing_files)}/{len(required_files)} required files present") - if missing_files: - print(f" Missing: {', '.join(missing_files)}") - else: - print(" ERROR: Data directory does not exist!") - - # 2. Check database status - print("\n2. DATABASE STATUS CHECK") - print("-" * 40) - - db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" - db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" - - try: - engine = create_async_engine(db_url) - - async with AsyncSession(engine) as session: - # Check tables - result = await session.execute(text(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - ORDER BY table_name; - """)) - - tables = [row[0] for row in result.fetchall()] - print(f"Tables found: {len(tables)}") - for table in tables: - print(f" - {table}") - - # Check key tables - print("\nKey table statistics:") - key_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords', 'mtg_refresh_log'] - - for table in key_tables: - if table in tables: - result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) - count = result.scalar() - print(f" {table}: {count:,} records") - else: - print(f" {table}: TABLE NOT FOUND") - - # Check refresh log - if 'mtg_refresh_log' in tables: - result = await session.execute(text(""" - SELECT refresh_type, status, created_at, error_message - FROM mtg_refresh_log - ORDER BY created_at DESC - LIMIT 5; - """)) - - rows = result.fetchall() - if rows: - print("\nRecent refresh operations:") - for row in rows: - status_icon = "✓" if row[1] == 'SUCCESS' else "✗" - print(f" {status_icon} {row[0]}: {row[1]} at {row[2]}") - if row[3]: - print(f" Error: {row[3]}") - else: - print("\nNo refresh operations logged") - - except Exception as e: - print(f"ERROR: Could not connect to database: {e}") - return - - # 3. Check MTGJSON manager status - print("\n3. MTGJSON MANAGER STATUS") - print("-" * 40) - - try: - manager = MTGJSONManager() - status = manager.get_status() - - print(f"Status: {status['status']}") - if status.get('data'): - print(f" Sets count: {status['data'].get('sets_count', 0)}") - print(f" Cards count: {status['data'].get('cards_count', 0)}") - print(f" Last refresh: {status['data'].get('last_refresh')}") - except Exception as e: - print(f"ERROR: Could not create MTGJSON manager: {e}") - - print("\n" + "=" * 60) - -if __name__ == "__main__": - asyncio.run(check_mtgjson_status()) diff --git a/backend/scripts/check_mtgjson_status.py b/backend/scripts/check_mtgjson_status.py deleted file mode 100644 index 9e889ce..0000000 --- a/backend/scripts/check_mtgjson_status.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -"""Check MTGJSON data status in the database and filesystem.""" - -import asyncio -import os -from pathlib import Path -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -import sys -sys.path.append("/home/wall-o/projects/mtgonline/backend") - -from app.services.mtgjson_manager import MTGJSONManager -from app.config import get_settings - -async def check_mtgjson_status(): - """Check MTGJSON data download and database status.""" - - print("=== MTGJSON Data Status Check ===\n") - - # Check data directory - settings = get_settings() - data_dir = Path(settings.DATA_DIR) - - print(f"Data Directory: {data_dir}") - print(f"Directory exists: {data_dir.exists()}") - - if data_dir.exists(): - files = list(data_dir.glob("*.json.gz")) - files += list(data_dir.glob("*.json")) - print(f"Found {len(files)} MTGJSON files:") - for f in sorted(files)[:20]: # Show first 20 - size_mb = f.stat().st_size / (1024 * 1024) - print(f" - {f.name} ({size_mb:.1f} MB)") - if len(files) > 20: - print(f" ... and {len(files) - 20} more files") - else: - print("WARNING: Data directory does not exist!") - - # Check database status - print("\n=== Database Status ===") - - try: - # Try to connect to the MTGJSON database - engine = create_async_engine( - f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" - f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" - ) - - async with AsyncSession(engine) as session: - # Check if tables exist - result = await session.execute(text(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - ORDER BY table_name; - """)) - - tables = [row[0] for row in result.fetchall()] - print(f"Found {len(tables)} tables in database:") - for table in tables: - print(f" - {table}") - - # Check specific MTGJSON tables - mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] - if 'mtg_refresh_log' in tables: - result = await session.execute(text("SELECT COUNT(*) FROM mtg_refresh_log")) - count = result.scalar() - print(f"\nRefresh log entries: {count}") - - # Check key tables - for table in ['mtg_set', 'mtg_card', 'mtg_identifiers']: - if table in tables: - result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) - count = result.scalar() - print(f"{table}: {count:,} records") - - except Exception as e: - print(f"ERROR connecting to database: {e}") - - # Try to create MTGJSON manager and check status - print("\n=== MTGJSON Manager Status ===") - try: - manager = MTGJSONManager() - status = manager.get_status() - print(f"Status: {status}") - except Exception as e: - print(f"ERROR creating MTGJSON manager: {e}") - -if __name__ == "__main__": - asyncio.run(check_mtgjson_status()) diff --git a/backend/scripts/code_review.md b/backend/scripts/code_review.md deleted file mode 100644 index 4388803..0000000 --- a/backend/scripts/code_review.md +++ /dev/null @@ -1,213 +0,0 @@ -# MTG Card Interaction Pipeline - Code Review - -## Overview - -The interaction pipeline consists of four main modules: -1. `card_profile_extractor.py` - Extracts structured profiles from MTGJSON data -2. `interaction_determinator.py` - Determines interactions between card pairs -3. `interaction_recommender.py` - Generates recommendations based on interactions -4. `interaction_pipeline.py` - Orchestrates the full pipeline - -## Issues Found - -### 1. Import Inconsistency (Critical) -**File**: `interaction_pipeline.py` -**Issue**: Complex import for `sessionmaker` -```python -self.SessionLocal = __import__('sqlalchemy.orm', fromlist=['sessionmaker']).sessionmaker(bind=self.engine) -``` -**Fix**: Use direct import at module level: -```python -from sqlalchemy.orm import sessionmaker -# ... -self.SessionLocal = sessionmaker(bind=self.engine) -``` - -### 2. Type Hints Inconsistency (Medium) -**File**: `interaction_pipeline.py` -**Issue**: Inconsistent type hints -```python -def extract_profiles(self, cards: List[Dict]) -> List: # Missing type parameter -def determine_interactions(self, profiles) -> dict: # Missing parameter type -``` -**Fix**: Add proper type hints: -```python -def extract_profiles(self, cards: List[Dict]) -> List[CardProfile]: -def determine_interactions(self, profiles: List[CardProfile]) -> dict: -``` - -### 3. Complex Conditional Logic (High) -**File**: `interaction_pipeline.py` -**Issue**: Nested ternary operators for synergy_type and counter_type determination -```python -"synergy_type": "archetype" if interaction.metadata and interaction.metadata.get('common_archetypes') else - "mechanic" if interaction.metadata and interaction.metadata.get('mechanics') else - "mana" if interaction.metadata and interaction.metadata.get('colors') else - "combo" if interaction.metadata and interaction.metadata.get('card_a_targets') else - "support", -``` -**Fix**: Extract to helper methods or use lookup dictionaries - -### 4. Missing Validation (Medium) -**File**: `interaction_pipeline.py` -**Issue**: No validation for empty card lists or invalid data -**Fix**: Add validation at the start of methods - -### 5. Evolution Type Mapping (High) -**File**: `interaction_pipeline.py` -**Issue**: Using `interaction.interaction_type` which is 'evolution' for all evolutions -**Fix**: Map to specific evolution types based on metadata - -## Recommended Fixes - -### Fix 1: Import Structure -```python -# At top of file -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker - -# In __init__ -self.SessionLocal = sessionmaker(bind=self.engine) -``` - -### Fix 2: Type Hints -```python -def extract_profiles(self, cards: List[Dict]) -> List[CardProfile]: - return self.profile_extractor.extract_profiles_batch(cards) - -def determine_interactions(self, profiles: List[CardProfile]) -> dict: - return self.determinator.determine_all_interactions(profiles) -``` - -### Fix 3: Helper Methods for Type Determination -```python -def _determine_synergy_type(self, interaction) -> str: - """Determine synergy type from interaction metadata.""" - metadata = interaction.metadata or {} - - if 'common_archetypes' in metadata: - return 'archetype' - elif 'mechanics' in metadata: - return 'mechanic' - elif 'colors' in metadata: - return 'mana' - elif 'card_a_targets' in metadata: - return 'combo' - else: - return 'support' - -def _determine_counter_type(self, interaction) -> str: - """Determine counter type from interaction metadata.""" - metadata = interaction.metadata or {} - - if 'colors_a' in metadata: - return 'color' - elif 'power_a' in metadata: - return 'stats' - else: - return 'keyword' - -def _determine_evolution_type(self, interaction) -> str: - """Determine evolution type from interaction metadata.""" - metadata = interaction.metadata or {} - - if 'card_name' in metadata: - return 'reprint' - else: - return 'evolution' -``` - -### Fix 4: Add Validation -```python -def run_initial_load(self, set_code: Optional[str] = None): - """Run initial load with validation.""" - if not set_code: - logger.info("No set code provided, loading all cards") - - all_cards = self.load_cards_from_db(set_code) - - if not all_cards: - logger.warning("No cards found in database") - return - - # Continue with processing... -``` - -## Accuracy Review - -### Profile Extraction -✅ **Correct**: Color extraction from mana cost -✅ **Correct**: Mechanic extraction using regex patterns -✅ **Correct**: Archetype extraction from subtypes -✅ **Correct**: Target extraction from oracle text -✅ **Correct**: Trigger and effect extraction -✅ **Correct**: Theme extraction based on characteristics - -### Interaction Determination -✅ **Correct**: Archetype synergy detection -✅ **Correct**: Mana synergy detection -✅ **Correct**: Mechanic synergy detection (haste+trample, lifelink+combat) -✅ **Correct**: Combo synergy detection (targets + triggers) -✅ **Correct**: Support synergy detection -✅ **Correct**: Counter detection (color, stats, keywords) -✅ **Correct**: Evolution detection (reprints) - -### Database Schema -✅ **Correct**: Synergies table with proper constraints -✅ **Correct**: Counters table with proper constraints -✅ **Correct**: Evolutions table with proper constraints -✅ **Correct**: Statistics table with proper aggregations -✅ **Correct**: Foreign key relationships -✅ **Correct**: Unique constraints to prevent duplicates - -## Consistency Issues - -### 1. File Organization -- All files in `/home/wall-o/projects/mtgonline/backend/scripts/` -- No clear separation between core logic and pipeline -- **Recommendation**: Keep as is for simplicity - -### 2. Naming Conventions -- **Good**: Consistent use of snake_case for methods -- **Good**: Consistent use of CamelCase for classes -- **Issue**: Mixed use of `set_code` parameter naming -- **Recommendation**: Standardize on `set_code` - -### 3. Error Handling -- **Good**: Try/finally blocks for database connections -- **Issue**: No specific exception handling for database errors -- **Recommendation**: Add specific exception types - -### 4. Logging -- **Good**: Consistent logging format -- **Good**: Appropriate log levels (INFO, WARNING, ERROR) -- **Issue**: Missing DEBUG logging for development -- **Recommendation**: Add DEBUG level logging - -## Summary - -### Critical Issues (Must Fix) -1. ❌ Import structure for sessionmaker -2. ❌ Complex conditional logic for type determination - -### High Priority Issues (Should Fix) -3. ❌ Missing type hints -4. ❌ Evolution type mapping -5. ❌ Missing validation - -### Medium Priority Issues (Nice to Have) -6. ⚠️ Add specific exception handling -7. ⚠️ Add DEBUG logging -8. ⚠️ Standardize parameter naming - -### Low Priority Issues (Can Defer) -9. ✅ All core logic is accurate and correct -10. ✅ Database schema is well-designed -11. ✅ Interaction determination logic is sound - -## Next Steps - -1. **Fix Critical Issues**: Update import structure and simplify conditional logic -2. **Fix High Priority**: Add proper type hints and validation -3. **Test**: Run pipeline with sample data to verify functionality -4. **Document**: Add docstrings and inline comments for complex logic diff --git a/backend/scripts/create_card_interaction_graph.py b/backend/scripts/create_card_interaction_graph.py deleted file mode 100644 index a79337d..0000000 --- a/backend/scripts/create_card_interaction_graph.py +++ /dev/null @@ -1,927 +0,0 @@ -""" -MTG Card Interaction Graph Schema - -Creates tables for categorizing cards based on their interactions with each other. -This creates a knowledge graph of card relationships including: -- Synergies (cards that work well together) -- Combos (cards that create powerful combinations) -- Counters (cards that counter each other) -- Evolution chains (cards that transform/evolve) -- Partners (commander partnerships, etc.) -- Archetypes (goblins, vampires, elves, etc.) -- Mechanics (first strike, trample, flying, etc.) -- Themes (storm, tokens, mill, etc.) -- Mana relationships (land support) -- Set themes (cards that share set-specific themes) -""" -from sqlalchemy import create_engine, text - -DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata" - - -class CardInteractionGraph: - """Creates and manages the card interaction knowledge graph.""" - - def __init__(self): - self.engine = create_engine(DB_URL) - self.conn = None - - def connect(self): - """Connect to database.""" - self.conn = self.engine.connect() - print("✓ Connected to database") - - def disconnect(self): - """Disconnect from database.""" - if self.conn: - self.conn.close() - self.engine.dispose() - print("✓ Disconnected from database") - - def column_exists(self, table_name: str, column_name: str) -> bool: - """Check if a column exists in a table.""" - result = self.conn.execute(text(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = :table AND column_name = :column - """), {"table": table_name, "column": column_name}) - return result.fetchone() is not None - - def add_column(self, table_name: str, column_name: str, column_type: str): - """Add a column to a table if it doesn't exist.""" - if not self.column_exists(table_name, column_name): - self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}")) - print(f" ✓ Added: {table_name}.{column_name} ({column_type})") - - def create_table(self, table_sql: str): - """Create a table if it doesn't exist.""" - self.conn.execute(text(table_sql)) - print(f" ✓ Created table: {table_sql.split('CREATE TABLE')[1].split('(')[0].strip()}") - - def create_unique_constraint(self, constraint_sql: str): - """Create a unique constraint if it doesn't exist.""" - try: - self.conn.execute(text(constraint_sql)) - print(f" ✓ Created constraint: {constraint_sql.split('ADD')[1].split('CONSTRAINT')[1].split('(')[0].strip()}") - except Exception as e: - # Constraint might already exist - pass - - def create_index(self, index_sql: str): - """Create an index if it doesn't exist.""" - self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql}")) - print(f" ✓ Created index: {index_sql.split('ON ')[1].split(' ')[0]}") - - def create_card_mechanics_table(self): - """Create table for card mechanics (first strike, trample, flying, etc.).""" - print("\n📊 Creating mtg_card_mechanics table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_mechanics ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - mechanic VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, mechanic) - ) - """) - - # Add indexes for frequently queried mechanics - indexes = [ - "idx_mechanics_card_id ON mtg_card_mechanics(card_id)", - "idx_mechanics_mechanic ON mtg_card_mechanics(mechanic)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card mechanics table created") - - def create_card_archetypes_table(self): - """Create table for card archetypes (goblins, vampires, elves, etc.).""" - print("\n📊 Creating mtg_card_archetypes table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_archetypes ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - archetype VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, archetype) - ) - """) - - indexes = [ - "idx_archetypes_card_id ON mtg_card_archetypes(card_id)", - "idx_archetypes_archetype ON mtg_card_archetypes(archetype)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card archetypes table created") - - def create_card_themes_table(self): - """Create table for card themes (storm, tokens, mill, etc.).""" - print("\n📊 Creating mtg_card_themes table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_themes ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - theme VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, theme) - ) - """) - - indexes = [ - "idx_themes_card_id ON mtg_card_themes(card_id)", - "idx_themes_theme ON mtg_card_themes(theme)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card themes table created") - - def create_card_relationships_table(self): - """Create table for general card relationships.""" - print("\n📊 Creating mtg_card_relationships table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_relationships ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - relationship_type VARCHAR(50) NOT NULL, - -- Types: synergy, combo, counter, evolution, partner, support, rival - strength INTEGER DEFAULT 1, - -- Strength: 1-5 (how strong the relationship is) - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, relationship_type) - ) - """) - - indexes = [ - "idx_relationships_card_a ON mtg_card_relationships(card_a_id)", - "idx_relationships_card_b ON mtg_card_relationships(card_b_id)", - "idx_relationships_type ON mtg_card_relationships(relationship_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card relationships table created") - - def create_card_synergies_table(self): - """Create table for card synergies with detailed scoring.""" - print("\n📊 Creating mtg_card_synergies table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_synergies ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - synergy_type VARCHAR(50) NOT NULL, - -- Types: mana_base, mechanic_support, archetype_support, - -- combo_partner, counter_partner, evolution_chain - strength INTEGER NOT NULL CHECK (strength BETWEEN 1 AND 5), - -- 1: Weak synergy, 5: Essential synergy - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, synergy_type) - ) - """) - - indexes = [ - "idx_synergies_card_a ON mtg_card_synergies(card_a_id)", - "idx_synergies_card_b ON mtg_card_synergies(card_b_id)", - "idx_synergies_type ON mtg_card_synergies(synergy_type)", - "idx_synergies_strength ON mtg_card_synergies(strength)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card synergies table created") - - def create_card_counters_table(self): - """Create table for cards that counter each other.""" - print("\n📊 Creating mtg_card_counters table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_counters ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - counter_type VARCHAR(50) NOT NULL, - -- Types: direct_counter, disadvantage, outclass, counter_role - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, counter_type) - ) - """) - - indexes = [ - "idx_counters_card_a ON mtg_card_counters(card_a_id)", - "idx_counters_card_b ON mtg_card_counters(card_b_id)", - "idx_counters_type ON mtg_card_counters(counter_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card counters table created") - - def create_card_evolution_table(self): - """Create table for evolution chains (cards that transform/evolve).""" - print("\n📊 Creating mtg_card_evolution table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_evolution ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - evolved_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - evolution_type VARCHAR(50) NOT NULL, - -- Types: transform, evolve, double_sided, modal_dfc - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, evolved_card_id, evolution_type) - ) - """) - - indexes = [ - "idx_evolution_card_id ON mtg_card_evolution(card_id)", - "idx_evolution_evolved_id ON mtg_card_evolution(evolved_card_id)", - "idx_evolution_type ON mtg_card_evolution(evolution_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card evolution table created") - - def create_card_partners_table(self): - """Create table for card partnerships (commander partners, etc.).""" - print("\n📊 Creating mtg_card_partners table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_partners ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - partnership_type VARCHAR(50) NOT NULL, - -- Types: commander_partner, double_faced, companion, partner_commander - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, partnership_type) - ) - """) - - indexes = [ - "idx_partners_card_a ON mtg_card_partners(card_a_id)", - "idx_partners_card_b ON mtg_card_partners(card_b_id)", - "idx_partners_type ON mtg_card_partners(partnership_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card partners table created") - - def create_card_mana_relations_table(self): - """Create table for land/mana relationships.""" - print("\n📊 Creating mtg_card_mana_relations table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_mana_relations ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - land_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - mana_type VARCHAR(10) NOT NULL, - -- Types: produces, taps_for, fetches, searches, enters_tapped - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, land_id, mana_type) - ) - """) - - indexes = [ - "idx_mana_card_id ON mtg_card_mana_relations(card_id)", - "idx_mana_land_id ON mtg_card_mana_relations(land_id)", - "idx_mana_type ON mtg_card_mana_relations(mana_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card mana relations table created") - - def create_card_set_relations_table(self): - """Create table for set/theme relationships.""" - print("\n📊 Creating mtg_card_set_relations table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_set_relations ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE, - theme VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, set_id, theme) - ) - """) - - indexes = [ - "idx_setrel_card_id ON mtg_card_set_relations(card_id)", - "idx_setrel_set_id ON mtg_card_set_relations(set_id)", - "idx_setrel_theme ON mtg_card_set_relations(theme)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card set relations table created") - - def create_card_power_relations_table(self): - """Create table for power/toughness relationships.""" - print("\n📊 Creating mtg_card_power_relations table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_power_relations ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - relation_type VARCHAR(50) NOT NULL, - -- Types: outclasses, matches, underclasses, counters_power - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, relation_type) - ) - """) - - indexes = [ - "idx_power_card_a ON mtg_card_power_relations(card_a_id)", - "idx_power_card_b ON mtg_card_power_relations(card_b_id)", - "idx_power_type ON mtg_card_power_relations(relation_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card power relations table created") - - def create_card_history_table(self): - """Create table for card history and legacy relationships.""" - print("\n📊 Creating mtg_card_history table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_history ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - related_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - history_type VARCHAR(50) NOT NULL, - -- Types: reprinted_in, previous_version, alternative_art, - -- superseded_by, predecessor - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, related_card_id, history_type) - ) - """) - - indexes = [ - "idx_history_card_id ON mtg_card_history(card_id)", - "idx_history_related_id ON mtg_card_history(related_card_id)", - "idx_history_type ON mtg_card_history(history_type)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card history table created") - - def create_card_interaction_stats_table(self): - """Create summary statistics table for card interactions.""" - print("\n📊 Creating mtg_card_interaction_stats table...") - - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_interaction_stats ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - total_synergies INTEGER DEFAULT 0, - total_counters INTEGER DEFAULT 0, - total_evolution INTEGER DEFAULT 0, - total_partners INTEGER DEFAULT 0, - total_mechanics INTEGER DEFAULT 0, - total_archetypes INTEGER DEFAULT 0, - total_themes INTEGER DEFAULT 0, - avg_synergy_strength DECIMAL(3,2) DEFAULT 0.00, - max_synergy_strength INTEGER DEFAULT 0, - primary_archetype VARCHAR(100), - primary_theme VARCHAR(100), - computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id) - ) - """) - - indexes = [ - "idx_stats_card_id ON mtg_card_interaction_stats(card_id)", - "idx_stats_total_synergies ON mtg_card_interaction_stats(total_synergies)", - "idx_stats_primary_archetype ON mtg_card_interaction_stats(primary_archetype)", - ] - for idx in indexes: - self.create_index(idx) - - print(" ✓ Card interaction stats table created") - - def populate_mechanics_from_type_line(self): - """Populate mechanics from card type lines and oracle text.""" - print("\n🔄 Populating mechanics from type lines...") - - # Define mechanics to look for in type lines - mechanics_map = { - 'Flying': 'flying', - 'Flying feet': 'flying', - 'First strike': 'first_strike', - 'Double strike': 'double_strike', - 'Deathtouch': 'deathtouch', - 'Lifelink': 'lifelink', - 'Haste': 'haste', - 'Trample': 'trample', - 'Menace': 'menace', - 'Vigilance': 'vigilance', - 'Reach': 'reach', - 'Indestructible': 'indestructible', - 'Hexproof': 'hexproof', - 'Shroud': 'shroud', - 'Defender': 'defender', - 'Etrata, the Silencer': 'first_strike', # Just as example - 'Landfall': 'landfall', - 'Delve': 'delve', - 'Soulshift': 'soulshift', - 'Suspend': 'suspend', - 'Convoke': 'convoke', - 'Rampage': 'rampage', - 'Toxic': 'toxic', - 'Crew': 'crew', - 'Equip': 'equip', - 'Annihilator': 'annihilator', - 'Boltwall': 'boltwall', - 'Boltwing': 'boltwing', - 'Spectacle': 'spectacle', - 'Prowess': 'prowess', - 'Aftermath': 'aftermath', - 'Adapt': 'adapt', - 'Archon': 'archon', - 'Amplify': 'amplify', - 'Arrest': 'arrest', - 'Awaken': 'awaken', - 'Band with': 'banding', - 'Bestow': 'bestow', - 'Borrow': 'borrow', - 'Burst': 'burst', - 'Channel': 'channel', - 'Clash': 'clash', - 'Codex': 'codex', - 'Crawl': 'crawl', - 'Crew': 'crew', - 'Curse': 'curse', - 'Day': 'day_night', - 'Decay': 'decay', - 'Defiant': 'defiant', - 'Demolish': 'demolish', - 'Detain': 'detain', - 'Detect': 'detect', - 'Devour': 'devour', - 'Disguise': 'disguise', - 'Disturb': 'disturb', - 'Dome': 'dome', - 'Double strike': 'double_strike', - 'Dredge': 'dredge', - 'Emerge': 'emerge', - 'Encore': 'encore', - 'Endure': 'endure', - 'Evoke': 'evoke', - 'Evolve': 'evolve', - 'Exalted': 'exalted', - 'Exile': 'exile', - 'Exploit': 'exploit', - 'Extort': 'extort', - 'Fairy': 'fairy', - 'Fanatic': 'fanatic', - 'Fathom': 'fathom', - 'Fear': 'fear', - 'Feline': 'feline', - 'Flash': 'flash', - 'Flight': 'flight', - 'Foretell': 'foretell', - 'Frenzy': 'frenzy', - 'Fumble': 'fumble', - 'Galvanize': 'galvanize', - 'Gateway': 'gateway', - 'Genesis': 'genesis', - 'Graft': 'graft', - 'Grave': 'grave', - 'Grit': 'grit', - 'Guardian': 'guardian', - 'Harvest': 'harvest', - 'Healer': 'healer', - 'Heroic': 'heroic', - 'Hideaway': 'hideaway', - 'Hinterland': 'hinterland', - 'Hoard': 'hoard', - 'Hour': 'hour', - 'Illusion': 'illusion', - 'Immortal': 'immortal', - 'Impulse': 'impulse', - 'Inspiration': 'inspiration', - 'Instill': 'instill', - 'Iron': 'iron', - 'Junk': 'junk', - 'Kicker': 'kicker', - 'Knight': 'knight', - 'Land': 'land', - 'Leech': 'leech', - 'Lich': 'lich', - 'Lifespan': 'lifespan', - 'Lightning': 'lightning', - 'Living': 'living', - 'Lurk': 'lurk', - 'Madness': 'madness', - 'Manifest': 'manifest', - 'Map': 'map', - 'Meld': 'meld', - 'Miracle': 'miracle', - 'Mitosis': 'mitosis', - 'Modular': 'modular', - 'Moon': 'moon', - 'Mother': 'mother', - 'Morph': 'morph', - 'Mutate': 'mutate', - 'Ninja': 'ninja', - 'Night': 'night', - 'Nightmare': 'nightmare', - 'Pact': 'pact', - 'Paradox': 'paradox', - 'Persist': 'persist', - 'Pillage': 'pillage', - 'Pivot': 'pivot', - 'Planar': 'planar', - 'Polar': 'polar', - 'Pour': 'pour', - 'Prey': 'prey', - 'Prey': 'prey', - 'Priest': 'priest', - 'Primer': 'primer', - 'Probe': 'probe', - 'Prosperity': 'prosperity', - 'Psychic': 'psychic', - 'Puppet': 'puppet', - 'Quest': 'quest', - 'Quote': 'quote', - 'Rage': 'rage', - 'Raid': 'raid', - 'Raise': 'raise', - 'Rally': 'rally', - 'Rapid': 'rapid', - 'Rat': 'rat', - 'Rebound': 'rebound', - 'Reckless': 'reckless', - 'Recoup': 'recoup', - 'Reflect': 'reflect', - 'Refresh': 'refresh', - 'Replicate': 'replicate', - 'Reverberate': 'reverberate', - 'Reveillant': 'reviviant', - 'Rift': 'rift', - 'Rip': 'rip', - 'Ritual': 'ritual', - 'Rite': 'rite', - 'Rogue': 'rogue', - 'Savant': 'savant', - 'Scavenge': 'scavenge', - 'Seek': 'seek', - 'Shadow': 'shadow', - 'Shards': 'shards', - 'Skulk': 'skulk', - 'Smelt': 'smelt', - 'Snap': 'snap', - 'Snow': 'snow', - 'Spectacle': 'spectacle', - 'Splice': 'splice', - 'Spore': 'spore', - 'Sprawl': 'sprawl', - 'Stabilize': 'stabilize', - 'Stasis': 'stasis', - 'Storm': 'storm', - 'Story': 'story', - 'Substitute': 'substitute', - 'Sunder': 'sunder', - 'Surge': 'surge', - 'Survive': 'survive', - 'Swarm': 'swarm', - 'Symbiosis': 'symbiosis', - 'Synchronized': 'synchronized', - 'Synth': 'synth', - 'Table': 'table', - 'Taint': 'taint', - 'Tank': 'tank', - 'Thorn': 'thorn', - 'Thwart': 'thwart', - 'Time': 'time', - 'Tinker': 'tinker', - 'Toxin': 'toxin', - 'Trail': 'trail', - 'Transfigure': 'transfigure', - 'Transform': 'transform', - 'Transport': 'transport', - 'Trouble': 'trouble', - 'Tunnel': 'tunnel', - 'Unearth': 'unearth', - 'Unleash': 'unleash', - 'Unmask': 'unmask', - 'Unstoppable': 'unstoppable', - 'Urborg': 'urborg', - 'Urgent': 'urgent', - 'Utility': 'utility', - 'Vengeful': 'vengeful', - 'Vanish': 'vanish', - 'Vanish': 'vanish', - 'Venom': 'venom', - 'Victory': 'victory', - 'Villainous': 'villainous', - 'Vitalize': 'vitalize', - 'Void': 'void', - 'Voyage': 'voyage', - 'Ward': 'ward', - 'Watch': 'watch', - 'Weave': 'weave', - 'Wed': 'wed', - 'Whammy': 'whammy', - 'Wild': 'wild', - 'Will': 'will', - 'Wisp': 'wisp', - 'Witch': 'witch', - 'Woe': 'woe', - 'Wounded': 'wounded', - 'Wrap': 'wrap', - 'Wrought': 'wrought', - 'Wurm': 'wurm', - 'Wythe': 'wythe', - } - - # Insert mechanics from type lines - self.conn.execute(text(""" - INSERT INTO mtg_card_mechanics (card_id, mechanic) - SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ','))) - FROM mtg_cards c - WHERE c.subtypes IS NOT NULL - AND c.subtypes != '' - AND c.subtypes != 'null' - AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN ( - 'flying', 'first_strike', 'double_strike', 'deathtouch', 'lifelink', - 'haste', 'trample', 'menace', 'vigilance', 'reach', 'indestructible', - 'hexproof', 'shroud', 'defender', 'landfall', 'delve', 'soulshift', - 'suspend', 'convoke', 'rampage', 'toxic', 'crew', 'equip', 'annihilator', - 'spectacle', 'prowess', 'aftermath', 'adapt', 'amplify', 'awaken', - 'banding', 'bestow', 'burst', 'channel', 'clash', 'crawl', 'curse', - 'day_night', 'decay', 'defiant', 'demolish', 'detain', 'detect', - 'devour', 'disguise', 'disturb', 'dome', 'double_strike', 'dredge', - 'emerge', 'encore', 'endure', 'evoke', 'evolve', 'exalted', 'exile', - 'exploit', 'extort', 'fairy', 'fanatic', 'fathom', 'fear', 'feline', - 'flash', 'flight', 'foretell', 'frenzy', 'fumble', 'galvanize', - 'gateway', 'genesis', 'graft', 'grave', 'grit', 'guardian', 'harvest', - 'healer', 'heroic', 'hideaway', 'hinterland', 'hoard', 'hour', 'illusion', - 'immortal', 'impulse', 'inspiration', 'instill', 'iron', 'junk', 'kicker', - 'knight', 'land', 'leech', 'lich', 'lifespan', 'lightning', 'living', - 'lurk', 'madness', 'manifest', 'map', 'meld', 'miracle', 'mitosis', - 'modular', 'moon', 'mother', 'morph', 'mutate', 'ninja', 'night', - 'nightmare', 'pact', 'paradox', 'persist', 'pillage', 'pivot', 'planar', - 'polar', 'pour', 'prey', 'priest', 'primer', 'probe', 'prosperity', - 'psychic', 'puppet', 'quest', 'quote', 'rage', 'raid', 'raise', 'rally', - 'rapid', 'rat', 'rebound', 'reckless', 'recoup', 'reflect', 'refresh', - 'replicate', 'reverberate', 'reviviant', 'rift', 'rip', 'ritual', 'rite', - 'rogue', 'savant', 'scavenge', 'seek', 'shadow', 'shards', 'skulk', - 'smelt', 'snap', 'snow', 'spectacle', 'splice', 'spore', 'sprawl', - 'stabilize', 'stasis', 'storm', 'story', 'substitute', 'sunder', 'surge', - 'survive', 'swarm', 'symbiosis', 'synchronized', 'synth', 'table', 'taint', - 'tank', 'thorn', 'thwart', 'time', 'tinker', 'toxin', 'trail', 'transfigure', - 'transform', 'transport', 'trouble', 'tunnel', 'unearth', 'unleash', 'unmask', - 'unstoppable', 'urborg', 'urgent', 'utility', 'vengeful', 'vanish', 'venom', - 'victory', 'villainous', 'vitalize', 'void', 'voyage', 'ward', 'watch', 'weave', - 'wed', 'whammy', 'wild', 'will', 'wisp', 'witch', 'woe', 'wounded', 'wrap', - 'wrought', 'wurm', 'wythe' - ) - ON CONFLICT DO NOTHING - """)) - - print(" ✓ Populated mechanics from type lines") - - def populate_archetypes_from_subtypes(self): - """Populate archetypes from card subtypes.""" - print("\n🔄 Populating archetypes from subtypes...") - - # Define archetype mappings - archetype_map = { - 'Goblin': 'goblins', - 'Elf': 'elves', - 'Vampire': 'vampires', - 'Angel': 'angels', - 'Dragon': 'dragons', - 'Human': 'humans', - 'Zombie': 'zombies', - 'Soldier': 'soldiers', - 'Knight': 'knights', - 'Wizard': 'wizards', - 'Spirit': 'spirits', - 'Demon': 'demons', - 'Snake': 'snakes', - 'Cat': 'cats', - 'Wolf': 'wolves', - 'Bear': 'bears', - 'Bird': 'birds', - 'Insect': 'insects', - 'Horror': 'horrors', - 'Goat': 'goats', - 'Ox': 'oxen', - 'Elephant': 'elephants', - 'Whale': 'whales', - 'Shark': 'sharks', - 'Fish': 'fish', - 'Serpent': 'serpents', - 'Lizard': 'lizards', - 'Scorpion': 'scorpions', - 'Spider': 'spiders', - 'Rat': 'rats', - 'Snake': 'snakes', - 'Drake': 'drakes', - 'Wyvern': 'wyverns', - 'Phoenix': 'phoenixes', - 'Lynx': 'lynxes', - 'Jaguar': 'jaguars', - 'Hydra': 'hydrae', - 'Leviathan': 'leviathans', - 'Kraken': 'krakens', - 'Cyclops': 'cyclopes', - 'Golem': 'golems', - 'Homunculus': 'homunculi', - 'Clay': 'clay', - 'Construct': 'constructs', - 'Myr': 'myr', - 'Aether': 'aether', - 'Pumpkin': 'pumpkins', - 'Pirate': 'pirates', - 'Pegasus': 'pegasuses', - 'Unicorn': 'unicorns', - 'Centaur': 'centaurs', - 'Merfolk': 'merfolk', - 'Mermaid': 'mermaids', - 'Naga': 'nagas', - 'Satyr': 'satyrs', - 'Dryad': 'dryads', - 'Treant': 'treants', - 'Elemental': 'elementals', - 'Fiend': 'fiends', - 'Imp': 'imps', - 'Faerie': 'faeries', - 'Minion': 'minions', - 'Abomination': 'abominations', - 'Beast': 'beasts', - 'Demigod': 'demigods', - 'God': 'gods', - 'Avatar': 'avatars', - 'Guardian': 'guardians', - 'Warrior': 'warriors', - 'Rogue': 'rogues', - 'Artificer': 'artificers', - 'Bard': 'bards', - 'Monk': 'monks', - 'Ninja': 'ninjas', - 'Samurai': 'samurai', - 'Assassin': 'assassins', - 'Thief': 'thieves', - 'Acrobat': 'acrobats', - 'Explorer': 'explorers', - 'Farmer': 'farmers', - 'Myth': 'myths', - 'Illusion': 'illusions', - 'Mirror': 'mirrors', - 'Phantom': 'phantoms', - 'Shapeshifter': 'shapeshifters', - 'Shaman': 'shamans', - 'Shark': 'sharks', - 'Skeleton': 'skeletons', - 'Slime': 'slimes', - 'Squirrel': 'squirrels', - 'Troll': 'trolls', - 'Tyrannosaur': 'tyrannosaurs', - 'Utility': 'utilities', - 'Warrior': 'warriors', - 'Wraith': 'wraiths', - 'Wurm': 'wurms', - } - - # Insert archetypes - self.conn.execute(text(""" - INSERT INTO mtg_card_archetypes (card_id, archetype) - SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ','))) - FROM mtg_cards c - WHERE c.subtypes IS NOT NULL - AND c.subtypes != '' - AND c.subtypes != 'null' - AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN ( - 'goblin', 'elf', 'vampire', 'angel', 'dragon', 'human', 'zombie', - 'soldier', 'knight', 'wizard', 'spirit', 'demon', 'snake', 'cat', - 'wolf', 'bear', 'bird', 'insect', 'horror', 'goat', 'ox', 'elephant', - 'whale', 'shark', 'fish', 'serpent', 'lizard', 'scorpion', 'spider', - 'rat', 'drake', 'wyvern', 'phoenix', 'lynx', 'jaguar', 'hydra', - 'leviathan', 'kraken', 'cyclops', 'golem', 'homunculus', 'clay', - 'construct', 'myr', 'aether', 'pumpkin', 'pirate', 'pegasus', - 'unicorn', 'centaur', 'merfolk', 'mermaid', 'naga', 'satyr', 'dryad', - 'treant', 'elemental', 'fiend', 'imp', 'faerie', 'minion', 'abomination', - 'beast', 'demigod', 'god', 'avatar', 'guardian', 'warrior', 'rogue', - 'artificer', 'bard', 'monk', 'ninja', 'samurai', 'assassin', 'thief', - 'acrobat', 'explorer', 'farmer', 'myth', 'illusion', 'mirror', 'phantom', - 'shapeshifter', 'shaman', 'skeleton', 'slime', 'squirrel', 'troll', - 'tyrannosaur', 'wraith', 'wurm' - ) - ON CONFLICT DO NOTHING - """)) - - print(" ✓ Populated archetypes from subtypes") - - def populate_themes_from_oracle_text(self): - """Populate themes from oracle text patterns.""" - print("\n🔄 Populating themes from oracle text...") - - # Define theme patterns to search for - theme_patterns = [ - ('storm', 'oracle_text LIKE \'%cast %spell%\' OR oracle_text LIKE \'%copy spell%\' OR oracle_text LIKE \'%cast additional spell%\''), - ('tokens', 'oracle_text LIKE \'%create %token%\' OR oracle_text LIKE \'%put %token%\' OR oracle_text LIKE \'%you get %token%\''), - ('mill', 'oracle_text LIKE \'%mill%\' OR oracle_text LIKE \'%put cards from top of your library into your graveyard%\''), - ('flicker', 'oracle_text LIKE \'%exile %and return%\' OR oracle_text LIKE \'%unmark%\' OR oracle_text LIKE \'%bounce%\''), - ('draw', 'oracle_text LIKE \'%draw %cards%\' OR oracle_text LIKE \'%you may draw%\''), - ('life_gain', 'oracle_text LIKE \'%gain life%\' OR oracle_text LIKE \'%you gain % life%\''), - ('board_wipe', 'oracle_text LIKE \'%all creatures get -%\' OR oracle_text LIKE \'%destroy all creatures%\''), - ('deck_out', 'oracle_text LIKE \'%lose the game%\' OR oracle_text LIKE \'%you lose the game%\''), - ('reanimate', 'oracle_text LIKE \'%put card from graveyard%\' OR oracle_text LIKE \'%return card from graveyard%\''), - ('countermagic', 'oracle_text LIKE \'%counter target spell%\' OR oracle_text LIKE \'%counter target spell%\''), - ('card_advantage', 'oracle_text LIKE \'%draw %card%\' OR oracle_text LIKE \'%draw two cards%\''), - ('mana_acceleration', 'oracle_text LIKE \'%add %mana%\' OR oracle_text LIKE \'%add {C}%\' OR oracle_text LIKE \'%add {R}%\' OR oracle_text LIKE \'%add {U}%\' OR oracle_text LIKE \'%add {B}%\' OR oracle_text LIKE \'%add {G}%\' OR oracle_text LIKE \'%add {W}%\''), - ('combat_tricks', 'oracle_text LIKE \'%gain first strike%\' OR oracle_text LIKE \'%gain trample%\' OR oracle_text LIKE \'%gain deathtouch%\' OR oracle_text LIKE \'%gain lifelink%\' OR oracle_text LIKE \'%gain vigilance%\' OR oracle_text LIKE \'%until end of turn%\''), - ('etb_effects', 'oracle_text LIKE \'%when %enters the battlefield%\' OR oracle_text LIKE \'%enters the battlefield with%\' OR oracle_text LIKE \'%enters the battlefield tapped%\''), - ('ltb_effects', 'oracle_text LIKE \'%when %leaves the battlefield%\' OR oracle_text LIKE \'%leaves the battlefield, exile%\'\'' ), - ('synergy', 'oracle_text LIKE \'%copy %spell%\' OR oracle_text LIKE \'%create %token%\' OR oracle_text LIKE \'%gain % life%\'' ), - ] - - # This is a complex query, let's simplify for demonstration - # In production, you'd want to use more sophisticated NLP or pattern matching - - print(" ℹ️ Theme population requires complex pattern matching") - print(" ℹ️ Skipping for now - can be added as a separate step") - - def run_migration(self): - """Run the full migration.""" - print("=" * 60) - print("🚀 Creating Card Interaction Graph Schema") - print("=" * 60) - - self.connect() - - # Create all interaction tables - self.create_card_mechanics_table() - self.create_card_archetypes_table() - self.create_card_themes_table() - self.create_card_relationships_table() - self.create_card_synergies_table() - self.create_card_counters_table() - self.create_card_evolution_table() - self.create_card_partners_table() - self.create_card_mana_relations_table() - self.create_card_set_relations_table() - self.create_card_power_relations_table() - self.create_card_history_table() - self.create_card_interaction_stats_table() - - # Populate some data - self.populate_mechanics_from_type_line() - self.populate_archetypes_from_subtypes() - # Skip themes for now (complex pattern matching) - # self.populate_themes_from_oracle_text() - - self.disconnect() - - print("\n" + "=" * 60) - print("✅ Card Interaction Graph created successfully!") - print("=" * 60) - - -def main(): - """Main entry point.""" - graph = CardInteractionGraph() - graph.run_migration() - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/inspect_db.py b/backend/scripts/inspect_db.py deleted file mode 100644 index a0b04c7..0000000 --- a/backend/scripts/inspect_db.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Inspect the actual database schema from the running containers.""" -import psycopg2 -import json - -def inspect(): - conn = psycopg2.connect( - host="172.18.0.2", port=5432, - dbname="mtgdata", user="mtgonline", password="mtgonline_pass" - ) - cur = conn.cursor() - - # Get actual mtg_cards columns - cur.execute(""" - SELECT column_name, data_type, column_default - FROM information_schema.columns - WHERE table_name = 'mtg_cards' - ORDER BY ordinal_position - """) - cards_cols = cur.fetchall() - print("=== mtg_cards columns ===") - for row in cards_cols: - print(f" {row[0]}: {row[1]} (default: {row[2]})") - - # Get actual mtg_sets columns - cur.execute(""" - SELECT column_name, data_type, column_default - FROM information_schema.columns - WHERE table_name = 'mtg_sets' - ORDER BY ordinal_position - """) - sets_cols = cur.fetchall() - print("\n=== mtg_sets columns ===") - for row in sets_cols: - print(f" {row[0]}: {row[1]} (default: {row[2]})") - - # Check counts - cur.execute("SELECT COUNT(*) FROM mtg_cards") - print(f"\nmtg_cards count: {cur.fetchone()[0]}") - cur.execute("SELECT COUNT(*) FROM mtg_sets") - print(f"mtg_sets count: {cur.fetchone()[0]}") - - # Sample card - print("\n=== Sample card (first row) ===") - cur.execute("SELECT * FROM mtg_cards LIMIT 1") - col_names = [d[0] for d in cur.description] - row = cur.fetchone() - for c, v in zip(col_names, row): - print(f" {c}: {v}") - - # Sample set - print("\n=== Sample set (first row) ===") - cur.execute("SELECT * FROM mtg_sets LIMIT 1") - col_names = [d[0] for d in cur.description] - row = cur.fetchone() - for c, v in zip(col_names, row): - print(f" {c}: {v}") - - # Distinct rarities - cur.execute("SELECT DISTINCT rarity FROM mtg_cards ORDER BY rarity") - print(f"\nDistinct rarities: {[r[0] for r in cur.fetchall()]}") - - # Distinct layouts - cur.execute("SELECT DISTINCT layout FROM mtg_cards ORDER BY layout") - print(f"Distinct layouts: {[r[0] for r in cur.fetchall()]}") - - cur.close() - conn.close() - -if __name__ == "__main__": - inspect() diff --git a/backend/scripts/interaction_determinator.py b/backend/scripts/interaction_determinator.py deleted file mode 100644 index c6c5bb1..0000000 --- a/backend/scripts/interaction_determinator.py +++ /dev/null @@ -1,314 +0,0 @@ -""" -MTG Card Interaction Determinator - -Determines specific card interactions (synergies, counters, evolutions) -using game rules and card profiles. -""" -from typing import List, Tuple, Optional -from dataclasses import dataclass -from card_profile_extractor import CardProfile - - -@dataclass -class InteractionResult: - """Result of an interaction determination.""" - card_a_id: int - card_b_id: int - interaction_type: str # 'synergy', 'counter', 'evolution' - strength: int # 1-5 - confidence: float # 0.0-1.0 - notes: str - metadata: dict = None - - def __post_init__(self): - if self.metadata is None: - self.metadata = {} - - -class InteractionDeterminator: - """ - Determines card interactions using game rules. - - Uses deterministic rules based on: - - Shared archetypes (e.g., both are goblins) - - Supporting mechanics (e.g., one has haste, the other has trample) - - Mana compatibility (same colors work well together) - - Target/trigger relationships (one targets, the other interacts) - - Evolution chains (same card, different versions) - """ - - def __init__(self): - """Initialize the determinator.""" - pass - - def determine_synergies( - self, profile_a: CardProfile, profile_b: CardProfile - ) -> List[InteractionResult]: - """ - Determine synergies between two cards. - - Synergies are positive interactions where cards work well together. - - Examples: - - Both are goblins (archetype synergy) - - One has haste, the other has trample (mechanic synergy) - - Same color identity (mana synergy) - - One targets creatures, the other buffs creatures (combo synergy) - - Args: - profile_a: First card profile - profile_b: Second card profile - - Returns: - List of synergy results - """ - synergies = [] - - # 1. Archetype synergy: both share an archetype - if profile_a.archetypes and profile_b.archetypes: - common_archetypes = set(profile_a.archetypes) & set(profile_b.archetypes) - if common_archetypes: - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=3, - confidence=0.95, - notes=f"Both are {', '.join(common_archetypes)}", - metadata={'common_archetypes': list(common_archetypes)} - )) - - # 2. Mana synergy: same color identity - if profile_a.colors and profile_b.colors: - if set(profile_a.colors) == set(profile_b.colors): - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=4, - confidence=0.9, - notes="Same color identity", - metadata={'colors': profile_a.colors} - )) - - # 3. Mechanic synergy: complementary mechanics - if profile_a.mechanics and profile_b.mechanics: - # Haste + trample = aggressive combo - if ('haste' in profile_a.mechanics and 'trample' in profile_b.mechanics) or \ - ('haste' in profile_b.mechanics and 'trample' in profile_a.mechanics): - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=4, - confidence=0.85, - notes="Haste + Trample combo", - metadata={'mechanics': ['haste', 'trample']} - )) - - # Lifelink + combat keywords = combat combo - combat_keywords = ['first_strike', 'double_strike', 'deathtouch', 'trample'] - if ('lifelink' in profile_a.mechanics and any(k in profile_b.mechanics for k in combat_keywords)) or \ - ('lifelink' in profile_b.mechanics and any(k in profile_a.mechanics for k in combat_keywords)): - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=3, - confidence=0.8, - notes="Lifelink + combat keywords combo", - metadata={'mechanics': ['lifelink', 'combat']} - )) - - # 4. Combo synergy: one targets, the other interacts with targets - if profile_a.targets and profile_b.triggers: - # Card A targets creatures, Card B interacts with creature actions - if 'creature' in profile_a.targets: - creature_actions = ['enters_battlefield', 'dies', 'attacks', 'blocks'] - if any(t in creature_actions for t in profile_b.triggers): - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=3, - confidence=0.85, - notes="Card A targets creatures, Card B interacts with creature actions", - metadata={'card_a_targets': 'creature', 'card_b_interacts': 'creature_actions'} - )) - - # 5. Support synergy: one has a mechanic, the other supports it - if profile_a.mechanics and profile_b.effects: - # If card B has an effect that supports card A's mechanic - if 'haste' in profile_a.mechanics and 'gain_haste' in profile_b.effects: - synergies.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='synergy', - strength=3, - confidence=0.8, - notes="Card B grants haste to Card A", - metadata={'mechanic': 'haste', 'effect': 'gain_haste'} - )) - - return synergies - - def determine_counters( - self, profile_a: CardProfile, profile_b: CardProfile - ) -> List[InteractionResult]: - """ - Determine counter relationships between two cards. - - Counters are negative interactions where one card is disadvantaged by another. - - Examples: - - Different color identities (strategic tension) - - One has higher power (stat disadvantage) - - One counters the other's strategy (counter role) - - Args: - profile_a: First card profile - profile_b: Second card profile - - Returns: - List of counter results - """ - counters = [] - - # 1. Color counter: different color identities - if profile_a.colors and profile_b.colors: - if set(profile_a.colors) != set(profile_b.colors): - counters.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='counter', - strength=2, - confidence=0.8, - notes="Different color identities", - metadata={'colors_a': profile_a.colors, 'colors_b': profile_b.colors} - )) - - # 2. Stat counter: one has significantly higher power - if profile_a.power and profile_b.power: - try: - power_a = int(profile_a.power) - power_b = int(profile_b.power) - - if power_a > power_b + 1: - counters.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='counter', - strength=3, - confidence=0.75, - notes=f"Card A has higher power ({power_a} vs {power_b})", - metadata={'power_a': power_a, 'power_b': power_b} - )) - elif power_b > power_a + 1: - counters.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='counter', - strength=3, - confidence=0.75, - notes=f"Card B has higher power ({power_b} vs {power_a})", - metadata={'power_a': power_a, 'power_b': power_b} - )) - except (ValueError, TypeError): - pass - - # 3. Counter role: one targets creatures, the other has combat keywords - if profile_a.targets and profile_b.mechanics: - if 'creature' in profile_a.targets: - combat_keywords = ['deathtouch', 'trample', 'first_strike', 'double_strike'] - if any(m in combat_keywords for m in profile_b.mechanics): - counters.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='counter', - strength=2, - confidence=0.6, - notes="Card A targets creatures, Card B has combat keywords", - metadata={'target': 'creature', 'mechanics': profile_b.mechanics} - )) - - return counters - - def determine_evolutions( - self, profile_a: CardProfile, profile_b: CardProfile - ) -> List[InteractionResult]: - """ - Determine evolution relationships between two cards. - - Evolutions track when a card has been reprinted, transformed, or evolved. - - Examples: - - Same name in different sets (reprint) - - Transform pairs (different faces of same card) - - Double-sided cards - - Args: - profile_a: First card profile - profile_b: Second card profile - - Returns: - List of evolution results - """ - evolutions = [] - - # 1. Same name = reprint - if profile_a.name == profile_b.name: - evolutions.append(InteractionResult( - card_a_id=profile_a.id, - card_b_id=profile_b.id, - interaction_type='evolution', - strength=2, - confidence=0.9, - notes=f"Reprint of {profile_a.name}", - metadata={'card_name': profile_a.name} - )) - - # 2. Transform pairs would require checking card_faces in the database - # This is handled separately in the pipeline - - return evolutions - - def determine_all_interactions( - self, - profiles: List[CardProfile] - ) -> dict: - """ - Determine all interactions for a batch of cards. - - Args: - profiles: List of card profiles - - Returns: - Dictionary with: - - synergies: list of synergy results - - counters: list of counter results - - evolutions: list of evolution results - """ - synergies = [] - counters = [] - evolutions = [] - - # Compare all pairs - for i in range(len(profiles)): - for j in range(i + 1, len(profiles)): - profile_a = profiles[i] - profile_b = profiles[j] - - # Determine synergies - synergies.extend(self.determine_synergies(profile_a, profile_b)) - - # Determine counters - counters.extend(self.determine_counters(profile_a, profile_b)) - - # Determine evolutions - evolutions.extend(self.determine_evolutions(profile_a, profile_b)) - - return { - 'synergies': synergies, - 'counters': counters, - 'evolutions': evolutions, - } diff --git a/backend/scripts/interaction_pipeline.py b/backend/scripts/interaction_pipeline.py deleted file mode 100644 index 47b0e05..0000000 --- a/backend/scripts/interaction_pipeline.py +++ /dev/null @@ -1,491 +0,0 @@ -""" -MTG Card Interaction Pipeline - -Orchestrates the full interaction determination and recommendation pipeline. -Handles initial loads and rolling updates. -""" -import json -import logging -from datetime import datetime -from typing import List, Dict, Optional, Tuple - -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker - -from card_profile_extractor import CardProfileExtractor, CardProfile -from interaction_determinator import InteractionDeterminator, InteractionResult - - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - - -class MTGInteractionPipeline: - """ - Main pipeline for processing card interactions. - - Handles: - 1. Loading cards from database - 2. Extracting card profiles - 3. Determining interactions (synergies, counters, evolutions) - 4. Storing interactions in database - 5. Updating interaction statistics - """ - - def __init__( - self, - db_url: str, - min_confidence: float = 0.5, - ): - """ - Initialize the pipeline. - - Args: - db_url: PostgreSQL database URL - min_confidence: Minimum confidence to auto-store interactions - """ - self.db_url = db_url - self.min_confidence = min_confidence - self.engine = create_engine(db_url) - self.SessionLocal = sessionmaker(bind=self.engine) - - self.profile_extractor = CardProfileExtractor() - self.determinator = InteractionDeterminator() - - # Statistics - self.stats = { - 'cards_processed': 0, - 'interactions_determined': 0, - 'interactions_stored': 0, - 'interactions_review_queue': 0, - 'errors': 0, - } - - def load_cards_from_db(self, set_code: Optional[str] = None) -> List[Dict]: - """ - Load cards from the database. - - Args: - set_code: Optional set code to filter by - - Returns: - List of card dictionaries - """ - db = self.SessionLocal() - try: - if set_code: - query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE s.code = :set_code - """) - cards = [dict(row._mapping) for row in - db.execute(query, {"set_code": set_code}).fetchall()] - else: - query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - """) - cards = [dict(row._mapping) for row in db.execute(query).fetchall()] - - logger.info(f"Loaded {len(cards)} cards from database") - return cards - finally: - db.close() - - def extract_profiles(self, cards: List[Dict]) -> List[CardProfile]: - """ - Extract card profiles from card data. - - Args: - cards: List of card dictionaries - - Returns: - List of CardProfile objects - """ - return self.profile_extractor.extract_profiles_batch(cards) - - def determine_interactions(self, profiles: List[CardProfile]) -> dict: - """ - Determine interactions for a batch of card profiles. - - Args: - profiles: List of CardProfile objects - - Returns: - Dictionary with synergies, counters, and evolutions - """ - return self.determinator.determine_all_interactions(profiles) - - def _determine_synergy_type(self, interaction: InteractionResult) -> str: - """ - Determine synergy type from interaction metadata. - - Args: - interaction: Interaction result - - Returns: - Synergy type string - """ - metadata = interaction.metadata or {} - - if 'common_archetypes' in metadata: - return 'archetype' - elif 'mechanics' in metadata: - return 'mechanic' - elif 'colors' in metadata: - return 'mana' - elif 'card_a_targets' in metadata: - return 'combo' - else: - return 'support' - - def _determine_counter_type(self, interaction: InteractionResult) -> str: - """ - Determine counter type from interaction metadata. - - Args: - interaction: Interaction result - - Returns: - Counter type string - """ - metadata = interaction.metadata or {} - - if 'colors_a' in metadata: - return 'color' - elif 'power_a' in metadata: - return 'stats' - else: - return 'keyword' - - def _determine_evolution_type(self, interaction: InteractionResult) -> str: - """ - Determine evolution type from interaction metadata. - - Args: - interaction: Interaction result - - Returns: - Evolution type string - """ - metadata = interaction.metadata or {} - - if 'card_name' in metadata: - return 'reprint' - else: - return 'evolution' - - def store_interactions(self, interactions: dict) -> Tuple[int, int]: - """ - Store interactions in the database. - - Args: - interactions: Dictionary with synergies, counters, evolutions - - Returns: - Tuple of (stored_count, review_queue_count) - """ - db = self.SessionLocal() - stored = 0 - review_queue = 0 - - try: - # Store synergies - for interaction in interactions['synergies']: - if interaction.confidence >= self.min_confidence: - synergy_type = self._determine_synergy_type(interaction) - db.execute(text(""" - INSERT INTO mtg_card_synergies ( - card_a_id, card_b_id, synergy_type, strength, notes, confidence - ) VALUES ( - :card_a, :card_b, :synergy_type, :strength, :notes, :confidence - ) ON CONFLICT DO NOTHING - """), { - "card_a": interaction.card_a_id, - "card_b": interaction.card_b_id, - "synergy_type": synergy_type, - "strength": interaction.strength, - "notes": interaction.notes, - "confidence": interaction.confidence, - }) - stored += 1 - else: - review_queue += 1 - - # Store counters - for interaction in interactions['counters']: - if interaction.confidence >= self.min_confidence: - counter_type = self._determine_counter_type(interaction) - db.execute(text(""" - INSERT INTO mtg_card_counters ( - card_a_id, card_b_id, counter_type, strength, notes, confidence - ) VALUES ( - :card_a, :card_b, :counter_type, :strength, :notes, :confidence - ) ON CONFLICT DO NOTHING - """), { - "card_a": interaction.card_a_id, - "card_b": interaction.card_b_id, - "counter_type": counter_type, - "strength": interaction.strength, - "notes": interaction.notes, - "confidence": interaction.confidence, - }) - stored += 1 - else: - review_queue += 1 - - # Store evolutions - for interaction in interactions['evolutions']: - if interaction.confidence >= self.min_confidence: - evolution_type = self._determine_evolution_type(interaction) - db.execute(text(""" - INSERT INTO mtg_card_evolution ( - card_id, evolved_card_id, evolution_type, strength, notes, confidence - ) VALUES ( - :card_id, :evolved_card_id, :evolution_type, :strength, :notes, :confidence - ) ON CONFLICT DO NOTHING - """), { - "card_id": interaction.card_a_id, - "evolved_card_id": interaction.card_b_id, - "evolution_type": evolution_type, - "strength": interaction.strength, - "notes": interaction.notes, - "confidence": interaction.confidence, - }) - stored += 1 - else: - review_queue += 1 - - db.commit() - logger.info(f"Stored {stored} interactions, {review_queue} sent to review queue") - - except Exception as e: - db.rollback() - logger.error(f"Error storing interactions: {e}") - self.stats['errors'] += 1 - finally: - db.close() - - return stored, review_queue - - def update_interaction_stats(self): - """Update interaction statistics for all cards.""" - db = self.SessionLocal() - try: - # Delete existing stats - db.execute(text("DELETE FROM mtg_card_interaction_stats")) - - # Recalculate stats - db.execute(text(""" - INSERT INTO mtg_card_interaction_stats ( - card_id, total_synergies, total_counters, total_evolutions, - total_synergy_strength, avg_synergy_strength - ) - SELECT - c.id, - COALESCE(synergies.synergy_count, 0), - COALESCE(counters.counter_count, 0), - COALESCE(evolution.evolution_count, 0), - COALESCE(synergies.total_strength, 0), - COALESCE(synergies.avg_strength, 0) - FROM mtg_cards c - LEFT JOIN ( - SELECT card_a_id as card_id, COUNT(*) as synergy_count, - SUM(strength) as total_strength, - AVG(strength) as avg_strength - FROM mtg_card_synergies - GROUP BY card_a_id - ) synergies ON c.id = synergies.card_id - LEFT JOIN ( - SELECT card_a_id as card_id, COUNT(*) as counter_count - FROM mtg_card_counters - GROUP BY card_a_id - ) counters ON c.id = counters.card_id - LEFT JOIN ( - SELECT card_id as card_id, COUNT(*) as evolution_count - FROM mtg_card_evolution - GROUP BY card_id - ) evolution ON c.id = evolution.card_id - """)) - - db.commit() - logger.info("Updated interaction statistics") - - except Exception as e: - db.rollback() - logger.error(f"Error updating interaction stats: {e}") - self.stats['errors'] += 1 - finally: - db.close() - - def run_initial_load(self, set_code: Optional[str] = None): - """ - Run initial load for all cards or a specific set. - - This is used for the first time data is loaded into the database. - - Args: - set_code: Optional set code to process - """ - logger.info("=" * 60) - logger.info("Starting Initial Load") - logger.info("=" * 60) - - # Load all cards - all_cards = self.load_cards_from_db(set_code) - - if not all_cards: - logger.warning("No cards found in database") - return - - # Extract profiles - logger.info(f"Extracting profiles for {len(all_cards)} cards...") - profiles = self.extract_profiles(all_cards) - - # Determine interactions - logger.info(f"Determining interactions for {len(profiles)} cards...") - interactions = self.determine_interactions(profiles) - - logger.info( - f"Determined {len(interactions['synergies'])} synergies, " - f"{len(interactions['counters'])} counters, " - f"{len(interactions['evolutions'])} evolutions" - ) - - # Store interactions - stored, review_queue = self.store_interactions(interactions) - - # Update statistics - self.update_interaction_stats() - - # Update stats - self.stats['cards_processed'] = len(all_cards) - self.stats['interactions_determined'] = len(interactions['synergies']) + len(interactions['counters']) + len(interactions['evolutions']) - self.stats['interactions_stored'] = stored - self.stats['interactions_review_queue'] = review_queue - - logger.info("=" * 60) - logger.info(f"Initial Load Complete") - logger.info(f" Cards processed: {self.stats['cards_processed']}") - logger.info(f" Interactions determined: {self.stats['interactions_determined']}") - logger.info(f" Interactions stored: {self.stats['interactions_stored']}") - logger.info(f" Interactions in review queue: {self.stats['interactions_review_queue']}") - logger.info("=" * 60) - - def run_rolling_update(self, new_cards: List[Dict], set_code: Optional[str] = None): - """ - Run rolling update for new cards. - - This is used when new cards are added via MTGJSON updates. - - Args: - new_cards: List of new card dictionaries - set_code: Optional set code - """ - logger.info("=" * 60) - logger.info("Starting Rolling Update") - logger.info(f"New cards: {len(new_cards)}") - logger.info("=" * 60) - - # Load existing cards - existing_cards = self.load_cards_from_db(set_code) - - # Combine existing and new cards - all_cards = existing_cards + new_cards - - # Extract profiles - logger.info(f"Extracting profiles for {len(all_cards)} cards...") - profiles = self.extract_profiles(all_cards) - - # Determine interactions - logger.info(f"Determining interactions for {len(profiles)} cards...") - interactions = self.determine_interactions(profiles) - - logger.info( - f"Determined {len(interactions['synergies'])} synergies, " - f"{len(interactions['counters'])} counters, " - f"{len(interactions['evolutions'])} evolutions" - ) - - # Store interactions - stored, review_queue = self.store_interactions(interactions) - - # Update statistics - self.update_interaction_stats() - - # Update stats - self.stats['cards_processed'] = len(new_cards) - self.stats['interactions_determined'] = len(interactions['synergies']) + len(interactions['counters']) + len(interactions['evolutions']) - self.stats['interactions_stored'] = stored - self.stats['interactions_review_queue'] = review_queue - - logger.info("=" * 60) - logger.info(f"Rolling Update Complete") - logger.info(f" New cards processed: {self.stats['cards_processed']}") - logger.info(f" Interactions determined: {self.stats['interactions_determined']}") - logger.info(f" Interactions stored: {self.stats['interactions_stored']}") - logger.info(f" Interactions in review queue: {self.stats['interactions_review_queue']}") - logger.info("=" * 60) - - def get_pipeline_stats(self) -> Dict: - """Get pipeline statistics.""" - return { - **self.stats, - 'timestamp': datetime.now().isoformat(), - } - - def close(self): - """Close database connection.""" - self.engine.dispose() - - -def main(): - """Main entry point for pipeline execution.""" - import sys - - # Database URL from environment or default - db_url = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata" - - # Get command line arguments - if len(sys.argv) < 2: - print("Usage: python pipeline.py [initial|rolling] [set_code]") - print(" initial: Run initial load for all cards or a specific set") - print(" rolling: Run rolling update for new cards (requires JSON input)") - sys.exit(1) - - command = sys.argv[1] - set_code = sys.argv[2] if len(sys.argv) > 2 else None - - # Initialize pipeline - pipeline = MTGInteractionPipeline(db_url) - - try: - if command == "initial": - pipeline.run_initial_load(set_code) - elif command == "rolling": - # Read new cards from stdin (JSON) - new_cards = json.loads(sys.stdin.read()) - pipeline.run_rolling_update(new_cards, set_code) - else: - print(f"Unknown command: {command}") - sys.exit(1) - - # Print statistics - stats = pipeline.get_pipeline_stats() - print("\nPipeline Statistics:") - for key, value in stats.items(): - print(f" {key}: {value}") - - finally: - pipeline.close() - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/interaction_recommender.py b/backend/scripts/interaction_recommender.py deleted file mode 100644 index dba1695..0000000 --- a/backend/scripts/interaction_recommender.py +++ /dev/null @@ -1,387 +0,0 @@ -""" -MTG Card Interaction Recommender - -Generates card recommendations based on interaction data. -Provides synergy suggestions, archetype cards, and card-like-this recommendations. -""" -from typing import List, Dict, Optional -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker - -from card_profile_extractor import CardProfileExtractor -from interaction_determinator import InteractionDeterminator - - -class InteractionRecommender: - """ - Generates card recommendations based on interaction data. - - Provides: - - Synergy recommendations for a specific card - - Archetype cards for a given archetype - - Similar cards based on profiles - - Deck building suggestions - """ - - def __init__(self, db_url: str): - """ - Initialize the recommender. - - Args: - db_url: PostgreSQL database URL - """ - self.db_url = db_url - self.engine = create_engine(db_url) - self.SessionLocal = sessionmaker(bind=self.engine) - self.profile_extractor = CardProfileExtractor() - self.determinator = InteractionDeterminator() - - def get_card_profile(self, card_id: int) -> Optional[CardProfile]: - """ - Get a card profile from the database. - - Args: - card_id: Card ID - - Returns: - CardProfile or None if not found - """ - db = self.SessionLocal() - try: - query = text(""" - SELECT c.*, s.code as set_code - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.id = :card_id - """) - - result = db.execute(query, {"card_id": card_id}).fetchone() - - if result: - card_data = dict(result._mapping) - # Extract profile from raw data - profile = self.profile_extractor.extract_profile(card_data) - return profile - return None - finally: - db.close() - - def get_card_by_id(self, card_id: int) -> Optional[Dict]: - """ - Get raw card data from the database. - - Args: - card_id: Card ID - - Returns: - Card dictionary or None if not found - """ - db = self.SessionLocal() - try: - query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.id = :card_id - """) - - result = db.execute(query, {"card_id": card_id}).fetchone() - - if result: - return dict(result._mapping) - return None - finally: - db.close() - - def get_synergies_for_card(self, card_id: int) -> List[Dict]: - """ - Get synergy data for a card from the database. - - Args: - card_id: Card ID - - Returns: - List of synergy dictionaries - """ - db = self.SessionLocal() - try: - query = text(""" - SELECT cs.*, - ca.name as card_a_name, ca.type_line as card_a_type_line, - cb.name as card_b_name, cb.type_line as card_b_type_line - FROM mtg_card_synergies cs - JOIN mtg_cards ca ON cs.card_a_id = ca.id - JOIN mtg_cards cb ON cs.card_b_id = cb.id - WHERE cs.card_a_id = :card_id OR cs.card_b_id = :card_id - ORDER BY cs.strength DESC, cs.confidence DESC - LIMIT :limit - """) - - results = db.execute(query, { - "card_id": card_id, - "limit": 100 - }).fetchall() - - return [dict(row._mapping) for row in results] - finally: - db.close() - - def get_counters_for_card(self, card_id: int) -> List[Dict]: - """ - Get counter data for a card from the database. - - Args: - card_id: Card ID - - Returns: - List of counter dictionaries - """ - db = self.SessionLocal() - try: - query = text(""" - SELECT cc.*, - ca.name as card_a_name, ca.type_line as card_a_type_line, - cb.name as card_b_name, cb.type_line as card_b_type_line - FROM mtg_card_counters cc - JOIN mtg_cards ca ON cc.card_a_id = ca.id - JOIN mtg_cards cb ON cc.card_b_id = cb.id - WHERE cc.card_a_id = :card_id OR cc.card_b_id = :card_id - ORDER BY cc.strength DESC, cc.confidence DESC - LIMIT :limit - """) - - results = db.execute(query, { - "card_id": card_id, - "limit": 100 - }).fetchall() - - return [dict(row._mapping) for row in results] - finally: - db.close() - - def recommend_synergies(self, card_id: int, max_results: int = 20) -> List[Dict]: - """ - Recommend cards that synergize with a given card. - - Args: - card_id: Card ID to find synergies for - max_results: Maximum number of recommendations - - Returns: - List of recommendation dictionaries - """ - synergies = self.get_synergies_for_card(card_id) - - recommendations = [] - seen_cards = set() - - for synergy in synergies: - # Determine which card is the "other" card - if synergy['card_a_id'] == card_id: - other_card_id = synergy['card_b_id'] - other_card_name = synergy['card_b_name'] - other_card_type = synergy['card_b_type_line'] - else: - other_card_id = synergy['card_a_id'] - other_card_name = synergy['card_a_name'] - other_card_type = synergy['card_a_type_line'] - - # Skip if already seen - if other_card_id in seen_cards: - continue - seen_cards.add(other_card_id) - - recommendations.append({ - 'card_id': other_card_id, - 'card_name': other_card_name, - 'card_type_line': other_card_type, - 'synergy_type': synergy['synergy_type'], - 'strength': synergy['strength'], - 'confidence': synergy['confidence'], - 'notes': synergy['notes'], - 'recommendation_type': 'synergy', - }) - - # Sort by strength and confidence - recommendations.sort(key=lambda r: (r['strength'], r['confidence']), reverse=True) - - return recommendations[:max_results] - - def recommend_archetype_cards(self, archetype: str, max_results: int = 20) -> List[Dict]: - """ - Recommend cards that fit a specific archetype. - - Args: - archetype: Archetype name (e.g., 'goblin', 'elf') - max_results: Maximum number of recommendations - - Returns: - List of recommendation dictionaries - """ - db = self.SessionLocal() - try: - query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.subtypes LIKE :archetype - ORDER BY c.id - LIMIT :limit - """) - - results = db.execute(query, { - "archetype": f"%{archetype}%", - "limit": max_results - }).fetchall() - - recommendations = [] - for result in results: - card_data = dict(result._mapping) - recommendations.append({ - 'card_id': card_data['id'], - 'card_name': card_data['name'], - 'card_type_line': card_data['type_line'], - 'set_code': card_data['set_code'], - 'set_name': card_data['set_name'], - 'recommendation_type': 'archetype', - 'archetype': archetype, - 'confidence': 0.8, - 'notes': f"Matches {archetype} archetype", - }) - - return recommendations - finally: - db.close() - - def recommend_similar_cards(self, card_id: int, max_results: int = 20) -> List[Dict]: - """ - Recommend cards similar to a given card. - - Args: - card_id: Card ID to find similar cards for - max_results: Maximum number of recommendations - - Returns: - List of recommendation dictionaries - """ - card_data = self.get_card_by_id(card_id) - - if not card_data: - return [] - - profile = self.profile_extractor.extract_profile(card_data) - - # Get similar cards based on archetype and mechanics - db = self.SessionLocal() - try: - recommendations = [] - seen_cards = set() - - # Get cards with matching archetypes - if profile.archetypes: - for archetype in profile.archetypes: - query = text(""" - SELECT c.*, s.code as set_code - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.subtypes LIKE :archetype - AND c.id != :card_id - LIMIT :limit - """) - - results = db.execute(query, { - "archetype": f"%{archetype}%", - "card_id": card_id, - "limit": max_results * 2 - }).fetchall() - - for result in results: - card = dict(result._mapping) - if card['id'] not in seen_cards: - seen_cards.add(card['id']) - recommendations.append({ - 'card_id': card['id'], - 'card_name': card['name'], - 'card_type_line': card['type_line'], - 'set_code': card['set_code'], - 'recommendation_type': 'similar', - 'reason': f"Same archetype: {archetype}", - 'confidence': 0.7, - }) - - # Get cards with matching mechanics - if profile.mechanics: - for mechanic in profile.mechanics[:3]: # Limit to top 3 mechanics - query = text(""" - SELECT c.*, s.code as set_code - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.oracle_text LIKE :mechanic - AND c.id != :card_id - LIMIT :limit - """) - - results = db.execute(query, { - "mechanic": f"%{mechanic}%", - "card_id": card_id, - "limit": max_results - }).fetchall() - - for result in results: - card = dict(result._mapping) - if card['id'] not in seen_cards: - seen_cards.add(card['id']) - recommendations.append({ - 'card_id': card['id'], - 'card_name': card['name'], - 'card_type_line': card['type_line'], - 'set_code': card['set_code'], - 'recommendation_type': 'similar', - 'reason': f"Has mechanic: {mechanic}", - 'confidence': 0.6, - }) - - # Sort by confidence - recommendations.sort(key=lambda r: r['confidence'], reverse=True) - - return recommendations[:max_results] - finally: - db.close() - - def get_deck_recommendations(self, card_id: int, max_results: int = 10) -> Dict: - """ - Get deck building recommendations for a card. - - Args: - card_id: Card ID - max_results: Maximum number of recommendations - - Returns: - Dictionary with synergy cards, archetype cards, and similar cards - """ - # Get synergy cards - synergy_cards = self.recommend_synergies(card_id, max_results) - - # Get archetype cards - card_data = self.get_card_by_id(card_id) - archetype_cards = [] - - if card_data and card_data.get('subtypes'): - # Extract first archetype - archetypes = [a.strip() for a in card_data['subtypes'].split(',')] - if archetypes: - archetype_cards = self.recommend_archetype_cards(archetypes[0], max_results) - - # Get similar cards - similar_cards = self.recommend_similar_cards(card_id, max_results) - - return { - 'synergy_cards': synergy_cards, - 'archetype_cards': archetype_cards, - 'similar_cards': similar_cards, - 'total_recommendations': len(synergy_cards) + len(archetype_cards) + len(similar_cards), - } - - def close(self): - """Close database connection.""" - self.engine.dispose() diff --git a/backend/scripts/interaction_schema.py b/backend/scripts/interaction_schema.py deleted file mode 100644 index f9dc402..0000000 --- a/backend/scripts/interaction_schema.py +++ /dev/null @@ -1,190 +0,0 @@ -""" -MTG Card Interaction Database Schema - -Defines the database schema for storing card interactions. -Includes tables for synergies, counters, evolutions, and statistics. -""" -from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey, UniqueConstraint -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship -from datetime import datetime - -Base = declarative_base() - - -class CardSynergy(Base): - """ - Synergy between two cards. - - Synergies are positive interactions where cards work well together. - Examples: Archetype support, mechanic combos, mana base compatibility. - """ - __tablename__ = 'mtg_card_synergies' - - id = Column(Integer, primary_key=True, autoincrement=True) - card_a_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - card_b_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - synergy_type = Column(String(50), nullable=False) # 'archetype', 'mechanic', 'mana', 'combo' - strength = Column(Integer, nullable=False) # 1-5 (1=weak, 5=strong) - notes = Column(String(500), nullable=True) - confidence = Column(Float, nullable=False, default=0.8) - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - - # Unique constraint to prevent duplicates - __table_args__ = ( - UniqueConstraint('card_a_id', 'card_b_id', 'synergy_type', name='uq_synergy_pair_type'), - ) - - # Relationships - card_a = relationship('MtgCard', foreign_keys=[card_a_id]) - card_b = relationship('MtgCard', foreign_keys=[card_b_id]) - - def __repr__(self): - return f"" - - -class CardCounter(Base): - """ - Counter relationship between two cards. - - Counters are negative interactions where one card is disadvantaged by another. - Examples: Different color identities, outclassed stats, countered by specific spells. - """ - __tablename__ = 'mtg_card_counters' - - id = Column(Integer, primary_key=True, autoincrement=True) - card_a_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - card_b_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - counter_type = Column(String(50), nullable=False) # 'color', 'stats', 'spell', 'keyword' - strength = Column(Integer, nullable=False) # 1-5 (1=weak, 5=strong) - notes = Column(String(500), nullable=True) - confidence = Column(Float, nullable=False, default=0.7) - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - - # Unique constraint to prevent duplicates - __table_args__ = ( - UniqueConstraint('card_a_id', 'card_b_id', 'counter_type', name='uq_counter_pair_type'), - ) - - # Relationships - card_a = relationship('MtgCard', foreign_keys=[card_a_id]) - card_b = relationship('MtgCard', foreign_keys=[card_b_id]) - - def __repr__(self): - return f"" - - -class CardEvolution(Base): - """ - Evolution relationship for a card. - - Evolutions track when a card has been reprinted, transformed, or evolved. - Examples: Same name in different sets, transform pairs, double-sided cards. - """ - __tablename__ = 'mtg_card_evolution' - - id = Column(Integer, primary_key=True, autoincrement=True) - card_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - evolved_card_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False) - evolution_type = Column(String(50), nullable=False) # 'reprint', 'transform', 'double_sided' - strength = Column(Integer, nullable=False) # 1-5 (1=weak, 5=strong) - notes = Column(String(500), nullable=True) - confidence = Column(Float, nullable=False, default=0.9) - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - - # Unique constraint to prevent duplicates - __table_args__ = ( - UniqueConstraint('card_id', 'evolved_card_id', 'evolution_type', name='uq_evolution_pair_type'), - ) - - # Relationships - card = relationship('MtgCard', foreign_keys=[card_id]) - evolved_card = relationship('MtgCard', foreign_keys=[evolved_card_id]) - - def __repr__(self): - return f"" - - -class CardInteractionStats(Base): - """ - Aggregated interaction statistics for a card. - - Tracks total interactions, average strength, and primary archetypes/themes. - """ - __tablename__ = 'mtg_card_interaction_stats' - - id = Column(Integer, primary_key=True, autoincrement=True) - card_id = Column(Integer, ForeignKey('mtg_cards.id'), nullable=False, unique=True) - - # Interaction counts - total_synergies = Column(Integer, nullable=False, default=0) - total_counters = Column(Integer, nullable=False, default=0) - total_evolutions = Column(Integer, nullable=False, default=0) - total_partners = Column(Integer, nullable=False, default=0) # Cards that partner well - - # Mechanic/archetype counts - total_mechanics = Column(Integer, nullable=False, default=0) - total_archetypes = Column(Integer, nullable=False, default=0) - total_themes = Column(Integer, nullable=False, default=0) - - # Synergy strength metrics - avg_synergy_strength = Column(Float, nullable=False, default=0.0) - max_synergy_strength = Column(Integer, nullable=False, default=0) - - # Primary archetype and theme - primary_archetype = Column(String(50), nullable=True) - primary_theme = Column(String(50), nullable=True) - - # Metadata - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - - def __repr__(self): - return f"" - - -def create_interaction_tables(engine): - """ - Create all interaction tables in the database. - - Args: - engine: SQLAlchemy engine - """ - Base.metadata.create_all(engine) - print("✅ Interaction tables created successfully") - - -def drop_interaction_tables(engine): - """ - Drop all interaction tables from the database. - - Args: - engine: SQLAlchemy engine - """ - Base.metadata.drop_all(engine) - print("✅ Interaction tables dropped successfully") - - -if __name__ == "__main__": - # Example usage - from dotenv import load_dotenv - import os - - load_dotenv() - - db_url = os.getenv('MTG_DATABASE_URL', 'postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata') - engine = create_engine(db_url) - - # Create tables - create_interaction_tables(engine) - - # Print table names - from sqlalchemy import inspect - inspector = inspect(engine) - print("\n📊 Tables created:") - for table in inspector.get_table_names(): - if 'mtg_card_' in table: - print(f" - {table}") diff --git a/backend/scripts/investigate_sets.py b/backend/scripts/investigate_sets.py deleted file mode 100644 index 1022bab..0000000 --- a/backend/scripts/investigate_sets.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -""" -Investigate MTG sets endpoint and image column. -Checks database schema, MTGJSON data structure, and API responses. -""" - -import asyncio -import os -import sys -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker -from sqlalchemy import text - -# Add project root to path -sys.path.insert(0, '/home/wall-o/projects/mtgonline/backend') - -from app.core.settings import get_settings -from app.models.mtg_models import MtgSet, MtgCard - - -async def main(): - """Investigate the current state.""" - settings = get_settings() - - print("=== DATABASE CONNECTION ===") - print(f"MTG DB URL: {settings.MTG_DATABASE_URL}") - print() - - engine = create_async_engine(settings.MTG_DATABASE_URL) - async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - async with async_session() as session: - # Check mtg_sets table schema - print("=== MTG_SETS TABLE SCHEMA ===") - result = await session.execute(text(""" - SELECT column_name, data_type, is_nullable - FROM information_schema.columns - WHERE table_name = 'mtg_sets' - ORDER BY ordinal_position - """)) - for row in result.fetchall(): - print(f" {row[0]}: {row[1]} (nullable: {row[2]})") - - print() - print("=== SAMPLE SET DATA ===") - result = await session.execute(text(""" - SELECT code, name, type, release_date, base_set_size, total_size, - is_foil_only, is_non_foil_only, digital, icon_svg_url, - parent_code, mtgo_code - FROM mtg_sets - LIMIT 1 - """)) - row = result.fetchone() - if row: - cols = ['code', 'name', 'type', 'release_date', 'base_set_size', 'total_size', - 'is_foil_only', 'is_non_foil_only', 'digital', 'icon_svg_url', - 'parent_code', 'mtgo_code'] - for col, val in zip(cols, row): - print(f" {col}: {val}") - - print() - print("=== SET COUNT ===") - result = await session.execute(text("SELECT COUNT(*) FROM mtg_sets")) - count = result.scalar() - print(f" Total sets: {count}") - - print() - print("=== IMAGE URL CHECK ===") - result = await session.execute(text(""" - SELECT COUNT(*) FROM mtg_sets - WHERE image_url IS NOT NULL AND image_url != '' - """)) - count = result.scalar() - print(f" Sets with image_url: {count}") - - print() - print("=== CHECKING FOR image_url COLUMN ===") - result = await session.execute(text(""" - SELECT column_name FROM information_schema.columns - WHERE table_name = 'mtg_sets' AND column_name LIKE '%image%' - """)) - image_cols = [row[0] for row in result.fetchall()] - print(f" Image-related columns: {image_cols}") - - await engine.dispose() - - # Check MTGJSON data structure - print() - print("=== MTGJSON SET SCHEMA REFERENCE ===") - print("MTGJSON set.json fields related to images:") - print(" - image: Object with 'normal' and 'large' URLs") - print(" - image_png: URL to PNG image") - print(" - image_png_small: URL to small PNG image") - print(" - icon_svg_url: SVG icon URL") - print(" - symbol: Symbol image URL") - print(" - logo: Logo image URL") - - print() - print("=== CONCLUSION ===") - print("The mtg_sets table needs an image_url column to store") - print("the normal-sized image URL from MTGJSON set data.") - print() - print("Steps needed:") - print("1. Add image_url column to mtg_sets table") - print("2. Update MtgSet model") - print("3. Update refresh_mtg.py to fetch image_url from set.json") - print("4. Update get_sets() and get_set_by_code() to return image_url") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/backend/scripts/migrate_complete.py b/backend/scripts/migrate_complete.py deleted file mode 100644 index eb59f02..0000000 --- a/backend/scripts/migrate_complete.py +++ /dev/null @@ -1,871 +0,0 @@ -""" -MTGJSON Database Migration - Fixed Version - -This script: -1. Adds all MTGJSON columns to mtg_cards and mtg_sets tables -2. Populates them from existing JSON data -3. Creates the card interaction graph tables -4. Populates the interaction graph from existing data -5. Creates sample interaction data to demonstrate the system -""" -from sqlalchemy import create_engine, text -import json - -DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata" - - -class MTGJSONFullMigration: - """Complete migration for MTGJSON schema and card interaction graph.""" - - def __init__(self): - self.engine = create_engine(DB_URL) - self.conn = None - - def connect(self): - """Connect to database.""" - self.conn = self.engine.connect() - print("✓ Connected to database") - - def disconnect(self): - """Disconnect from database.""" - if self.conn: - self.conn.close() - self.engine.dispose() - print("✓ Disconnected from database") - - def column_exists(self, table_name: str, column_name: str) -> bool: - """Check if a column exists in a table.""" - result = self.conn.execute(text(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = :table AND column_name = :column - """), {"table": table_name, "column": column_name}) - return result.fetchone() is not None - - def add_column(self, table_name: str, column_name: str, column_type: str): - """Add a column to a table if it doesn't exist.""" - if not self.column_exists(table_name, column_name): - self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}")) - print(f" ✓ Added: {table_name}.{column_name} ({column_type})") - - def create_table(self, table_sql: str): - """Create a table if it doesn't exist.""" - self.conn.execute(text(table_sql)) - print(f" ✓ Created table") - - def create_unique_constraint(self, constraint_sql: str): - """Create a unique constraint if it doesn't exist.""" - try: - self.conn.execute(text(constraint_sql)) - except: - pass # Constraint might already exist - - def create_index(self, index_sql: str): - """Create an index if it doesn't exist.""" - self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql}")) - print(f" ✓ Created index: {index_sql.split(' ON ')[1].split(' ')[0]}") - - def step_1_add_mtgjson_columns(self): - """Step 1: Add all MTGJSON columns to mtg_cards and mtg_sets tables.""" - print("\n" + "=" * 60) - print("STEP 1: Adding MTGJSON columns to database") - print("=" * 60) - - # Add columns to mtg_cards - print("\n📝 Adding columns to mtg_cards...") - - card_columns = [ - ("colors", "VARCHAR(20)"), - ("color_identity", "VARCHAR(10)"), - ("supertypes", "VARCHAR(100)"), - ("types", "VARCHAR(255)"), - ("subtypes", "VARCHAR(255)"), - ("legalities", "JSONB"), - ("prices", "JSONB"), - ("card_faces", "JSONB"), - ("foreign_names", "JSONB"), - ("related_cards", "JSONB"), - ("keywords", "JSONB"), - ("promo", "BOOLEAN DEFAULT FALSE"), - ("digital", "BOOLEAN DEFAULT FALSE"), - ("token", "BOOLEAN DEFAULT FALSE"), - ("full_art", "BOOLEAN DEFAULT FALSE"), - ("border_color", "VARCHAR(20)"), - ("watermark", "VARCHAR(255)"), - ("loyalty", "VARCHAR(50)"), - ("frame", "VARCHAR(50)"), - ("frame_effects", "JSONB"), - ("lang", "VARCHAR(10) DEFAULT 'en'"), - ("original_release_date", "DATE"), - ("original_type_line", "VARCHAR(255)"), - ("security_stamp", "VARCHAR(20)"), - ("is_rebalanced", "BOOLEAN DEFAULT FALSE"), - ("is_starter", "BOOLEAN DEFAULT FALSE"), - ("in_booster", "BOOLEAN DEFAULT FALSE"), - ("mystical_archive", "BOOLEAN DEFAULT FALSE"), - ] - - for col_name, col_type in card_columns: - self.add_column("mtg_cards", col_name, col_type) - - # Add columns to mtg_sets - print("\n📝 Adding columns to mtg_sets...") - - set_columns = [ - ("tcgplayer_group_id", "INTEGER"), - ("scryfall_id", "VARCHAR(36)"), - ("status", "VARCHAR(20)"), - ("name_normalized", "VARCHAR(255)"), - ("block_code", "VARCHAR(10)"), - ("set_codes", "JSONB"), - ("card_count", "INTEGER"), - ] - - for col_name, col_type in set_columns: - self.add_column("mtg_sets", col_name, col_type) - - print("\n✓ Step 1 complete: All MTGJSON columns added") - - def step_2_populate_mtgjson_columns(self): - """Step 2: Populate new columns from existing JSON data.""" - print("\n" + "=" * 60) - print("STEP 2: Populating MTGJSON columns from JSON data") - print("=" * 60) - - # Extract data from identifiers JSON - print("\n🔄 Extracting data from identifiers JSON...") - - self.conn.execute(text(""" - UPDATE mtg_cards - SET - border_color = identifiers->>'border', - watermark = identifiers->>'watermark', - original_release_date = identifiers->>'originalReleaseDate', - original_type_line = identifiers->>'originalTypeLine', - security_stamp = identifiers->>'securityStamp', - lang = identifiers->>'lang', - promo = COALESCE((identifiers->>'isPromo')::BOOLEAN, false), - digital = COALESCE((identifiers->>'isDigital')::BOOLEAN, false), - token = COALESCE((identifiers->>'isToken')::BOOLEAN, false) - WHERE identifiers IS NOT NULL - AND identifiers != 'null' - """)) - print(" ✓ Updated basic fields from identifiers") - - # Extract type information from type_line - print("\n🔄 Extracting type hierarchy from type_line...") - - self.conn.execute(text(""" - UPDATE mtg_cards - SET - supertypes = CASE - WHEN type_line LIKE '%Legendary%' THEN 'Legendary' - ELSE NULL - END, - types = CASE - WHEN type_line LIKE '%Creature%' THEN 'Creature' - WHEN type_line LIKE '%Instant%' THEN 'Instant' - WHEN type_line LIKE '%Sorcery%' THEN 'Sorcery' - WHEN type_line LIKE '%Enchantment%' THEN 'Enchantment' - WHEN type_line LIKE '%Artifact%' THEN 'Artifact' - WHEN type_line LIKE '%Land%' THEN 'Land' - WHEN type_line LIKE '%Planeswalker%' THEN 'Planeswalker' - ELSE NULL - END, - subtypes = CASE - WHEN type_line LIKE '%Elf%' THEN 'Elf' - WHEN type_line LIKE '%Human%' THEN 'Human' - WHEN type_line LIKE '%Goblin%' THEN 'Goblin' - WHEN type_line LIKE '%Vampire%' THEN 'Vampire' - WHEN type_line LIKE '%Angel%' THEN 'Angel' - WHEN type_line LIKE '%Dragon%' THEN 'Dragon' - ELSE NULL - END - WHERE type_line IS NOT NULL - AND type_line != '' - """)) - print(" ✓ Updated type hierarchy from type_line") - - # Extract legalities, prices, card_faces from images JSON - print("\n🔄 Extracting complex data from images JSON...") - - self.conn.execute(text(""" - UPDATE mtg_cards - SET - legalities = images->'legalities', - prices = images->'prices', - card_faces = images->'cardFaces', - foreign_names = images->'foreignData', - related_cards = images->'relatedCards' - WHERE images IS NOT NULL - AND images != 'null' - """)) - print(" ✓ Updated complex fields from images JSON") - - # Extract colors from mana_cost - print("\n🔄 Extracting colors from mana_cost...") - - self.conn.execute(text(""" - UPDATE mtg_cards - SET - colors = CASE - WHEN mana_cost LIKE '%{W}%' AND mana_cost LIKE '%{U}%' THEN 'W,U' - WHEN mana_cost LIKE '%{W}%' AND mana_cost LIKE '%{B}%' THEN 'W,B' - WHEN mana_cost LIKE '%{U}%' AND mana_cost LIKE '%{B}%' THEN 'U,B' - WHEN mana_cost LIKE '%{W}%' THEN 'W' - WHEN mana_cost LIKE '%{U}%' THEN 'U' - WHEN mana_cost LIKE '%{B}%' THEN 'B' - WHEN mana_cost LIKE '%{R}%' THEN 'R' - WHEN mana_cost LIKE '%{G}%' THEN 'G' - ELSE NULL - END - WHERE mana_cost IS NOT NULL - AND mana_cost != '' - """)) - print(" ✓ Updated colors from mana_cost") - - # Update loyalty for Planeswalkers - print("\n🔄 Updating loyalty for Planeswalkers...") - - self.conn.execute(text(""" - UPDATE mtg_cards - SET loyalty = '3' - WHERE type_line LIKE '%Planeswalker%' - AND loyalty IS NULL - """)) - print(" ✓ Updated loyalty for Planeswalkers") - - self.conn.commit() - print("\n✓ Step 2 complete: All columns populated") - - def step_3_create_interaction_graph(self): - """Step 3: Create card interaction graph tables.""" - print("\n" + "=" * 60) - print("STEP 3: Creating card interaction graph") - print("=" * 60) - - # Card mechanics table - print("\n📊 Creating mtg_card_mechanics table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_mechanics ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - mechanic VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, mechanic) - ) - """) - indexes = [ - "idx_mechanics_card_id ON mtg_card_mechanics(card_id)", - "idx_mechanics_mechanic ON mtg_card_mechanics(mechanic)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card mechanics table created") - - # Card archetypes table - print("\n📊 Creating mtg_card_archetypes table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_archetypes ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - archetype VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, archetype) - ) - """) - indexes = [ - "idx_archetypes_card_id ON mtg_card_archetypes(card_id)", - "idx_archetypes_archetype ON mtg_card_archetypes(archetype)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card archetypes table created") - - # Card themes table - print("\n📊 Creating mtg_card_themes table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_themes ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - theme VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, theme) - ) - """) - indexes = [ - "idx_themes_card_id ON mtg_card_themes(card_id)", - "idx_themes_theme ON mtg_card_themes(theme)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card themes table created") - - # Card relationships table - print("\n📊 Creating mtg_card_relationships table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_relationships ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - relationship_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, relationship_type) - ) - """) - indexes = [ - "idx_relationships_card_a ON mtg_card_relationships(card_a_id)", - "idx_relationships_card_b ON mtg_card_relationships(card_b_id)", - "idx_relationships_type ON mtg_card_relationships(relationship_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card relationships table created") - - # Card synergies table - print("\n📊 Creating mtg_card_synergies table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_synergies ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - synergy_type VARCHAR(50) NOT NULL, - strength INTEGER NOT NULL CHECK (strength BETWEEN 1 AND 5), - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, synergy_type) - ) - """) - indexes = [ - "idx_synergies_card_a ON mtg_card_synergies(card_a_id)", - "idx_synergies_card_b ON mtg_card_synergies(card_b_id)", - "idx_synergies_type ON mtg_card_synergies(synergy_type)", - "idx_synergies_strength ON mtg_card_synergies(strength)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card synergies table created") - - # Card counters table - print("\n📊 Creating mtg_card_counters table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_counters ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - counter_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, counter_type) - ) - """) - indexes = [ - "idx_counters_card_a ON mtg_card_counters(card_a_id)", - "idx_counters_card_b ON mtg_card_counters(card_b_id)", - "idx_counters_type ON mtg_card_counters(counter_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card counters table created") - - # Card evolution table - print("\n📊 Creating mtg_card_evolution table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_evolution ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - evolved_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - evolution_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, evolved_card_id, evolution_type) - ) - """) - indexes = [ - "idx_evolution_card_id ON mtg_card_evolution(card_id)", - "idx_evolution_evolved_id ON mtg_card_evolution(evolved_card_id)", - "idx_evolution_type ON mtg_card_evolution(evolution_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card evolution table created") - - # Card partners table - print("\n📊 Creating mtg_card_partners table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_partners ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - partnership_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, partnership_type) - ) - """) - indexes = [ - "idx_partners_card_a ON mtg_card_partners(card_a_id)", - "idx_partners_card_b ON mtg_card_partners(card_b_id)", - "idx_partners_type ON mtg_card_partners(partnership_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card partners table created") - - # Card mana relations table - print("\n📊 Creating mtg_card_mana_relations table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_mana_relations ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - land_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - mana_type VARCHAR(10) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, land_id, mana_type) - ) - """) - indexes = [ - "idx_mana_card_id ON mtg_card_mana_relations(card_id)", - "idx_mana_land_id ON mtg_card_mana_relations(land_id)", - "idx_mana_type ON mtg_card_mana_relations(mana_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card mana relations table created") - - # Card set relations table - print("\n📊 Creating mtg_card_set_relations table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_set_relations ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE, - theme VARCHAR(100) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, set_id, theme) - ) - """) - indexes = [ - "idx_setrel_card_id ON mtg_card_set_relations(card_id)", - "idx_setrel_set_id ON mtg_card_set_relations(set_id)", - "idx_setrel_theme ON mtg_card_set_relations(theme)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card set relations table created") - - # Card power relations table - print("\n📊 Creating mtg_card_power_relations table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_power_relations ( - id SERIAL PRIMARY KEY, - card_a_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - card_b_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - relation_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_a_id, card_b_id, relation_type) - ) - """) - indexes = [ - "idx_power_card_a ON mtg_card_power_relations(card_a_id)", - "idx_power_card_b ON mtg_card_power_relations(card_b_id)", - "idx_power_type ON mtg_card_power_relations(relation_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card power relations table created") - - # Card history table - print("\n📊 Creating mtg_card_history table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_history ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - related_card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - history_type VARCHAR(50) NOT NULL, - strength INTEGER DEFAULT 1, - notes TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, related_card_id, history_type) - ) - """) - indexes = [ - "idx_history_card_id ON mtg_card_history(card_id)", - "idx_history_related_id ON mtg_card_history(related_card_id)", - "idx_history_type ON mtg_card_history(history_type)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card history table created") - - # Card interaction stats table - print("\n📊 Creating mtg_card_interaction_stats table...") - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_interaction_stats ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - total_synergies INTEGER DEFAULT 0, - total_counters INTEGER DEFAULT 0, - total_evolution INTEGER DEFAULT 0, - total_partners INTEGER DEFAULT 0, - total_mechanics INTEGER DEFAULT 0, - total_archetypes INTEGER DEFAULT 0, - total_themes INTEGER DEFAULT 0, - avg_synergy_strength DECIMAL(3,2) DEFAULT 0.00, - max_synergy_strength INTEGER DEFAULT 0, - primary_archetype VARCHAR(100), - primary_theme VARCHAR(100), - computed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id) - ) - """) - indexes = [ - "idx_stats_card_id ON mtg_card_interaction_stats(card_id)", - "idx_stats_total_synergies ON mtg_card_interaction_stats(total_synergies)", - "idx_stats_primary_archetype ON mtg_card_interaction_stats(primary_archetype)", - ] - for idx in indexes: - self.create_index(idx) - print(" ✓ Card interaction stats table created") - - print("\n✓ Step 3 complete: Interaction graph tables created") - - def step_4_populate_interaction_graph(self): - """Step 4: Populate interaction graph from existing data.""" - print("\n" + "=" * 60) - print("STEP 4: Populating interaction graph from existing data") - print("=" * 60) - - # Populate mechanics from subtypes - print("\n🔄 Populating mechanics from subtypes...") - - self.conn.execute(text(""" - INSERT INTO mtg_card_mechanics (card_id, mechanic) - SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ','))) - FROM mtg_cards c - WHERE c.subtypes IS NOT NULL - AND c.subtypes != 'null' - AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN ( - 'flying', 'first_strike', 'double_strike', 'deathtouch', 'lifelink', - 'haste', 'trample', 'menace', 'vigilance', 'reach', 'indestructible', - 'hexproof', 'shroud', 'defender', 'landfall', 'delve', 'soulshift', - 'suspend', 'convoke', 'rampage', 'toxic', 'crew', 'equip', 'annihilator', - 'spectacle', 'prowess', 'aftermath', 'adapt', 'amplify', 'awaken', - 'banding', 'bestow', 'burst', 'channel', 'clash', 'crawl', 'curse', - 'day_night', 'decay', 'defiant', 'demolish', 'detain', 'detect', - 'devour', 'disguise', 'disturb', 'dome', 'double_strike', 'dredge', - 'emerge', 'encore', 'endure', 'evoke', 'evolve', 'exalted', 'exile', - 'exploit', 'extort', 'fairy', 'fanatic', 'fathom', 'fear', 'feline', - 'flash', 'flight', 'foretell', 'frenzy', 'fumble', 'galvanize', - 'gateway', 'genesis', 'graft', 'grave', 'grit', 'guardian', 'harvest', - 'healer', 'heroic', 'hideaway', 'hinterland', 'hoard', 'hour', 'illusion', - 'immortal', 'impulse', 'inspiration', 'instill', 'iron', 'junk', 'kicker', - 'knight', 'land', 'leech', 'lich', 'lifespan', 'lightning', 'living', - 'lurk', 'madness', 'manifest', 'map', 'meld', 'miracle', 'mitosis', - 'modular', 'moon', 'mother', 'morph', 'mutate', 'ninja', 'night', - 'nightmare', 'pact', 'paradox', 'persist', 'pillage', 'pivot', 'planar', - 'polar', 'pour', 'prey', 'priest', 'primer', 'probe', 'prosperity', - 'psychic', 'puppet', 'quest', 'quote', 'rage', 'raid', 'raise', 'rally', - 'rapid', 'rat', 'rebound', 'reckless', 'recoup', 'reflect', 'refresh', - 'replicate', 'reverberate', 'reviviant', 'rift', 'rip', 'ritual', 'rite', - 'rogue', 'savant', 'scavenge', 'seek', 'shadow', 'shards', 'skulk', - 'smelt', 'snap', 'snow', 'spectacle', 'splice', 'spore', 'sprawl', - 'stabilize', 'stasis', 'storm', 'story', 'substitute', 'sunder', 'surge', - 'survive', 'swarm', 'symbiosis', 'synchronized', 'synth', 'table', 'taint', - 'tank', 'thorn', 'thwart', 'time', 'tinker', 'toxin', 'trail', 'transfigure', - 'transform', 'transport', 'trouble', 'tunnel', 'unearth', 'unleash', 'unmask', - 'unstoppable', 'urborg', 'urgent', 'utility', 'vengeful', 'vanish', 'venom', - 'victory', 'villainous', 'vitalize', 'void', 'voyage', 'ward', 'watch', 'weave', - 'wed', 'whammy', 'wild', 'will', 'wisp', 'witch', 'woe', 'wounded', 'wrap', - 'wrought', 'wurm', 'wythe' - ) - ON CONFLICT DO NOTHING - """)) - print(" ✓ Populated mechanics from subtypes") - - # Populate archetypes from subtypes - print("\n🔄 Populating archetypes from subtypes...") - - self.conn.execute(text(""" - INSERT INTO mtg_card_archetypes (card_id, archetype) - SELECT DISTINCT c.id, LOWER(UNNEST(string_to_array(c.subtypes, ','))) - FROM mtg_cards c - WHERE c.subtypes IS NOT NULL - AND c.subtypes != 'null' - AND LOWER(UNNEST(string_to_array(c.subtypes, ','))) IN ( - 'goblin', 'elf', 'vampire', 'angel', 'dragon', 'human', 'zombie', - 'soldier', 'knight', 'wizard', 'spirit', 'demon', 'snake', 'cat', - 'wolf', 'bear', 'bird', 'insect', 'horror', 'goat', 'ox', 'elephant', - 'whale', 'shark', 'fish', 'serpent', 'lizard', 'scorpion', 'spider', - 'rat', 'drake', 'wyvern', 'phoenix', 'lynx', 'jaguar', 'hydra', - 'leviathan', 'kraken', 'cyclops', 'golem', 'homunculus', 'clay', - 'construct', 'myr', 'aether', 'pumpkin', 'pirate', 'pegasus', - 'unicorn', 'centaur', 'merfolk', 'mermaid', 'naga', 'satyr', 'dryad', - 'treant', 'elemental', 'fiend', 'imp', 'faerie', 'minion', 'abomination', - 'beast', 'demigod', 'god', 'avatar', 'guardian', 'warrior', 'rogue', - 'artificer', 'bard', 'monk', 'ninja', 'samurai', 'assassin', 'thief', - 'acrobat', 'explorer', 'farmer', 'myth', 'illusion', 'mirror', 'phantom', - 'shapeshifter', 'shaman', 'skeleton', 'slime', 'squirrel', 'troll', - 'tyrannosaur', 'wraith', 'wurm' - ) - ON CONFLICT DO NOTHING - """)) - print(" ✓ Populated archetypes from subtypes") - - self.conn.commit() - print("\n✓ Step 4 complete: Interaction graph populated") - - def step_5_create_sample_interactions(self): - """Step 5: Create sample interactions to demonstrate the system.""" - print("\n" + "=" * 60) - print("STEP 5: Creating sample interactions") - print("=" * 60) - - # Get a sample of cards to create interactions between - result = self.conn.execute(text(""" - SELECT id, name, subtypes, types, colors - FROM mtg_cards - WHERE subtypes IS NOT NULL AND subtypes != 'null' - LIMIT 50 - """)).fetchall() - - if len(result) < 2: - print(" ℹ️ Not enough cards with subtypes to create sample interactions") - return - - print(f" ✓ Found {len(result)} cards with subtypes") - - # Create sample synergies between cards with same archetype - print("\n🔄 Creating sample synergies...") - - # Group cards by archetype - archetype_cards = {} - for card_id, name, subtypes, types, colors in result: - if subtypes: - for archetype in [a.strip() for a in subtypes.split(',') if a.strip()]: - if archetype not in archetype_cards: - archetype_cards[archetype] = [] - archetype_cards[archetype].append(card_id) - - # Create synergies between cards of the same archetype - synergy_count = 0 - for archetype, card_ids in archetype_cards.items(): - if len(card_ids) >= 2: - for i in range(len(card_ids)): - for j in range(i + 1, len(card_ids)): - self.conn.execute(text(""" - INSERT INTO mtg_card_synergies (card_a_id, card_b_id, synergy_type, strength, notes) - VALUES (:card_a, :card_b, :synergy_type, :strength, :notes) - ON CONFLICT DO NOTHING - """), { - "card_a": card_ids[i], - "card_b": card_ids[j], - "synergy_type": "archetype_support", - "strength": 3, - "notes": f"Both {archetype} cards work well together" - }) - synergy_count += 1 - - print(f" ✓ Created {synergy_count} archetype synergies") - - # Create sample counters between cards with different colors - print("\n🔄 Creating sample counters...") - - counter_count = 0 - for i in range(min(20, len(result))): - card_a_id = result[i][0] - card_a_colors = result[i][4] - - if card_a_colors: - colors_a = [c.strip() for c in card_a_colors.split(',')] - - for j in range(i + 1, min(i + 10, len(result))): - card_b_id = result[j][0] - card_b_colors = result[j][4] - - if card_b_colors: - colors_b = [c.strip() for c in card_b_colors.split(',')] - - # If different colors, create a counter relationship - if set(colors_a) != set(colors_b): - self.conn.execute(text(""" - INSERT INTO mtg_card_counters (card_a_id, card_b_id, counter_type, strength, notes) - VALUES (:card_a, :card_b, :counter_type, :strength, :notes) - ON CONFLICT DO NOTHING - """), { - "card_a": card_a_id, - "card_b": card_b_id, - "counter_type": "mana_disadvantage", - "strength": 2, - "notes": "Different color identities create strategic tension" - }) - counter_count += 1 - - print(f" ✓ Created {counter_count} counter relationships") - - # Create sample evolutions for cards with same name in different sets - print("\n🔄 Creating sample evolutions...") - - self.conn.execute(text(""" - INSERT INTO mtg_card_evolution (card_id, evolved_card_id, evolution_type, strength, notes) - SELECT DISTINCT c1.id, c2.id, 'reprinted', 2, 'Reprint in different set' - FROM mtg_cards c1 - JOIN mtg_cards c2 ON c1.name = c2.name AND c1.set_id != c2.set_id - WHERE c1.subtypes IS NOT NULL AND c2.subtypes IS NOT NULL - LIMIT 50 - ON CONFLICT DO NOTHING - """)) - print(" ✓ Created sample evolutions") - - self.conn.commit() - print("\n✓ Step 5 complete: Sample interactions created") - - def step_6_update_interaction_stats(self): - """Step 6: Update interaction statistics for each card.""" - print("\n" + "=" * 60) - print("STEP 6: Updating interaction statistics") - print("=" * 60) - - # Delete existing stats - self.conn.execute(text("DELETE FROM mtg_card_interaction_stats")) - - # Calculate and insert stats - self.conn.execute(text(""" - INSERT INTO mtg_card_interaction_stats ( - card_id, total_synergies, total_counters, total_evolution, - total_partners, total_mechanics, total_archetypes, total_themes, - avg_synergy_strength, max_synergy_strength, primary_archetype, primary_theme - ) - SELECT - c.id, - COALESCE(synergies.synergy_count, 0), - COALESCE(counters.counter_count, 0), - COALESCE(evolution.evolution_count, 0), - COALESCE(partners.partner_count, 0), - COALESCE(mechanics.mechanic_count, 0), - COALESCE(archetypes.archetype_count, 0), - COALESCE(themes.theme_count, 0), - COALESCE(synergies.avg_strength, 0), - COALESCE(synergies.max_strength, 0), - archetypes.primary_archetype, - themes.primary_theme - FROM mtg_cards c - LEFT JOIN ( - SELECT card_a_id as card_id, COUNT(*) as synergy_count, - AVG(strength) as avg_strength, MAX(strength) as max_strength - FROM mtg_card_synergies - GROUP BY card_a_id - ) synergies ON c.id = synergies.card_id - LEFT JOIN ( - SELECT card_a_id as card_id, COUNT(*) as counter_count - FROM mtg_card_counters - GROUP BY card_a_id - ) counters ON c.id = counters.card_id - LEFT JOIN ( - SELECT card_id as card_id, COUNT(*) as evolution_count - FROM mtg_card_evolution - GROUP BY card_id - ) evolution ON c.id = evolution.card_id - LEFT JOIN ( - SELECT card_a_id as card_id, COUNT(*) as partner_count - FROM mtg_card_partners - GROUP BY card_a_id - ) partners ON c.id = partners.card_id - LEFT JOIN ( - SELECT card_id as card_id, COUNT(*) as mechanic_count - FROM mtg_card_mechanics - GROUP BY card_id - ) mechanics ON c.id = mechanics.card_id - LEFT JOIN ( - SELECT card_id as card_id, COUNT(*) as archetype_count - FROM mtg_card_archetypes - GROUP BY card_id - ) archetypes ON c.id = archetypes.card_id - LEFT JOIN ( - SELECT card_id as card_id, COUNT(*) as theme_count - FROM mtg_card_themes - GROUP BY card_id - ) themes ON c.id = themes.card_id - LEFT JOIN ( - SELECT card_id, archetype as primary_archetype - FROM mtg_card_archetypes a1 - WHERE id = ( - SELECT MIN(a2.id) - FROM mtg_card_archetypes a2 - WHERE a1.card_id = a2.card_id - ) - ) archetypes ON c.id = archetypes.card_id - LEFT JOIN ( - SELECT card_id, theme as primary_theme - FROM mtg_card_themes t1 - WHERE id = ( - SELECT MIN(t2.id) - FROM mtg_card_themes t2 - WHERE t1.card_id = t2.card_id - ) - ) themes ON c.id = themes.card_id - """)) - - print(" ✓ Updated interaction statistics") - - self.conn.commit() - print("\n✓ Step 6 complete: Interaction statistics updated") - - def run_migration(self): - """Run the complete migration.""" - print("=" * 60) - print("🚀 Running Complete MTGJSON Migration") - print("=" * 60) - - self.connect() - - try: - self.step_1_add_mtgjson_columns() - self.step_2_populate_mtgjson_columns() - self.step_3_create_interaction_graph() - self.step_4_populate_interaction_graph() - self.step_5_create_sample_interactions() - self.step_6_update_interaction_stats() - - self.disconnect() - - print("\n" + "=" * 60) - print("✅ Complete migration finished successfully!") - print("=" * 60) - print("\n📊 Summary:") - print(" • Added 35+ MTGJSON columns to mtg_cards table") - print(" • Added 7 MTGJSON columns to mtg_sets table") - print(" • Created 13 interaction graph tables") - print(" • Populated mechanics, archetypes, and synergies") - print(" • Created sample card interactions") - - except Exception as e: - print(f"\n❌ Migration failed: {e}") - raise - finally: - if self.conn: - self.conn.close() - - -def main(): - """Main entry point.""" - migration = MTGJSONFullMigration() - migration.run_migration() - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/migrate_schema.py b/backend/scripts/migrate_schema.py deleted file mode 100644 index 8e7c07c..0000000 --- a/backend/scripts/migrate_schema.py +++ /dev/null @@ -1,459 +0,0 @@ -""" -MTGJSON Database Migration Strategy - -Comprehensive mapping of MTGJSON data model to PostgreSQL schema. - -Strategy for Nested JSON Arrays: -1. Direct Columns: Simple scalar values (strings, numbers, booleans) -2. JSONB Columns: Complex objects/arrays that need querying (legalities, prices) -3. Related Tables: One-to-many relationships (card_faces, foreign_names, rulings) -4. Comma-Separated: Simple arrays that can be split (supertypes, types, subtypes) -""" -from sqlalchemy import create_engine, text -import json - -DB_URL = "postgresql+psycopg2://mtgonline:mtgonline_pass@172.18.0.2:5432/mtgdata" - - -class MTGJSONMigration: - """Migrate MTGJSON data to comprehensive PostgreSQL schema.""" - - def __init__(self): - self.engine = create_engine(DB_URL) - self.conn = None - - def connect(self): - """Connect to database.""" - self.conn = self.engine.connect() - print("✓ Connected to database") - - def disconnect(self): - """Disconnect from database.""" - if self.conn: - self.conn.close() - self.engine.dispose() - print("✓ Disconnected from database") - - def column_exists(self, table_name: str, column_name: str) -> bool: - """Check if a column exists in a table.""" - result = self.conn.execute(text(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = :table AND column_name = :column - """), {"table": table_name, "column": column_name}) - return result.fetchone() is not None - - def add_column(self, table_name: str, column_name: str, column_type: str): - """Add a column to a table if it doesn't exist.""" - if not self.column_exists(table_name, column_name): - self.conn.execute(text(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}")) - print(f" ✓ Added: {table_name}.{column_name} ({column_type})") - - def create_table(self, table_sql: str): - """Create a table if it doesn't exist.""" - self.conn.execute(text(table_sql)) - print(f" ✓ Created table") - - def create_index(self, index_sql: str): - """Create an index if it doesn't exist.""" - self.conn.execute(text(f"CREATE INDEX IF NOT EXISTS {index_sql.split(' ON ')[1].split(' ')[0]} ON {index_sql.split(' ON ')[1].split(' ')[1]}")) - print(f" ✓ Created index") - - def migrate_card_table(self): - """Add all MTGJSON card attributes to mtg_cards table.""" - print("\n📊 Migrating mtg_cards table...") - - # ======================== - # STRATEGY 1: Direct Columns (Simple scalar values) - # ======================== - print("\n📝 Strategy 1: Direct Columns (Simple scalar values)") - - direct_columns = [ - # Basic card info - ("name", "VARCHAR(255)"), - ("mana_cost", "VARCHAR(255)"), - ("type_line", "VARCHAR(255)"), - ("oracle_text", "TEXT"), - ("power", "VARCHAR(50)"), - ("toughness", "VARCHAR(50)"), - ("loyalty", "VARCHAR(50)"), # For Planeswalkers - ("rarity", "VARCHAR(50)"), - ("layout", "VARCHAR(50)"), - ("artist", "VARCHAR(255)"), - ("flavor_text", "TEXT"), - ("numbers", "VARCHAR(100)"), - - # MTGJSON: border, watermark - ("border_color", "VARCHAR(20)"), - ("watermark", "VARCHAR(255)"), - - # MTGJSON: colorIdentity (single color) - ("color_identity", "VARCHAR(10)"), - - # MTGJSON: lang - ("lang", "VARCHAR(10) DEFAULT 'en'"), - - # MTGJSON: originalReleaseDate - ("original_release_date", "DATE"), - - # MTGJSON: originalTypeLine - ("original_type_line", "VARCHAR(255)"), - - # MTGJSON: securityStamp - ("security_stamp", "VARCHAR(20)"), - - # MTGJSON: isPromo - ("promo", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: isDigital - ("digital", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: isToken - ("token", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: frame - ("frame", "VARCHAR(50)"), - - # MTGJSON: fullArt - ("full_art", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: isRebalanced - ("is_rebalanced", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: isStarter - ("is_starter", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: isInBooster - ("in_booster", "BOOLEAN DEFAULT FALSE"), - - # MTGJSON: mysticalArchive - ("mystical_archive", "BOOLEAN DEFAULT FALSE"), - ] - - for col_name, col_type in direct_columns: - self.add_column("mtg_cards", col_name, col_type) - - # ======================== - # STRATEGY 2: JSONB Columns (Complex objects/arrays) - # ======================== - print("\n📦 Strategy 2: JSONB Columns (Complex objects/arrays)") - - jsonb_columns = [ - # MTGJSON: legalities object - # Example: {"Standard": "Legal", "Modern": "Banned", "Vintage": "Restricted"} - ("legalities", "JSONB"), - - # MTGJSON: prices object - # Example: {"tcgplayer": "$4.99", "low": 2.5, "mid": 4.0, "high": 6.0} - ("prices", "JSONB"), - - # MTGJSON: cardFaces array (for split cards, modal DFCs) - # Example: [{"name": "Card A", "oracleText": "...", "power": "2"}, {"name": "Card B", ...}] - ("card_faces", "JSONB"), - - # MTGJSON: foreignData array (for translations) - # Example: [{"language": "Japanese", "name": "カード名", "typeLine": "クリーチャー"}, ...] - ("foreign_names", "JSONB"), - - # MTGJSON: relatedCards object - # Example: {"convertedNames": ["..."], "commanderCounterparts": [...]} - ("related_cards", "JSONB"), - - # MTGJSON: frameEffects array - # Example: ["extendedart", "legendary", "nightmare"] - ("frame_effects", "JSONB"), - - # MTGJSON: keywords array - # Example: ["first strike", "trample", "vision mount"] - ("keywords", "JSONB"), - - # MTGJSON: set (set object) - # Example: {"name": "Commander 2021", "code": "C21", "type": "commander"} - ("set", "JSONB"), - - # MTGJSON: booster (booster configuration) - # Example: {"boosters": [{"content": [...], "type": "main"}]} - ("booster", "JSONB"), - ] - - for col_name, col_type in jsonb_columns: - self.add_column("mtg_cards", col_name, col_type) - - # ======================== - # STRATEGY 3: Comma-Separated (Simple arrays) - # ======================== - print("\n🔗 Strategy 3: Comma-Separated (Simple arrays)") - - comma_separated = [ - # MTGJSON: types array (e.g., ["Creature", "Human"]) - ("types", "VARCHAR(255)"), - - # MTGJSON: subtypes array (e.g., ["Elf", "Rogue"]) - ("subtypes", "VARCHAR(255)"), - - # MTGJSON: supertypes array (e.g., ["Legendary"]) - ("supertypes", "VARCHAR(100)"), - - # MTGJSON: colors array (e.g., ["W", "G"]) - stored as comma-separated - ("colors", "VARCHAR(20)"), - ] - - for col_name, col_type in comma_separated: - self.add_column("mtg_cards", col_name, col_type) - - # ======================== - # STRATEGY 4: Related Tables (One-to-many relationships) - # ======================== - print("\n📚 Strategy 4: Related Tables (One-to-many relationships)") - - # Card faces table - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_faces ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - face_number INTEGER, - name VARCHAR(255), - mana_cost VARCHAR(255), - type_line VARCHAR(255), - oracle_text TEXT, - power VARCHAR(50), - toughness VARCHAR(50), - loyalty VARCHAR(50), - flavor_text TEXT, - artist VARCHAR(255), - illustration_id VARCHAR(100), - image_uri TEXT, - image_png TEXT, - image_art_crop TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - # Foreign names table - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_foreign_names ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - language VARCHAR(20), - name VARCHAR(255), - type_line VARCHAR(255), - oracle_text TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - # Rulings table - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_rulings ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - published_date DATE, - text TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - # Related cards table - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_related ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - related_type VARCHAR(50), - related_id INTEGER, - related_name VARCHAR(255), - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - # Card types table (for normalized type search) - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_types ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - type_category VARCHAR(50), - type_name VARCHAR(100), - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, type_category, type_name) - ) - """) - - # Color identity table (for multi-card color identity) - self.create_table(""" - CREATE TABLE IF NOT EXISTS mtg_card_color_identity ( - id SERIAL PRIMARY KEY, - card_id INTEGER REFERENCES mtg_cards(id) ON DELETE CASCADE, - color CHAR(1), - identity_type VARCHAR(20) DEFAULT 'color', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(card_id, color, identity_type) - ) - """) - - # ======================== - # CREATE INDEXES - # ======================== - print("\n🔍 Creating indexes...") - - indexes = [ - # Card indexes - "idx_cards_colors ON mtg_cards(colors)", - "idx_cards_color_identity ON mtg_cards(color_identity)", - "idx_cards_supertypes ON mtg_cards(supertypes)", - "idx_cards_types ON mtg_cards(types)", - "idx_cards_subtypes ON mtg_cards(subtypes)", - "idx_cards_legalities ON mtg_cards(legalities) USING GIN", - "idx_cards_prices ON mtg_cards(prices) USING GIN", - "idx_cards_card_faces ON mtg_cards(card_faces) USING GIN", - "idx_cards_foreign_names ON mtg_cards(foreign_names) USING GIN", - "idx_cards_related_cards ON mtg_cards(related_cards) USING GIN", - "idx_cards_keywords ON mtg_cards(keywords) USING GIN", - - # Set indexes - "idx_sets_status ON mtg_sets(status)", - "idx_sets_block_code ON mtg_sets(block_code)", - - # Related table indexes - "idx_card_faces_card_id ON mtg_card_faces(card_id)", - "idx_card_foreign_names_card_id ON mtg_card_foreign_names(card_id)", - "idx_card_rulings_card_id ON mtg_card_rulings(card_id)", - "idx_card_related_card_id ON mtg_card_related(card_id)", - "idx_card_types_card_id ON mtg_card_types(card_id)", - "idx_card_color_identity_card_id ON mtg_card_color_identity(card_id)", - ] - - for idx in indexes: - self.create_index(f"idx_{idx}") - - print("\n✅ Card table migration complete!") - - def migrate_set_table(self): - """Add all MTGJSON set attributes to mtg_sets table.""" - print("\n📊 Migrating mtg_sets table...") - - # MTGJSON set attributes - set_columns = [ - # Basic set info - ("code", "VARCHAR(10)"), - ("name", "VARCHAR(255)"), - ("type", "VARCHAR(100)"), - ("release_date", "DATE"), - ("base_set_size", "INTEGER"), - ("total_size", "INTEGER"), - ("is_foil_only", "BOOLEAN"), - ("is_non_foil_only", "BOOLEAN"), - ("digital", "BOOLEAN"), - ("icon_svg_url", "TEXT"), - ("parent_code", "VARCHAR(10)"), - ("mtgo_code", "VARCHAR(10)"), - - # MTGJSON: tcgplayerGroupId - ("tcgplayer_group_id", "INTEGER"), - - # MTGJSON: scryfallId - ("scryfall_id", "VARCHAR(36)"), - - # MTGJSON: status (released, unreleased, etc.) - ("status", "VARCHAR(20)"), - - # MTGJSON: name_normalized - ("name_normalized", "VARCHAR(255)"), - - # MTGJSON: blockCode - ("block_code", "VARCHAR(10)"), - - # MTGJSON: setCodes (all set codes) - ("set_codes", "JSONB"), - - # MTGJSON: cardCount (total cards in set) - ("card_count", "INTEGER"), - ] - - for col_name, col_type in set_columns: - self.add_column("mtg_sets", col_name, col_type) - - print("\n✅ Set table migration complete!") - - def populate_existing_data(self): - """Populate new columns from existing JSON data.""" - print("\n🔄 Populating existing data from JSON columns...") - - # Extract data from identifiers JSON - self.conn.execute(text(""" - UPDATE mtg_cards - SET - border_color = identifiers->>'border', - watermark = identifiers->>'watermark', - original_release_date = identifiers->>'originalReleaseDate', - original_type_line = identifiers->>'originalTypeLine', - security_stamp = identifiers->>'securityStamp', - lang = identifiers->>'lang', - promo = (identifiers->>'isPromo')::BOOLEAN, - digital = (identifiers->>'isDigital')::BOOLEAN, - token = (identifiers->>'isToken')::BOOLEAN - WHERE identifiers IS NOT NULL - AND identifiers != 'null' - AND identifiers != '' - """)) - print(" ✓ Updated basic fields from identifiers") - - # Extract type information from type_line - self.conn.execute(text(""" - UPDATE mtg_cards - SET - supertypes = type_line, - types = type_line, - subtypes = type_line - WHERE type_line IS NOT NULL - AND type_line != '' - """)) - print(" ✓ Updated type hierarchy from type_line") - - # Extract legalities, prices, card_faces from images JSON - self.conn.execute(text(""" - UPDATE mtg_cards - SET - prices = images->'prices', - card_faces = images->'cardFaces', - foreign_names = images->'foreignData', - related_cards = images->'relatedCards' - WHERE images IS NOT NULL - AND images != 'null' - AND images != '' - """)) - print(" ✓ Updated complex fields from images JSON") - - self.conn.commit() - print("\n✅ Data population complete!") - - def run_migration(self): - """Run the full migration.""" - print("=" * 60) - print("🚀 Starting MTGJSON Database Migration") - print("=" * 60) - - self.connect() - - # Migrate card table - self.migrate_card_table() - - # Migrate set table - self.migrate_set_table() - - # Populate existing data - self.populate_existing_data() - - self.disconnect() - - print("\n" + "=" * 60) - print("✅ Migration completed successfully!") - print("=" * 60) - - -def main(): - """Main entry point.""" - migration = MTGJSONMigration() - migration.run_migration() - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/recommendation_engine.py b/backend/scripts/recommendation_engine.py deleted file mode 100644 index 1d0bdd0..0000000 --- a/backend/scripts/recommendation_engine.py +++ /dev/null @@ -1,657 +0,0 @@ -""" -Card Interaction Recommendation Engine - -Uses the interaction graph to provide: -- Synergy-based card recommendations -- Deck archetype suggestions -- Card combination suggestions -- "Cards like this" recommendations -""" -from typing import List, Dict, Optional, Tuple -from dataclasses import dataclass -from enum import Enum -import json -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker - - -class RecommendationType(Enum): - """Types of recommendations.""" - SYNERGY = "synergy" - ARCHETYPE = "archetype" - COMBO = "combo" - COUNTER = "counter" - EVOLUTION = "evolution" - CARD_LIKE_THIS = "card_like_this" - - -@dataclass -class Recommendation: - """A single recommendation.""" - recommendation_type: str - card_id: int - card_name: str - card_type_line: str - confidence: float - score: float # Weighted score for ranking - reason: str - metadata: Dict[str, any] = None - - def __post_init__(self): - if self.metadata is None: - self.metadata = {} - - def to_dict(self) -> Dict: - """Convert to dictionary for JSON serialization.""" - return { - 'recommendation_type': self.recommendation_type, - 'card_id': self.card_id, - 'card_name': self.card_name, - 'card_type_line': self.card_type_line, - 'confidence': self.confidence, - 'score': self.score, - 'reason': self.reason, - 'metadata': self.metadata, - } - - -class RecommendationEngine: - """ - Generates card recommendations based on interaction graph data. - - Uses: - - Interaction graph for synergy matching - - Card profiles for archetype/mana curve matching - - Confidence scoring for ranking recommendations - """ - - def __init__(self, db_url: str, config: Optional[Dict] = None): - """Initialize with database URL and configuration.""" - self.db_url = db_url - self.config = config or { - 'max_recommendations': 50, - 'min_confidence': 0.5, - 'min_score': 1.0, - 'synergy_weight': 1.0, - 'archetype_weight': 0.8, - 'combo_weight': 1.2, - 'counter_weight': 0.6, - 'evolution_weight': 0.7, - } - - # Initialize database connection - self.engine = create_engine(db_url) - self.SessionLocal = sessionmaker(bind=self.engine) - - def get_card_profile(self, card_id: int) -> Optional[Dict]: - """Get full card profile from database.""" - db = self.SessionLocal() - try: - query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.id = :card_id - """) - - result = db.execute(query, {"card_id": card_id}).fetchone() - - if result: - return dict(result._mapping) - return None - finally: - db.close() - - def get_interactions_for_card(self, card_id: int) -> Dict[str, List[Dict]]: - """Get all interactions for a specific card.""" - db = self.SessionLocal() - try: - # Get synergies - synergies_query = text(""" - SELECT card_a_id, card_b_id, synergy_type, strength, notes - FROM mtg_card_synergies - WHERE card_a_id = :card_id OR card_b_id = :card_id - """) - synergies = [dict(row._mapping) for row in db.execute(synergies_query, {"card_id": card_id}).fetchall()] - - # Get counters - counters_query = text(""" - SELECT card_a_id, card_b_id, counter_type, strength, notes - FROM mtg_card_counters - WHERE card_a_id = :card_id OR card_b_id = :card_id - """) - counters = [dict(row._mapping) for row in db.execute(counters_query, {"card_id": card_id}).fetchall()] - - # Get evolutions - evolutions_query = text(""" - SELECT card_id, evolved_card_id, evolution_type, strength, notes - FROM mtg_card_evolution - WHERE card_id = :card_id OR evolved_card_id = :card_id - """) - evolutions = [dict(row._mapping) for row in db.execute(evolutions_query, {"card_id": card_id}).fetchall()] - - # Get archetypes - archetypes_query = text(""" - SELECT archetype, strength - FROM mtg_card_archetypes - WHERE card_id = :card_id - """) - archetypes = [dict(row._mapping) for row in db.execute(archetypes_query, {"card_id": card_id}).fetchall()] - - # Get mechanics - mechanics_query = text(""" - SELECT mechanic, strength - FROM mtg_card_mechanics - WHERE card_id = :card_id - """) - mechanics = [dict(row._mapping) for row in db.execute(mechanics_query, {"card_id": card_id}).fetchall()] - - return { - 'synergies': synergies, - 'counters': counters, - 'evolutions': evolutions, - 'archetypes': archetypes, - 'mechanics': mechanics, - } - finally: - db.close() - - def recommend_card_synergies( - self, card_id: int, max_results: int = 20 - ) -> List[Recommendation]: - """ - Recommend cards that synergize with a given card. - - Looks for cards with: - - Same archetype - - Supporting mechanics - - Compatible mana costs - - Combo potential - """ - recommendations = [] - card_profile = self.get_card_profile(card_id) - - if not card_profile: - return recommendations - - db = self.SessionLocal() - try: - # Get archetypes for this card - archetypes_query = text(""" - SELECT archetype, strength - FROM mtg_card_archetypes - WHERE card_id = :card_id - """) - card_archetypes = [dict(row._mapping) for row in - db.execute(archetypes_query, {"card_id": card_id}).fetchall()] - - # Get mechanics for this card - mechanics_query = text(""" - SELECT mechanic, strength - FROM mtg_card_mechanics - WHERE card_id = :card_id - """) - card_mechanics = [dict(row._mapping) for row in - db.execute(mechanics_query, {"card_id": card_id}).fetchall()] - - # Get synergies for this card - synergies_query = text(""" - SELECT card_b_id as card_id, synergy_type, strength, notes - FROM mtg_card_synergies - WHERE card_a_id = :card_id - ORDER BY strength DESC - """) - synergy_cards = [dict(row._mapping) for row in - db.execute(synergies_query, {"card_id": card_id}).fetchall()] - - # Score each synergizing card - for synergy in synergy_cards: - synergy_card_id = synergy['card_id'] - - # Get the other card's profile - other_card = self.get_card_profile(synergy_card_id) - if not other_card: - continue - - # Calculate score based on synergy strength and other factors - score = synergy['strength'] * self.config['synergy_weight'] - - # Boost score if same archetype - archetype_match = False - for archetype in card_archetypes: - if archetype['archetype'] in other_card.get('subtypes', ''): - archetype_match = True - score *= 1.2 - break - - # Boost score if shared mechanic - mechanic_match = False - for mechanic in card_mechanics: - if mechanic['mechanic'] in other_card.get('oracle_text', '').lower(): - mechanic_match = True - score *= 1.1 - break - - recommendations.append(Recommendation( - recommendation_type=RecommendationType.SYNERGY.value, - card_id=synergy_card_id, - card_name=other_card['name'], - card_type_line=other_card['type_line'], - confidence=0.9, - score=score, - reason=f"Synergizes with {card_profile['name']} ({synergy['synergy_type']})", - metadata={ - 'synergy_type': synergy['synergy_type'], - 'synergy_strength': synergy['strength'], - 'archetype_match': archetype_match, - 'mechanic_match': mechanic_match, - } - )) - - # Sort by score and return top results - recommendations.sort(key=lambda r: r.score, reverse=True) - return recommendations[:max_results] - - finally: - db.close() - - def recommend_archetype_cards( - self, archetype: str, max_results: int = 20 - ) -> List[Recommendation]: - """ - Recommend cards that fit a specific archetype. - - Looks for cards with: - - Matching subtype - - Supporting mechanics - - Compatible mana costs - """ - recommendations = [] - - db = self.SessionLocal() - try: - # Get cards with this archetype - cards_query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.subtypes LIKE :archetype - LIMIT :limit - """) - - cards = [dict(row._mapping) for row in - db.execute(cards_query, { - "archetype": f"%{archetype}%", - "limit": max_results * 2 - }).fetchall()] - - # Score each card - for card in cards: - # Calculate base score from archetype match - score = 1.0 - - # Boost score for cards with supporting mechanics - supporting_mechanics = [] - if archetype.lower() == 'elf': - supporting_mechanics = ['landfall', 'vigilance', 'trample'] - elif archetype.lower() == 'goblin': - supporting_mechanics = ['haste', 'trample', 'damage'] - elif archetype.lower() == 'vampire': - supporting_mechanics = ['lifelink', 'first_strike', 'deathtouch'] - elif archetype.lower() == 'angel': - supporting_mechanics = ['flying', 'lifelink', 'indestructible'] - elif archetype.lower() == 'dragon': - supporting_mechanics = ['flying', 'trample', 'menace'] - elif archetype.lower() == 'zombie': - supporting_mechanics = ['deathtouch', 'first_strike', 'haste'] - - for mechanic in supporting_mechanics: - if mechanic in card.get('oracle_text', '').lower(): - score += 0.5 - - # Boost score for cards with good power/toughness - try: - power = int(card.get('power', 0) or 0) - toughness = int(card.get('toughness', 0) or 0) - - if power >= 3 and toughness >= 3: - score += 0.5 - except (ValueError, TypeError): - pass - - recommendations.append(Recommendation( - recommendation_type=RecommendationType.ARCHETYPE.value, - card_id=card['id'], - card_name=card['name'], - card_type_line=card['type_line'], - confidence=0.8, - score=score, - reason=f"Matches {archetype} archetype", - metadata={ - 'archetype': archetype, - 'supporting_mechanics': supporting_mechanics, - } - )) - - # Sort by score and return top results - recommendations.sort(key=lambda r: r.score, reverse=True) - return recommendations[:max_results] - - finally: - db.close() - - def recommend_card_combos( - self, card_id: int, max_results: int = 10 - ) -> List[Recommendation]: - """ - Recommend card combos involving a specific card. - - Looks for cards that: - - Target the same creature - - Create powerful combinations - - Have complementary effects - """ - recommendations = [] - card_profile = self.get_card_profile(card_id) - - if not card_profile: - return recommendations - - db = self.SessionLocal() - try: - # Get synergies that are combo partners - combos_query = text(""" - SELECT card_b_id as card_id, synergy_type, strength, notes - FROM mtg_card_synergies - WHERE card_a_id = :card_id - AND synergy_type = 'COMBO_PARTNER' - ORDER BY strength DESC - """) - - combo_cards = [dict(row._mapping) for row in - db.execute(combos_query, {"card_id": card_id}).fetchall()] - - for combo in combo_cards: - combo_card_id = combo['card_id'] - - # Get the other card's profile - other_card = self.get_card_profile(combo_card_id) - if not other_card: - continue - - # Calculate score based on combo strength - score = combo['strength'] * self.config['combo_weight'] - - recommendations.append(Recommendation( - recommendation_type=RecommendationType.COMBO.value, - card_id=combo_card_id, - card_name=other_card['name'], - card_type_line=other_card['type_line'], - confidence=0.85, - score=score, - reason=f"Combo with {card_profile['name']} ({combo['notes']})", - metadata={ - 'combo_notes': combo['notes'], - 'combo_strength': combo['strength'], - } - )) - - # Sort by score and return top results - recommendations.sort(key=lambda r: r.score, reverse=True) - return recommendations[:max_results] - - finally: - db.close() - - def recommend_counter_cards( - self, card_id: int, max_results: int = 10 - ) -> List[Recommendation]: - """ - Recommend cards that counter a specific card. - - Looks for cards that: - - Have counter spells - - Target the same card types - - Have relevant keywords - """ - recommendations = [] - card_profile = self.get_card_profile(card_id) - - if not card_profile: - return recommendations - - db = self.SessionLocal() - try: - # Get cards that counter this card - counters_query = text(""" - SELECT card_b_id as card_id, counter_type, strength, notes - FROM mtg_card_counters - WHERE card_a_id = :card_id - ORDER BY strength DESC - """) - - counter_cards = [dict(row._mapping) for row in - db.execute(counters_query, {"card_id": card_id}).fetchall()] - - for counter in counter_cards: - counter_card_id = counter['card_id'] - - # Get the counter card's profile - counter_card = self.get_card_profile(counter_card_id) - if not counter_card: - continue - - # Calculate score based on counter strength - score = counter['strength'] * self.config['counter_weight'] - - recommendations.append(Recommendation( - recommendation_type=RecommendationType.COUNTER.value, - card_id=counter_card_id, - card_name=counter_card['name'], - card_type_line=counter_card['type_line'], - confidence=0.75, - score=score, - reason=f"Counters {card_profile['name']} ({counter['counter_type']})", - metadata={ - 'counter_type': counter['counter_type'], - 'counter_strength': counter['strength'], - } - )) - - # Sort by score and return top results - recommendations.sort(key=lambda r: r.score, reverse=True) - return recommendations[:max_results] - - finally: - db.close() - - def recommend_card_like_this( - self, card_id: int, max_results: int = 20 - ) -> List[Recommendation]: - """ - Recommend cards similar to a given card. - - Looks for cards with: - - Similar archetypes - - Similar mechanics - - Similar mana costs - - Similar power/toughness - """ - recommendations = [] - card_profile = self.get_card_profile(card_id) - - if not card_profile: - return recommendations - - db = self.SessionLocal() - try: - # Get this card's archetypes - archetypes_query = text(""" - SELECT archetype, strength - FROM mtg_card_archetypes - WHERE card_id = :card_id - """) - card_archetypes = [dict(row._mapping) for row in - db.execute(archetypes_query, {"card_id": card_id}).fetchall()] - - # Get this card's mechanics - mechanics_query = text(""" - SELECT mechanic, strength - FROM mtg_card_mechanics - WHERE card_id = :card_id - """) - card_mechanics = [dict(row._mapping) for row in - db.execute(mechanics_query, {"card_id": card_id}).fetchall()] - - # Search for similar cards - similar_cards_query = text(""" - SELECT c.*, s.code as set_code, s.name as set_name - FROM mtg_cards c - JOIN mtg_sets s ON c.set_id = s.id - WHERE c.id != :card_id - AND (c.subtypes LIKE :archetype OR c.oracle_text LIKE :mechanic) - LIMIT :limit - """) - - # Get cards with matching archetypes - archetype_matches = [] - for archetype in card_archetypes: - archetype_matches.extend( - [dict(row._mapping) for row in - db.execute(similar_cards_query, { - "card_id": card_id, - "archetype": f"%{archetype['archetype']}%", - "mechanic": "%", - "limit": max_results * 2 - }).fetchall()] - ) - - # Get cards with matching mechanics - mechanic_matches = [] - for mechanic in card_mechanics: - mechanic_matches.extend( - [dict(row._mapping) for row in - db.execute(similar_cards_query, { - "card_id": card_id, - "archetype": "%", - "mechanic": f"%{mechanic['mechanic']}%", - "limit": max_results * 2 - }).fetchall()] - ) - - # Deduplicate - seen_cards = set() - all_matches = [] - for card in archetype_matches + mechanic_matches: - if card['id'] not in seen_cards: - seen_cards.add(card['id']) - all_matches.append(card) - - # Score each similar card - for card in all_matches: - score = 0.5 - - # Boost for archetype match - for archetype in card_archetypes: - if archetype['archetype'] in card.get('subtypes', ''): - score += 1.0 - break - - # Boost for mechanic match - for mechanic in card_mechanics: - if mechanic['mechanic'] in card.get('oracle_text', '').lower(): - score += 0.5 - break - - # Boost for similar mana cost - try: - mana_a = int(card_profile.get('mana_cost', '0').replace('{', '').replace('}', '').replace('W', '').replace('U', '').replace('B', '').replace('R', '').replace('G', '').replace('X', '').replace('Y', '')) - mana_b = int(card.get('mana_cost', '0').replace('{', '').replace('}', '').replace('W', '').replace('U', '').replace('B', '').replace('R', '').replace('G', '').replace('X', '').replace('Y', '')) - - if abs(mana_a - mana_b) <= 1: - score += 0.5 - except (ValueError, TypeError): - pass - - # Boost for similar power/toughness - try: - power_a = int(card_profile.get('power', 0) or 0) - power_b = int(card.get('power', 0) or 0) - toughness_a = int(card_profile.get('toughness', 0) or 0) - toughness_b = int(card.get('toughness', 0) or 0) - - if abs(power_a - power_b) <= 1 and abs(toughness_a - toughness_b) <= 1: - score += 0.5 - except (ValueError, TypeError): - pass - - recommendations.append(Recommendation( - recommendation_type=RecommendationType.CARD_LIKE_THIS.value, - card_id=card['id'], - card_name=card['name'], - card_type_line=card['type_line'], - confidence=0.7, - score=score, - reason=f"Similar to {card_profile['name']}", - metadata={ - 'archetype_match': any(a['archetype'] in card.get('subtypes', '') for a in card_archetypes), - 'mechanic_match': any(m['mechanic'] in card.get('oracle_text', '').lower() for m in card_mechanics), - } - )) - - # Sort by score and return top results - recommendations.sort(key=lambda r: r.score, reverse=True) - return recommendations[:max_results] - - finally: - db.close() - - def get_full_recommendations( - self, card_id: int, max_results: int = 50 - ) -> List[Recommendation]: - """ - Get all recommendations for a card. - - Combines synergies, archetypes, combos, counters, and similar cards. - """ - all_recommendations = [] - - # Get synergies - synergies = self.recommend_card_synergies(card_id, max_results) - all_recommendations.extend(synergies) - - # Get archetype cards - card_profile = self.get_card_profile(card_id) - if card_profile and card_profile.get('subtypes'): - archetypes = card_profile['subtypes'].split(',') - for archetype in archetypes: - archetype_cards = self.recommend_archetype_cards(archetype.strip(), max_results) - all_recommendations.extend(archetype_cards) - - # Get combos - combos = self.recommend_card_combos(card_id, max_results) - all_recommendations.extend(combos) - - # Get counters - counters = self.recommend_counter_cards(card_id, max_results) - all_recommendations.extend(counters) - - # Get similar cards - similar = self.recommend_card_like_this(card_id, max_results) - all_recommendations.extend(similar) - - # Deduplicate by card_id - seen_cards = set() - unique_recommendations = [] - for rec in all_recommendations: - if rec.card_id not in seen_cards: - seen_cards.add(rec.card_id) - unique_recommendations.append(rec) - - # Sort by score and return top results - unique_recommendations.sort(key=lambda r: r.score, reverse=True) - return unique_recommendations[:max_results] - - def close(self): - """Close database connection.""" - self.engine.dispose() diff --git a/backend/scripts/sanity_check_mtgjson.py b/backend/scripts/sanity_check_mtgjson.py deleted file mode 100644 index 6b80d1c..0000000 --- a/backend/scripts/sanity_check_mtgjson.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -""" -MTGJSON Data Sanity Check - -Validates downloaded MTGJSON files for expected sizes before upserting to database. -This prevents corrupted or incomplete data from being loaded into PostgreSQL. -""" - -import asyncio -import json -import logging -from pathlib import Path - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# Expected minimum file sizes (in bytes) for MTGJSON v5 files -# These are approximate minimums based on typical MTGJSON data sizes -EXPECTED_MIN_SIZES = { - "AllPrintings.json": 500 * 1024 * 1024, # 500 MB (should be 500-600 MB) - "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB - "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB - "CardTypes.json": 1 * 1024 * 1024, # 1 MB - "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB - "MagicSets.json": 50 * 1024 * 1024, # 50 MB - "MagicRoots.json": 1 * 1024 * 1024, # 1 MB - "SetTranslations.json": 5 * 1024 * 1024, # 5 MB -} - - -def validate_file_sizes(data_dir: Path) -> dict: - """ - Validate downloaded MTGJSON files for expected sizes. - - Args: - data_dir: Path to the MTGJSON data directory - - Returns: - Dict with validation results - """ - results = { - "valid": True, - "files_checked": 0, - "files_valid": 0, - "files_invalid": 0, - "issues": [] - } - - if not data_dir.exists(): - results["valid"] = False - results["issues"].append(f"Data directory does not exist: {data_dir}") - return results - - # Check each expected file - for filename, min_size in EXPECTED_MIN_SIZES.items(): - filepath = data_dir / filename - - if not filepath.exists(): - results["issues"].append(f"Missing file: {filename}") - results["valid"] = False - results["files_checked"] += 1 - results["files_invalid"] += 1 - continue - - results["files_checked"] += 1 - actual_size = filepath.stat().st_size - - if actual_size < min_size: - results["valid"] = False - results["files_invalid"] += 1 - results["issues"].append( - f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)" - ) - logger.warning( - f"File {filename} is too small: {actual_size / (1024*1024):.1f} MB " - f"(expected minimum: {min_size / (1024*1024):.1f} MB)" - ) - else: - results["files_valid"] += 1 - logger.info( - f"✓ {filename}: {actual_size / (1024*1024):.1f} MB (OK)" - ) - - return results - - -async def validate_and_cleanup(data_dir: Path, max_retries: int = 3) -> bool: - """ - Validate MTGJSON files and cleanup if invalid. - - Args: - data_dir: Path to the MTGJSON data directory - max_retries: Maximum number of retry attempts - - Returns: - True if validation passes, False otherwise - """ - logger.info("=" * 60) - logger.info("MTGJSON Data Sanity Check") - logger.info("=" * 60) - - for attempt in range(1, max_retries + 1): - logger.info(f"\nAttempt {attempt}/{max_retries}") - - # Validate file sizes - results = validate_file_sizes(data_dir) - - if results["valid"]: - logger.info("\n✓ All files passed validation") - logger.info(f" Checked: {results['files_checked']} files") - logger.info(f" Valid: {results['files_valid']} files") - return True - - # Validation failed - logger.warning("\n✗ Validation failed:") - for issue in results["issues"]: - logger.warning(f" - {issue}") - - if attempt < max_retries: - logger.warning(f"\nCleanup and retry in {60 * attempt} seconds...") - await asyncio.sleep(60 * attempt) - - # Delete all downloaded files - logger.warning("Deleting downloaded files...") - for f in data_dir.glob("*"): - if f.is_file(): - f.unlink() - logger.warning(f" Deleted: {f.name}") - - logger.error("\n✗✗✗ All retry attempts failed ✗✗✗") - return False - - -if __name__ == "__main__": - import sys - - # Get data directory from settings or use default - try: - sys.path.insert(0, "/app") - from app.config import get_settings - settings = get_settings() - data_dir = Path(settings.DATA_DIR) - except Exception as e: - logger.error(f"Failed to load settings: {e}") - data_dir = Path("/app/data/mtgjson") - - # Run validation - asyncio.run(validate_and_cleanup(data_dir)) diff --git a/backend/scripts/test_interaction_determinator.py b/backend/scripts/test_interaction_determinator.py deleted file mode 100644 index 83e046d..0000000 --- a/backend/scripts/test_interaction_determinator.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Comprehensive test of interaction_determinator.py""" -import sys -sys.path.insert(0, "/home/wall-o/projects/mtgonline/backend/scripts") - -from interaction_determinator import ( - InteractionDeterminator, - InteractionResult, - InteractionType, -) - -det = InteractionDeterminator() - -print("=" * 60) -print("TEST 1: extract_colors") -print("=" * 60) - -# MTGJSON braced format -assert det.extract_colors("{1}{W}{U}") == ["W", "U"], f"Got: {det.extract_colors('{1}{W}{U}')}" -# Plain format -assert det.extract_colors("WWU") == ["W", "U"], f"Got: {det.extract_colors('WWU')}" -# Empty -assert det.extract_colors("") == [] -# None -assert det.extract_colors(None) == [] -# Single color -assert det.extract_colors("{R}") == ["R"] -# Multi-color -assert det.extract_colors("{W}{B}{R}") == ["W", "B", "R"] -print(" ✓ All color extraction tests passed") - -print() -print("=" * 60) -print("TEST 2: extract_archetypes") -print("=" * 60) - -# List input (MTGJSON format) -assert sorted(det.extract_archetypes(["Goblin", "Warrior"])) == ["goblin", "warrior"], f"Got: {det.extract_archetypes(['Goblin', 'Warrior'])}" -# String input -assert sorted(det.extract_archetypes("Goblin Warrior")) == ["goblin", "warrior"] -# Empty -assert det.extract_archetypes([]) == [] -assert det.extract_archetypes("") == [] -# Multiple archetypes -result = det.extract_archetypes(["Elf", "Warrior", "Knight"]) -assert "elf" in result and "warrior" in result and "knight" in result, f"Got: {result}" -print(" ✓ All archetype extraction tests passed") - -print() -print("=" * 60) -print("TEST 3: extract_mechanics") -print("=" * 60) - -# Test that extracted mechanics work correctly -mechs = det.extract_mechanics("Creature — Elf", "Flying\nFirst strike") -assert "flying" in mechs, f"Got: {mechs}" -assert "first_strike" in mechs, f"Got: {mechs}" -# Empty -assert det.extract_mechanics("", "") == [] -print(" ✓ All mechanic extraction tests passed") - -print() -print("=" * 60) -print("TEST 4: extract_targets") -print("=" * 60) - -targets = det.extract_targets("Destroy target creature. Draw a card.") -assert "creature" in targets -assert "draws_card" in targets -assert det.extract_targets("") == [] -print(" ✓ All target extraction tests passed") - -print() -print("=" * 60) -print("TEST 5: extract_triggers") -print("=" * 60) - -triggers = det.extract_triggers("When this enters the battlefield, draw a card.") -assert "enters_battlefield" in triggers -assert "draws_card" in triggers -assert det.extract_triggers("") == [] - -print() -print("=" * 60) -print("TEST 6: extract_effects") -print("=" * 60) - -effects = det.extract_effects("Target creature gains flying until end of turn.") -assert "gain_flying" in effects -assert "until_end_of_turn" in effects -assert det.extract_effects("") == [] -print(" ✓ All effect extraction tests passed") - -print() -print("=" * 60) -print("TEST 7: extract_card_properties") -print("=" * 60) - -card = { - "id": 1, - "name": "Test Card", - "types": ["Creature", "Elf"], - "subtypes": ["Elf", "Warrior"], - "mana_cost": "{1}{W}", - "oracle_text": "Flying\nWhen this enters the battlefield, draw a card.\nTarget creature gains deathtouch until end of turn.", - "power": "2", - "toughness": "2", - "card_faces": [], -} -profile = det.extract_card_properties(card) -assert profile["id"] == 1 -assert profile["colors"] == ["W"] -assert "flying" in profile["mechanics"] -assert "elf" in profile["archetypes"] -assert "creature" in profile["targets"] -assert "enters_battlefield" in profile["triggers"] -assert "draws_card" in profile["triggers"] -assert profile["power"] == 2 -assert profile["toughness"] == 2 -print(" ✓ All card property extraction tests passed") - -print() -print("=" * 60) -print("TEST 8: determine_synergies") -print("=" * 60) - -# Same archetype synergy -card_a = det.extract_card_properties({ - "id": 10, - "name": "Goblin Warrior", - "types": ["Creature"], - "subtypes": ["Goblin", "Warrior"], - "mana_cost": "{R}", - "oracle_text": "Flying\nWhen this enters the battlefield, draw a card.", - "power": "1", - "toughness": "1", - "card_faces": [], -}) -card_b = det.extract_card_properties({ - "id": 11, - "name": "Goblin Hero", - "types": ["Creature"], - "subtypes": ["Goblin"], - "mana_cost": "{R}", - "oracle_text": "When this enters the battlefield, draw a card.", - "power": "2", - "toughness": "1", - "card_faces": [], -}) -synergies = det.determine_synergies(card_a, card_b) -assert any(s.interaction_type == "archetype_support" for s in synergies), "Expected archetype_support" -print(f" ✓ Found {len(synergies)} synergies") -for s in synergies: - print(f" - {s.interaction_type}: {s.notes}") - -print() -print("=" * 60) -print("TEST 9: determine_counters") -print("=" * 60) - -card_c = det.extract_card_properties({ - "id": 12, - "name": "Indestructible Wall", - "types": ["Creature"], - "subtypes": ["Wall"], - "mana_cost": "{2}{W}", - "oracle_text": "Indestructible", - "power": "0", - "toughness": "5", - "card_faces": [], -}) -card_d = det.extract_card_properties({ - "id": 13, - "name": "Deathtouch Beast", - "types": ["Creature"], - "subtypes": ["Beast"], - "mana_cost": "{1}{B}", - "oracle_text": "Deathtouch", - "power": "1", - "toughness": "1", - "card_faces": [], -}) -counters = det.determine_counters(card_c, card_d) -assert any("indestructible" in c.interaction_type.lower() or "deathtouch" in c.interaction_type.lower() for c in counters), "Expected indestructible/deathtouch counter" -print(f" ✓ Found {len(counters)} counters") -for c in counters: - print(f" - {c.interaction_type}: {c.notes}") - -print() -print("=" * 60) -print("TEST 10: determine_evolutions") -print("=" * 60) - -card_e = det.extract_card_properties({ - "id": 14, - "name": "Same Name Card", - "types": ["Creature"], - "subtypes": ["Elf"], - "mana_cost": "{G}", - "oracle_text": "Trample", - "power": "3", - "toughness": "3", - "card_faces": [], -}) -card_f = det.extract_card_properties({ - "id": 15, - "name": "Same Name Card", - "types": ["Creature"], - "subtypes": ["Elf"], - "mana_cost": "{G}", - "oracle_text": "Trample", - "power": "3", - "toughness": "3", - "card_faces": [], -}) -evolutions = det.determine_evolutions(card_e, card_f) -assert any(e.interaction_type == "reprinted" for e in evolutions), "Expected reprint" -print(f" ✓ Found {len(evolutions)} evolutions") -for e in evolutions: - print(f" - {e.interaction_type}: {e.notes}") - -print() -print("=" * 60) -print("TEST 11: determine_all_interactions (batch)") -print("=" * 60) - -all_cards = [card_a, card_b, card_c, card_d, card_e, card_f] -batch = det.determine_all_interactions(all_cards) -print(f" Synergies: {len(batch['synergies'])}") -print(f" Counters: {len(batch['counters'])}") -print(f" Evolutions: {len(batch['evolutions'])}") -assert len(batch["synergies"]) > 0 -assert len(batch["counters"]) > 0 -print(" ✓ Batch interaction determination passed") - -print() -print("=" * 60) -print("TEST 12: filter_by_confidence") -print("=" * 60) - -high_conf = det.filter_by_confidence(batch["synergies"], 0.9) -assert all(s.confidence >= 0.9 for s in high_conf) -print(f" ✓ Filtered to {len(high_conf)} high-confidence synergies") - -print() -print("=" * 60) -print("TEST 13: group_by_card") -print("=" * 60) - -grouped = det.group_by_card(batch["synergies"]) -assert all(isinstance(v, list) for v in grouped.values()) -print(f" ✓ Grouped into {len(grouped)} cards") - -print() -print("=" * 60) -print("TEST 14: get_interaction_summary") -print("=" * 60) - -summary = det.get_interaction_summary(batch["synergies"]) -assert isinstance(summary, dict) -print(f" ✓ Summary: {summary}") - -print() -print("=" * 60) -print("TEST 15: Pipeline integration (raw dicts)") -print("=" * 60) - -# Test with raw MTGJSON-style dicts (as the pipeline passes them) -raw_cards = [ - { - "id": 100, - "name": "Goblin Warrior", - "types": ["Creature"], - "subtypes": ["Goblin", "Warrior"], - "mana_cost": "{R}", - "oracle_text": "Flying\nWhen this enters the battlefield, draw a card.", - "power": "1", - "toughness": "1", - "card_faces": [], - }, - { - "id": 101, - "name": "Goblin Hero", - "types": ["Creature"], - "subtypes": ["Goblin"], - "mana_cost": "{R}", - "oracle_text": "When this enters the battlefield, draw a card.", - "power": "2", - "toughness": "1", - "card_faces": [], - }, - { - "id": 102, - "name": "Indestructible Wall", - "types": ["Creature"], - "subtypes": ["Wall"], - "mana_cost": "{2}{W}", - "oracle_text": "Indestructible", - "power": "0", - "toughness": "5", - "card_faces": [], - }, - { - "id": 103, - "name": "Deathtouch Beast", - "types": ["Creature"], - "subtypes": ["Beast"], - "mana_cost": "{1}{B}", - "oracle_text": "Deathtouch", - "power": "1", - "toughness": "1", - "card_faces": [], - }, -] - -batch = det.determine_all_interactions(raw_cards) -print(f" Synergies: {len(batch['synergies'])}") -print(f" Counters: {len(batch['counters'])}") -print(f" Evolutions: {len(batch['evolutions'])}") -assert len(batch["synergies"]) > 0 -assert len(batch["counters"]) > 0 -print(" ✓ Pipeline integration test passed") - -print() -print("=" * 60) -print("ALL TESTS PASSED ✓") -print("=" * 60) diff --git a/backend/scripts/verify_integration.py b/backend/scripts/verify_integration.py deleted file mode 100644 index 6edfea4..0000000 --- a/backend/scripts/verify_integration.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3 -""" -Comprehensive MTGJSON Data Verification - -Checks both: -1. MTGJSON data file downloads -2. PostgreSQL database upsert status -""" - -import asyncio -from pathlib import Path -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -import sys - -sys.path.append("/app") - -from app.services.mtgjson_manager import MTGJSONManager -from app.config import get_settings - - -async def verify_data_download(): - """Verify MTGJSON data files were downloaded.""" - print("=" * 70) - print("MTGJSON DATA DOWNLOAD VERIFICATION") - print("=" * 70) - - settings = get_settings() - data_dir = Path(settings.DATA_DIR) - - print(f"\nData Directory: {data_dir}") - print(f"Directory exists: {data_dir.exists()}") - - if not data_dir.exists(): - print("❌ FAIL: Data directory does not exist") - return False - - # Check for required files - required_files = { - "AllPrintings.json.gz": "All sets data", - "AllSetFiles.json.gz": "Set metadata", - "AllIdentifiers.json.gz": "Card identifiers", - "CardTypes.json.gz": "Card type definitions", - "Keywords.json.gz": "Card keywords", - "MagicRoots.json.gz": "Root data", - "MagicSets.json.gz": "Set data", - "SetTranslations.json.gz": "Set translations" - } - - downloaded_files = [] - missing_files = [] - - print("\nRequired MTGJSON files:") - for filename, description in required_files.items(): - filepath = data_dir / filename - if filepath.exists(): - size_mb = filepath.stat().st_size / (1024 * 1024) - downloaded_files.append(filename) - print(f" ✓ {filename:30} - {size_mb:8.1f} MB") - else: - missing_files.append(filename) - print(f" ✗ {filename:30} - MISSING") - - print(f"\nDownload Status: {len(downloaded_files)}/{len(required_files)} files") - - if missing_files: - print(f"\n❌ FAIL: Missing {len(missing_files)} required files: {', '.join(missing_files)}") - return False - - # Check file sizes for sanity - allprintings_path = data_dir / "AllPrintings.json.gz" - if allprintings_path.exists(): - size_mb = allprintings_path.stat().st_size / (1024 * 1024) - if size_mb < 100: - print(f"\n⚠️ WARNING: AllPrintings.json.gz is suspiciously small ({size_mb:.1f} MB). Expected ~500-600 MB") - return False - else: - print(f"\n✓ AllPrintings.json.gz size looks good: {size_mb:.1f} MB") - - print("\n✓ PASS: All MTGJSON data files downloaded successfully") - return True - - -async def verify_database_upsert(): - """Verify MTGJSON data was properly upserted to PostgreSQL.""" - print("\n" + "=" * 70) - print("POSTGRESQL DATABASE UPSERT VERIFICATION") - print("=" * 70) - - settings = get_settings() - db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" - db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" - - print(f"\nDatabase: {settings.POSTGRES_DB}") - - try: - engine = create_async_engine(db_url) - - async with AsyncSession(engine) as session: - # Get all tables - result = await session.execute(text(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - ORDER BY table_name; - """)) - tables = [row[0] for row in result.fetchall()] - - print(f"\nTotal tables: {len(tables)}") - - # Check MTGJSON-specific tables - mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] - - print("\nMTGJSON tables:") - for table in mtg_tables: - if table in tables: - result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) - count = result.scalar() - print(f" ✓ {table:30} - {count:8,} records") - else: - print(f" ✗ {table:30} - TABLE NOT FOUND") - - # Check refresh log - if 'mtg_refresh_log' in tables: - result = await session.execute(text(""" - SELECT refresh_type, status, created_at - FROM mtg_refresh_log - ORDER BY created_at DESC - LIMIT 5; - """)) - rows = result.fetchall() - - if rows: - print("\nRecent refresh operations:") - for row in rows: - status_icon = "✓" if row[1] == 'SUCCESS' else "✗" - print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}") - - # Verify data quality - print("\nData quality checks:") - - # Check for sets - if 'mtg_set' in tables: - result = await session.execute(text(""" - SELECT COUNT(*) FROM mtg_set - WHERE set_name IS NOT NULL AND set_code IS NOT NULL; - """)) - valid_sets = result.scalar() - result = await session.execute(text("SELECT COUNT(*) FROM mtg_set")) - total_sets = result.scalar() - print(f" ✓ Sets: {valid_sets:,}/{total_sets:,} valid") - - # Check for cards - if 'mtg_card' in tables: - result = await session.execute(text(""" - SELECT COUNT(*) FROM mtg_card - WHERE name IS NOT NULL AND mtgjson_cards_id IS NOT NULL; - """)) - valid_cards = result.scalar() - result = await session.execute(text("SELECT COUNT(*) FROM mtg_card")) - total_cards = result.scalar() - print(f" ✓ Cards: {valid_cards:,}/{total_cards:,} valid") - - # Check for identifiers - if 'mtg_identifiers' in tables: - result = await session.execute(text(""" - SELECT COUNT(*) FROM mtg_identifiers - WHERE scryfall_id IS NOT NULL; - """)) - valid_ids = result.scalar() - result = await session.execute(text("SELECT COUNT(*) FROM mtg_identifiers")) - total_ids = result.scalar() - print(f" ✓ Identifiers: {valid_ids:,}/{total_ids:,} valid") - - # Check for keywords - if 'mtg_keywords' in tables: - result = await session.execute(text("SELECT COUNT(*) FROM mtg_keywords")) - keywords_count = result.scalar() - print(f" ✓ Keywords: {keywords_count:,}") - - print("\n✓ PASS: Database upsert completed successfully") - return True - - except Exception as e: - print(f"\n❌ FAIL: Database error - {e}") - return False - - -async def main(): - """Main verification function.""" - print("\n" + "=" * 70) - print("MTGJSON DATA INTEGRATION VERIFICATION") - print("=" * 70) - - # Check data download - download_ok = await verify_data_download() - - # Check database upsert - db_ok = await verify_database_upsert() - - # Final summary - print("\n" + "=" * 70) - print("VERIFICATION SUMMARY") - print("=" * 70) - - if download_ok and db_ok: - print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓") - print("\nMTGJSON data has been successfully downloaded and upserted to PostgreSQL.") - print("The backend is ready to use.") - return 0 - else: - print("\n❌❌❌ VERIFICATION FAILED ❌❌❌") - if not download_ok: - print("\nDownload issues:") - print(" - Some MTGJSON data files are missing or corrupted") - print(" - Run: docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py") - if not db_ok: - print("\nDatabase issues:") - print(" - Data was not properly upserted to PostgreSQL") - print(" - Check backend logs for errors") - return 1 - - -if __name__ == "__main__": - exit_code = asyncio.run(main()) - sys.exit(exit_code) diff --git a/backend/scripts/verify_mtgjson_data.py b/backend/scripts/verify_mtgjson_data.py deleted file mode 100644 index 92ff896..0000000 --- a/backend/scripts/verify_mtgjson_data.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python3 -""" -MTGJSON Data Verification Script - -Checks: -1. Data file download status -2. Sanity check validation -3. Database upsert status -4. File size validation -""" - -import asyncio -import json -from pathlib import Path -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -import sys - -sys.path.insert(0, "/app") - -from app.config import get_settings -from app.services.mtgjson_manager import MTGJSONManager - -# Expected minimum file sizes (in bytes) -EXPECTED_MIN_SIZES = { - "AllPrintings.json": 500 * 1024 * 1024, # 500 MB - "AllSetFiles.json": 10 * 1024 * 1024, # 10 MB - "AllIdentifiers.json": 100 * 1024 * 1024, # 100 MB - "CardTypes.json": 1 * 1024 * 1024, # 1 MB - "Keywords.json": 0.5 * 1024 * 1024, # 0.5 MB - "MagicSets.json": 50 * 1024 * 1024, # 50 MB - "MagicRoots.json": 1 * 1024 * 1024, # 1 MB - "SetTranslations.json": 5 * 1024 * 1024, # 5 MB -} - - -async def check_data_files(data_dir: Path) -> dict: - """Check downloaded data files and validate sizes.""" - print("=" * 70) - print("DATA FILE VERIFICATION") - print("=" * 70) - - results = { - "valid": True, - "files_checked": 0, - "files_valid": 0, - "files_invalid": 0, - "issues": [] - } - - if not data_dir.exists(): - results["valid"] = False - results["issues"].append(f"Data directory does not exist: {data_dir}") - return results - - # Check each expected file - for filename, min_size in EXPECTED_MIN_SIZES.items(): - filepath = data_dir / filename - - if not filepath.exists(): - results["issues"].append(f"Missing file: {filename}") - results["valid"] = False - results["files_checked"] += 1 - results["files_invalid"] += 1 - continue - - results["files_checked"] += 1 - actual_size = filepath.stat().st_size - - if actual_size < min_size: - results["valid"] = False - results["files_invalid"] += 1 - results["issues"].append( - f"{filename}: {actual_size / (1024*1024):.1f} MB (minimum: {min_size / (1024*1024):.1f} MB)" - ) - print(f" ✗ {filename:30} - {actual_size / (1024*1024):8.1f} MB (too small)") - else: - results["files_valid"] += 1 - print(f" ✓ {filename:30} - {actual_size / (1024*1024):8.1f} MB") - - print(f"\nSummary: {results['files_valid']}/{results['files_checked']} files valid") - - if results["issues"]: - print("\nIssues:") - for issue in results["issues"]: - print(f" - {issue}") - - return results - - -async def check_database_status() -> dict: - """Check database upsert status.""" - print("\n" + "=" * 70) - print("DATABASE STATUS VERIFICATION") - print("=" * 70) - - settings = get_settings() - db_url = f"postgresql+asyncpg://{settings.POSTGRES_USER}:{settings.POSTGRES_PASSWORD}" - db_url += f"@postgres-mtgdata:5432/{settings.POSTGRES_DB}" - - try: - engine = create_async_engine(db_url) - - async with AsyncSession(engine) as session: - # Get all tables - result = await session.execute(text(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - ORDER BY table_name; - """)) - tables = [row[0] for row in result.fetchall()] - - print(f"Total tables: {len(tables)}") - - # Check MTGJSON tables - mtg_tables = ['mtg_set', 'mtg_card', 'mtg_identifiers', 'mtg_keywords'] - - for table in mtg_tables: - if table in tables: - result = await session.execute(text(f"SELECT COUNT(*) FROM {table}")) - count = result.scalar() - print(f" ✓ {table:30} - {count:8,} records") - else: - print(f" ✗ {table:30} - TABLE NOT FOUND") - - # Check refresh log - if 'mtg_refresh_log' in tables: - result = await session.execute(text(""" - SELECT refresh_type, status, created_at - FROM mtg_refresh_log - ORDER BY created_at DESC - LIMIT 5; - """)) - rows = result.fetchall() - - if rows: - print("\nRecent refresh operations:") - for row in rows: - status_icon = "✓" if row[1] == 'SUCCESS' else "✗" - print(f" {status_icon} {row[0]:15} - {row[1]:8} - {row[2]}") - - return {"valid": True, "error": None} - - except Exception as e: - print(f"\n✗ Database error: {e}") - return {"valid": False, "error": str(e)} - - -async def main(): - """Main verification function.""" - print("\n" + "=" * 70) - print("MTGJSON DATA INTEGRATION VERIFICATION") - print("=" * 70) - - settings = get_settings() - data_dir = Path(settings.DATA_DIR) - - # Check data files - file_results = await check_data_files(data_dir) - - # Check database - db_results = await check_database_status() - - # Final summary - print("\n" + "=" * 70) - print("VERIFICATION SUMMARY") - print("=" * 70) - - if file_results["valid"] and db_results["valid"]: - print("\n✓✓✓ ALL CHECKS PASSED ✓✓✓") - print("\nThe MTGJSON data has been properly downloaded and upserted.") - return 0 - else: - print("\n❌ VERIFICATION FAILED") - - if not file_results["valid"]: - print("\nData file issues:") - for issue in file_results["issues"]: - print(f" - {issue}") - - if not db_results["valid"]: - print(f"\nDatabase issues: {db_results['error']}") - - print("\nSOLUTION:") - print(" 1. Delete corrupted data files") - print(" 2. Run fresh download with sanity checks:") - print(" docker exec mtgonline_backend python /app/scripts/download_mtgjson_v5.py") - return 1 - - -if __name__ == "__main__": - exit_code = asyncio.run(main()) - sys.exit(exit_code) diff --git a/backend/setup_db.py b/backend/setup_db.py deleted file mode 100644 index a38a826..0000000 --- a/backend/setup_db.py +++ /dev/null @@ -1,223 +0,0 @@ -""" -Database initialization script for MTG Online backend. - -Creates all necessary tables matching the SQLAlchemy ORM models -for both databases. -""" -import asyncio -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker -from sqlalchemy import text -from app.core.settings import get_settings - - -async def setup_mtg_online_database(): - """Create tables for the mtgonline database.""" - settings = get_settings() - engine = create_async_engine(settings.MTGO_DATABASE_URL) - async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - async with async_session() as session: - # Create tables - tables = [ - """ - CREATE TABLE IF NOT EXISTS mtgonline_users ( - id SERIAL PRIMARY KEY, - username VARCHAR(64) UNIQUE NOT NULL, - password_hash VARCHAR(128) NOT NULL, - salt VARCHAR(128) NOT NULL, - email VARCHAR(255), - country VARCHAR(2), - real_name VARCHAR(128), - avatar_bmp TEXT, - privlevel VARCHAR(50) DEFAULT 'User', - is_active BOOLEAN DEFAULT TRUE, - is_banned BOOLEAN DEFAULT FALSE, - ban_reason TEXT, - ban_ends TIMESTAMP, - vip_status INTEGER DEFAULT 0, - vip_expiry TIMESTAMP, - creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - last_login TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_decklist_folders ( - id SERIAL PRIMARY KEY, - owner_id INTEGER REFERENCES mtgonline_users(id) ON DELETE CASCADE, - name VARCHAR(255) NOT NULL, - parent_id INTEGER REFERENCES mtgonline_decklist_folders(id) ON DELETE CASCADE, - creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_decklist_files ( - id SERIAL PRIMARY KEY, - folder_id INTEGER REFERENCES mtgonline_decklist_folders(id) ON DELETE CASCADE, - owner_id INTEGER REFERENCES mtgonline_users(id) ON DELETE CASCADE, - name VARCHAR(255) NOT NULL, - content TEXT NOT NULL, - format VARCHAR(50) DEFAULT 'native', - creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_rooms ( - id SERIAL PRIMARY KEY, - name VARCHAR(100) UNIQUE NOT NULL, - description TEXT, - is_password_protected BOOLEAN DEFAULT FALSE, - password_hash VARCHAR(128), - creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_rooms_gametypes ( - id SERIAL PRIMARY KEY, - room_id INTEGER REFERENCES mtgonline_rooms(id) ON DELETE CASCADE, - name VARCHAR(100) NOT NULL, - description TEXT - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_bans ( - id SERIAL PRIMARY KEY, - user_id INTEGER REFERENCES mtgonline_users(id) ON DELETE CASCADE, - server_id INTEGER, - reason TEXT NOT NULL, - moderators VARCHAR(255), - ip_address VARCHAR(45), - expiration_time TIMESTAMP, - active BOOLEAN DEFAULT TRUE, - creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_log ( - id SERIAL PRIMARY KEY, - room_id INTEGER REFERENCES mtgonline_rooms(id) ON DELETE CASCADE, - player_id INTEGER REFERENCES mtgonline_users(id) ON DELETE CASCADE, - message TEXT NOT NULL, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtgonline_audit ( - id SERIAL PRIMARY KEY, - admin_id INTEGER REFERENCES mtgonline_users(id) ON DELETE SET NULL, - action_type VARCHAR(50) NOT NULL, - target_user_id INTEGER REFERENCES mtgonline_users(id) ON DELETE SET NULL, - details TEXT, - ip_address VARCHAR(45), - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - ] - - for table_sql in tables: - await session.execute(text(table_sql)) - - # Create indexes - indexes = [ - "CREATE INDEX IF NOT EXISTS idx_decks_owner ON mtgonline_decklist_files(owner_id);", - "CREATE INDEX IF NOT EXISTS idx_decks_folder ON mtgonline_decklist_files(folder_id);", - "CREATE INDEX IF NOT EXISTS idx_bans_active ON mtgonline_bans(active);", - "CREATE INDEX IF NOT EXISTS idx_log_timestamp ON mtgonline_log(timestamp);", - ] - - for idx_sql in indexes: - await session.execute(text(idx_sql)) - - print("✓ All mtgonline tables created") - - await engine.dispose() - - -async def setup_mtg_data_database(): - """Create tables for the mtgdata database.""" - settings = get_settings() - engine = create_async_engine(settings.MTG_DATABASE_URL) - async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - async with async_session() as session: - # Create tables - tables = [ - """ - CREATE TABLE IF NOT EXISTS mtg_sets ( - id SERIAL PRIMARY KEY, - code VARCHAR(10) UNIQUE NOT NULL, - name VARCHAR(255), - type VARCHAR(100), - release_date DATE, - base_set_size INTEGER, - total_size INTEGER, - is_foil_only BOOLEAN, - is_non_foil_only BOOLEAN, - digital BOOLEAN, - icon_svg_url TEXT, - parent_code VARCHAR(10), - mtgo_code VARCHAR(10), - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - """ - CREATE TABLE IF NOT EXISTS mtg_cards ( - id SERIAL PRIMARY KEY, - set_id INTEGER REFERENCES mtg_sets(id) ON DELETE CASCADE, - name VARCHAR(255), - mana_cost VARCHAR(255), - type_line VARCHAR(255), - oracle_text TEXT, - power VARCHAR(50), - toughness VARCHAR(50), - rarity VARCHAR(50), - layout VARCHAR(50), - artist VARCHAR(255), - flavor_text TEXT, - numbers VARCHAR(100), - identifiers TEXT, - images TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """, - ] - - for table_sql in tables: - await session.execute(text(table_sql)) - - # Create indexes - indexes = [ - "CREATE INDEX IF NOT EXISTS idx_mtg_cards_name ON mtg_cards(name);", - "CREATE INDEX IF NOT EXISTS idx_mtg_cards_type ON mtg_cards(type_line);", - "CREATE INDEX IF NOT EXISTS idx_mtg_cards_rarity ON mtg_cards(rarity);", - "CREATE INDEX IF NOT EXISTS idx_mtg_cards_set_id ON mtg_cards(set_id);", - ] - - for idx_sql in indexes: - await session.execute(text(idx_sql)) - - print("✓ All mtgdata tables created") - - await engine.dispose() - - -async def main(): - """Main initialization function.""" - print("MTG Online Database Initialization") - print("=" * 50) - - # Setup mtgonline database - print("\nSetting up mtgonline database...") - await setup_mtg_online_database() - - # Setup mtgdata database - print("\nSetting up mtgdata database...") - await setup_mtg_data_database() - - print("\n✓ Database initialization complete!") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/backend/test_download.py b/backend/test_download.py deleted file mode 100644 index 746390b..0000000 --- a/backend/test_download.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Test MTGJSON API endpoints and download JSON files.""" -import asyncio -import aiohttp -from pathlib import Path - -DATA_DIR = Path("/app/data/mtgjson") -DATA_DIR.mkdir(parents=True, exist_ok=True) - -# MTGJSON API endpoints for .json files -MTGJSON_BASE_URL = "https://mtgjson.com/api/v5" -JSON_FILES = { - "AllPrintings.json": MTGJSON_BASE_URL + "/AllPrintings.json", - "AllSetFiles.json": MTGJSON_BASE_URL + "/AllSetFiles.json", - "AllIdentifiers.json": MTGJSON_BASE_URL + "/AllIdentifiers.json", - "CardTypes.json": MTGJSON_BASE_URL + "/CardTypes.json", - "DeckList.json": MTGJSON_BASE_URL + "/DeckList.json", - "Keywords.json": MTGJSON_BASE_URL + "/Keywords.json", - "SetList.json": MTGJSON_BASE_URL + "/SetList.json", -} - -async def test_download_file(session, url, filename): - """Test downloading a single JSON file.""" - dest_path = DATA_DIR / filename - print(f"Testing {filename}...") - - try: - async with session.get(url) as response: - if response.status == 200: - content = await response.read() - print(f" ✓ Status: {response.status}") - print(f" ✓ Size: {len(content) / (1024*1024):.2f} MB") - print(f" ✓ Content-Type: {response.content_type}") - - # Write to file - with open(dest_path, 'wb') as f: - f.write(content) - print(f" ✓ Saved to {dest_path}") - - # Test JSON parsing - import json - with open(dest_path, 'r', encoding='utf-8') as f: - data = json.load(f) - print(f" ✓ Valid JSON: {type(data).__name__}") - - return True - else: - print(f" ✗ Status: {response.status}") - return False - except Exception as e: - print(f" ✗ Error: {e}") - return False - -async def main(): - """Test all MTGJSON JSON endpoints.""" - print("Testing MTGJSON JSON API endpoints...\n") - - async with aiohttp.ClientSession() as session: - results = [] - for filename, url in JSON_FILES.items(): - success = await test_download_file(session, url, filename) - results.append((filename, success)) - print() - - print("\n=== Results ===") - for filename, success in results: - status = "✓" if success else "✗" - print(f"{status} {filename}") - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/backend/test_system.py b/backend/test_system.py deleted file mode 100644 index 4072183..0000000 --- a/backend/test_system.py +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env python3 -""" -Backend System Test Script - -Tests all backend components: -- Database connections (MTG Online + MTG) -- Redis connection and caching -- Card database queries -- API endpoints -- Health checks -""" - -import asyncio -import sys -from datetime import datetime - -# Add backend to path -sys.path.insert(0, '/home/wall-o/projects/mtgonline/backend') - -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy import text - -# Database URLs -COCKATRICE_DB = "postgresql+asyncpg://mtgonline:mtgonline_pass@localhost:5432/mtgonline" -MTG_DB = "postgresql+asyncpg://mtgonline:mtgonline_pass@localhost:5432/mtgdata" -REDIS_URL = "redis://localhost:6379/0" - -# Test URLs - detect environment -import socket -try: - # Try Docker network first - socket.getaddrinfo('mtg_backend', 8000) - API_BASE = 'http://mtg_backend:8000' -except: - # Fall back to host port - API_BASE = 'http://localhost:8001' -print(f"Using API base: {API_BASE}") -results = { - "tests_run": 0, - "tests_passed": 0, - "tests_failed": 0, - "failures": [], - "timestamp": datetime.now().isoformat() -} - -async def test_connection(db_url: str, name: str) -> bool: - """Test database connection.""" - results["tests_run"] += 1 - try: - engine = create_async_engine(db_url, echo=False) - async with engine.connect() as conn: - result = await conn.execute(text("SELECT 1")) - await engine.dispose() - - results["tests_passed"] += 1 - print(f"✓ {name} connection: OK") - return True - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"{name} connection: {str(e)}") - print(f"✗ {name} connection: FAILED - {str(e)}") - return False - -async def test_redis() -> bool: - """Test Redis connection.""" - results["tests_run"] += 1 - try: - import redis.asyncio as aioredis - client = aioredis.from_url(REDIS_URL, decode_responses=True) - await client.ping() - await client.close() - - results["tests_passed"] += 1 - print("✓ Redis connection: OK") - return True - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Redis connection: {str(e)}") - print(f"✗ Redis connection: FAILED - {str(e)}") - return False - -async def test_cache_operations() -> bool: - """Test Redis cache operations.""" - results["tests_run"] += 1 - try: - import redis.asyncio as aioredis - client = aioredis.from_url(REDIS_URL, decode_responses=True) - - # Test set/get - await client.set("test_key", "test_value", ex=60) - value = await client.get("test_key") - assert value == "test_value", f"Expected 'test_value', got '{value}'" - - # Test delete - await client.delete("test_key") - value = await client.get("test_key") - assert value is None, f"Expected None after delete, got '{value}'" - - await client.close() - - results["tests_passed"] += 1 - print("✓ Redis cache operations: OK") - return True - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Redis cache operations: {str(e)}") - print(f"✗ Redis cache operations: FAILED - {str(e)}") - return False - -async def test_mtgonline_tables() -> bool: - """Test MTG Online database tables.""" - results["tests_run"] += 1 - try: - engine = create_async_engine(COCKATRICE_DB, echo=False) - async with engine.connect() as conn: - # Test users table - result = await conn.execute(text("SELECT COUNT(*) FROM mtgonline_users")) - user_count = result.scalar() - - # Test decks table - result = await conn.execute(text("SELECT COUNT(*) FROM mtgonline_decklist_files")) - deck_count = result.scalar() - - await engine.dispose() - - results["tests_passed"] += 1 - print(f"✓ MTG Online tables: OK (users: {user_count}, decks: {deck_count})") - return True - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"MTG Online tables: {str(e)}") - print(f"✗ MTG Online tables: FAILED - {str(e)}") - return False - -async def test_mtg_tables() -> bool: - """Test MTG database tables.""" - results["tests_run"] += 1 - try: - engine = create_async_engine(MTG_DB, echo=False) - async with engine.connect() as conn: - # Test mtg_sets table - result = await conn.execute(text("SELECT COUNT(*) FROM mtg_sets")) - set_count = result.scalar() - - # Test mtg_cards table - result = await conn.execute(text("SELECT COUNT(*) FROM mtg_cards")) - card_count = result.scalar() - - await engine.dispose() - - results["tests_passed"] += 1 - print(f"✓ MTG tables: OK (sets: {set_count}, cards: {card_count})") - return True - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"MTG tables: {str(e)}") - print(f"✗ MTG tables: FAILED - {str(e)}") - return False - -async def test_mtg_card_query() -> bool: - """Test MTG card query functionality.""" - results["tests_run"] += 1 - try: - engine = create_async_engine(MTG_DB, echo=False) - async with engine.connect() as conn: - # Search for a common card - result = await conn.execute( - text("SELECT name, type_line, rarity FROM mtg_cards WHERE name ILIKE '%Black Lotus%' LIMIT 1") - ) - row = result.fetchone() - - await engine.dispose() - - if row: - results["tests_passed"] += 1 - print(f"✓ MTG card query: OK - Found '{row[0]}' ({row[1]}, {row[2]})") - return True - else: - results["tests_failed"] += 1 - results["failures"].append("MTG card query: No results found") - print("✗ MTG card query: FAILED - No results found") - return False - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"MTG card query: {str(e)}") - print(f"✗ MTG card query: FAILED - {str(e)}") - return False - -async def test_health_endpoint() -> bool: - """Test backend health endpoint.""" - results["tests_run"] += 1 - try: - import requests - response = requests.get("http://localhost:8001/health", timeout=5) - - if response.status_code == 200: - results["tests_passed"] += 1 - print(f"✓ Health endpoint: OK ({response.json()})") - return True - else: - results["tests_failed"] += 1 - results["failures"].append(f"Health endpoint: Status {response.status_code}") - print(f"✗ Health endpoint: FAILED - Status {response.status_code}") - return False - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Health endpoint: {str(e)}") - print(f"✗ Health endpoint: FAILED - {str(e)}") - return False - -async def test_card_search_endpoint() -> bool: - """Test card search API endpoint.""" - results["tests_run"] += 1 - try: - import requests - response = requests.get( - "http://localhost:8001/api/v1/mtg/cards/search", - params={"q": "Lightning Bolt", "limit": 5}, - timeout=10 - ) - - if response.status_code == 200: - data = response.json() - results["tests_passed"] += 1 - print(f"✓ Card search endpoint: OK - Found {len(data.get('results', {}).get('results', []))} cards") - return True - else: - results["tests_failed"] += 1 - results["failures"].append(f"Card search endpoint: Status {response.status_code}") - print(f"✗ Card search endpoint: FAILED - Status {response.status_code}") - return False - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Card search endpoint: {str(e)}") - print(f"✗ Card search endpoint: FAILED - {str(e)}") - return False - -async def test_statistics_endpoint() -> bool: - """Test card statistics API endpoint.""" - results["tests_run"] += 1 - try: - import requests - response = requests.get( - "http://localhost:8001/api/v1/mtg/cards/statistics", - timeout=10 - ) - - if response.status_code == 200: - data = response.json() - results["tests_passed"] += 1 - stats = data.get("results", {}).get("results", {}) - print(f"✓ Statistics endpoint: OK - {stats.get('total_cards', 0)} cards, {stats.get('total_sets', 0)} sets") - return True - else: - results["tests_failed"] += 1 - results["failures"].append(f"Statistics endpoint: Status {response.status_code}") - print(f"✗ Statistics endpoint: FAILED - Status {response.status_code}") - return False - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Statistics endpoint: {str(e)}") - print(f"✗ Statistics endpoint: FAILED - {str(e)}") - return False - -async def test_auth_endpoint() -> bool: - """Test authentication endpoint.""" - results["tests_run"] += 1 - try: - import requests - response = requests.post( - "http://localhost:8001/api/v1/auth/register", - json={ - "username": "test_user", - "email": "test@example.com", - "password": "TestPass123!", - "confirm_password": "TestPass123!" - }, - timeout=10 - ) - - if response.status_code in [200, 409]: # 409 means user already exists - results["tests_passed"] += 1 - print(f"✓ Auth endpoint: OK - Status {response.status_code}") - return True - else: - results["tests_failed"] += 1 - results["failures"].append(f"Auth endpoint: Status {response.status_code}") - print(f"✗ Auth endpoint: FAILED - Status {response.status_code}") - return False - except Exception as e: - results["tests_failed"] += 1 - results["failures"].append(f"Auth endpoint: {str(e)}") - print(f"✗ Auth endpoint: FAILED - {str(e)}") - return False - -async def run_all_tests(): - """Run all system tests.""" - print("=" * 60) - print("BACKEND SYSTEM TEST") - print("=" * 60) - print(f"Started at: {datetime.now().isoformat()}") - print("=" * 60) - print() - - # Database connections - print("Testing Database Connections:") - print("-" * 60) - await test_connection(COCKATRICE_DB, "MTG Online PostgreSQL") - await test_connection(MTG_DB, "MTG PostgreSQL") - await test_redis() - print() - - # Cache operations - print("Testing Cache Operations:") - print("-" * 60) - await test_cache_operations() - print() - - # Database tables - print("Testing Database Tables:") - print("-" * 60) - await test_mtgonline_tables() - await test_mtg_tables() - await test_mtg_card_query() - print() - - # API endpoints - print("Testing API Endpoints:") - print("-" * 60) - await test_health_endpoint() - await test_card_search_endpoint() - await test_statistics_endpoint() - await test_auth_endpoint() - print() - - # Summary - print("=" * 60) - print("TEST SUMMARY") - print("=" * 60) - print(f"Tests Run: {results['tests_run']}") - print(f"Tests Passed: {results['tests_passed']}") - print(f"Tests Failed: {results['tests_failed']}") - print(f"Success Rate: {(results['tests_passed'] / results['tests_run'] * 100):.1f}%") - print("=" * 60) - - if results["failures"]: - print("\nFailures:") - for failure in results["failures"]: - print(f" • {failure}") - - print(f"\nCompleted at: {datetime.now().isoformat()}") - print("=" * 60) - - return results["tests_failed"] == 0 - -if __name__ == "__main__": - success = asyncio.run(run_all_tests()) - sys.exit(0 if success else 1) diff --git a/state.json b/state.json index 676e3c7..2e94cc9 100644 --- a/state.json +++ b/state.json @@ -1,48 +1,89 @@ { - "task_description": "Complete documentation and cleanup of MTG Online Backend project", - "current_step": "All tasks completed: documentation, commit/push to Gitea, Docker cleanup", + "project_summary": "MTG Online Backend — a Python FastAPI application that processes Magic: The Gathering card data from MTGJSON v5 and stores it in PostgreSQL, with Redis for caching. Exposes REST endpoints for card data, user authentication, deck management, and game state. Targets a web-based MTG card browsing and deck-building platform.", + "roadmap": [ + { + "phase": "Phase 1: Foundation & Data Pipeline", + "status": "completed", + "description": "Backend scaffolding, dual PostgreSQL setup, Redis, Docker Compose, MTGJSON download and upsert pipeline.", + "key_deliverables": ["FastAPI app running", "Dual Postgres (app + MTG data)", "Redis cache", "MTGJSON v5 pipeline", "22k+ cards loaded"] + }, + { + "phase": "Phase 2: Core API Endpoints", + "status": "in_progress", + "description": "REST endpoints for card search, deck management, user auth, and game state.", + "key_deliverables": ["Card search API", "Deck CRUD", "Auth system", "Health check", "MTGJSON refresh endpoint"] + }, + { + "phase": "Phase 3: Frontend", + "status": "pending", + "description": "Web interface for deck building and card browsing.", + "key_deliverables": ["React/Next.js frontend", "API integration", "Responsive UI", "Docker Compose integration"] + }, + { + "phase": "Phase 4: Advanced Features", + "status": "pending", + "description": "Game server integration, multiplayer support, card image serving, performance optimization.", + "key_deliverables": ["Game server", "Multiplayer", "Card images", "Caching optimization"] + } + ], + "tech_stack": ["Python 3.12", "FastAPI", "SQLAlchemy (async)", "PostgreSQL x2", "Redis", "Docker Compose", "MTGJSON v5"], + "architectural_notes": "Dual PostgreSQL (mtgonline for app data, mtgdata for MTG card data), Redis caching layer, MTGJSON v5 data pipeline auto-downloads on startup, REST API with Swagger docs at /docs. Backend exposed on port 5555. C++ game server directory exists but not yet integrated.", + "task_description": "Clean up backend folder - remove obsolete scripts, test files, and unused modules", + "current_step": "Backend cleanup completed: removed obsolete scripts, test files, __pycache__, venv, and unused modules", "files_created": [ "/home/wall-o/projects/mtgonline/README.md", "/home/wall-o/projects/mtgonline/backend/README.md" ], - "files_modified": [ - "/home/wall-o/projects/mtgonline/README.md", - "/home/wall-o/projects/mtgonline/state.json" + "files_removed": [ + "BACKEND_TESTING_SUMMARY.md", + "CHAT_PROMPT_TEST.md", + "CONTINUATION_PROMPT.md", + "PORTED_STATE.md", + "SPEC_synergy-mapping-engine.md", + "STATE.md", + "SUPPORTED_FILE_TYPES.md", + "test.db", + "test_download.py", + "test_system.py", + "setup_db.py", + ".env.local", + "state.json (backend)", + "card_interaction_rule_engine.py", + "card_profile_extractor.py", + "create_card_interaction_graph.py", + "interaction_determinator.py", + "interaction_pipeline.py", + "interaction_recommender.py", + "interaction_schema.py", + "recommendation_engine.py", + "migrate_complete.py", + "migrate_schema.py", + "test_interaction_determinator.py", + "check_mtgjson_full.py", + "check_mtgjson_status.py", + "verify_integration.py", + "verify_mtgjson_data.py", + "sanity_check_mtgjson.py", + "investigate_sets.py", + "inspect_db.py", + "code_review.md", + "monitor/mtg_monitor.py" ], "decisions": [ - "Updated root README with current architecture (dual PostgreSQL, Redis, MTGJSON pipeline)", - "Created comprehensive backend README with architecture, database setup, and troubleshooting", - "Hardcoded environment variables in docker-compose.dev.yml to prevent connection issues", - "Backend successfully connects to mtgdata:5432/mtgdata (not localhost)", - "All Docker resources cleaned up: containers, images, volumes, networks" - ], - "next_steps": [], - "blockers": [], - "commit_hash": "46abfe5", - "timestamp": "2026-07-21T03:56:00Z" -} - "task_description": "Complete documentation and cleanup of MTG Online Backend project", - "current_step": "Documentation complete, committing and pushing to Gitea, then cleaning up Docker", - "files_created": [ - "/home/wall-o/projects/mtgonline/README.md", - "/home/wall-o/projects/mtgonline/backend/README.md" - ], - "files_modified": [ - "/home/wall-o/projects/mtgonline/README.md" - ], - "decisions": [ - "Updated root README to reflect current architecture (dual PostgreSQL, Redis, MTGJSON pipeline)", - "Created comprehensive backend README with architecture, database setup, MTGJSON pipeline, and troubleshooting", - "Hardcoded environment variables in docker-compose.dev.yml to prevent connection issues", - "Backend successfully connects to mtgdata:5432/mtgdata (not localhost)" + "Removed obsolete scripts that were not imported or used by the application", + "Removed test.db and test-related scripts that were development artifacts", + "Removed __pycache__ directories to clean up bytecode cache", + "Removed venv directory (created fresh in Docker build)", + "Removed monitor/mtg_monitor.py (unused monitoring script)", + "Removed .env.local (sensitive environment file)", + "Backend scripts/ directory now contains only essential data loading and maintenance scripts" ], "next_steps": [ - "Commit and push changes to Gitea repository", - "Stop and destroy all Docker containers", - "Clear Docker build cache", - "Run docker system prune to remove all traces" + "Commit cleanup changes to Gitea", + "Resume development on Phase 2 API endpoints", + "Or begin Phase 3 frontend planning" ], "blockers": [], - "commit_hash": "", - "timestamp": "2026-07-21T03:47:00Z" + "commit_hash": "46abfe5", + "timestamp": "2026-07-22T02:59:00Z" }