From 16c504dfb5f768c597fe66de935e8a97ed6cb823 Mon Sep 17 00:00:00 2001 From: Syco21 Date: Fri, 13 Mar 2026 20:07:05 +0100 Subject: [PATCH] Card Import Works --- src/controllers/importController.js | 150 +++++++++++++++++++++++++++- src/models/cardImageModel.js | 13 +++ src/models/cardModel.js | 34 +++++++ src/models/cardSetRarityModel.js | 13 +++ src/models/rarityModel.js | 22 ++++ src/routes/importRoutes.js | 3 +- src/services/ygoproService.js | 13 ++- 7 files changed, 242 insertions(+), 6 deletions(-) diff --git a/src/controllers/importController.js b/src/controllers/importController.js index ec8c2db..c58dcd4 100644 --- a/src/controllers/importController.js +++ b/src/controllers/importController.js @@ -1,7 +1,16 @@ -const { fetchAllSets } = require('../services/ygoproService'); -const { upsertSet } = require('../models/setModel'); +const { + upsertCard +} = require('../models/cardModel'); +const { + upsertRarity, + getRarityId +} = require('../models/rarityModel'); +const { insertCardSetRarity } = require('../models/cardSetRarityModel'); +const { insertCardImage } = require('../models/cardImageModel'); +const { fetchAllCards } = require('../services/ygoproService'); +const db = require('../config/db'); -const BATCH_SIZE = 10; +const BATCH_SIZE = 50; async function importSets(req, res) { try { @@ -27,4 +36,137 @@ async function importSets(req, res) { } } -module.exports = { importSets }; \ No newline at end of file +async function importCards(req, res) { + const startTime = Date.now(); + + try { + const cards = await fetchAllCards(); + const totalCards = cards.length; + + let addedCards = 0, addedImages = 0, addedRarities = 0; + + // Cache sets and rarities to reduce repeated DB lookups + const rarityCache = {}; + const setCache = {}; + + // Preload sets into cache + const [setsRows] = await db.execute('SELECT id, set_code FROM sets'); + for (const row of setsRows) { + setCache[row.set_code] = row.id; + } + + // Preload rarities into cache + const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities'); + for (const row of rarityRows) { + rarityCache[row.rarity_name] = row.id; + } + + // Process cards in batches + for (let i = 0; i < totalCards; i += BATCH_SIZE) { + const batch = cards.slice(i, i + BATCH_SIZE); + + // Wrap batch in a transaction + const conn = await db.getConnection(); + try { + await conn.beginTransaction(); + + await Promise.all(batch.map(async (card) => { + // Upsert card + await conn.execute(` + INSERT INTO cards + (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + name = VALUES(name), + card_type = VALUES(card_type), + frame_type = VALUES(frame_type), + level = VALUES(level), + race = VALUES(race), + attribute = VALUES(attribute), + link_val = VALUES(link_val), + tcg_date = VALUES(tcg_date), + ocg_date = VALUES(ocg_date) + `, [ + card.id, + card.name, + card.type, + card.frameType, + card.level || null, + card.race || null, + card.attribute || null, + card.linkval || null, + card.tcg_date || null, + card.ocg_date || null + ]); + addedCards++; + + // Process rarities and card_sets_rarity + if (Array.isArray(card.card_sets)) { + for (const set of card.card_sets) { + // Upsert rarity if not cached + if (!rarityCache[set.set_rarity]) { + await conn.execute(` + INSERT INTO rarities (rarity_name, rarity_code) + VALUES (?, ?) + ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code) + `, [set.set_rarity, set.set_rarity_code]); + const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]); + rarityCache[set.set_rarity] = rows[0].id; + addedRarities++; + } + const rarity_id = rarityCache[set.set_rarity]; + + // Get set_id from cache + const set_id = setCache[set.set_code]; + if (set_id) { + await conn.execute(` + INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code) + `, [card.id, set_id, rarity_id, set.set_code]); + } + } + } + + // Process card images + if (Array.isArray(card.card_images)) { + for (const img of card.card_images) { + await conn.execute(` + INSERT INTO card_images (card_id, image_url) + VALUES (?, ?) + `, [card.id, img.image_url]); + addedImages++; + } + } + })); + + await conn.commit(); + } catch (err) { + await conn.rollback(); + throw err; + } finally { + conn.release(); + } + + // Release batch memory + batch.length = 0; + } + + const endTime = Date.now(); + const durationSeconds = ((endTime - startTime) / 1000).toFixed(2); + + res.json({ + total_cards: totalCards, + cards_added: addedCards, + images_added: addedImages, + rarities_added: addedRarities, + duration_seconds: parseFloat(durationSeconds) + }); + + } catch (err) { + console.error('Error importing cards:', err); + res.status(500).json({ error: 'Failed to import cards' }); + } +} + +module.exports = { importCards, importSets }; \ No newline at end of file diff --git a/src/models/cardImageModel.js b/src/models/cardImageModel.js index e69de29..5e85fb4 100644 --- a/src/models/cardImageModel.js +++ b/src/models/cardImageModel.js @@ -0,0 +1,13 @@ +const db = require('../config/db'); + +async function insertCardImage(image) { + const { card_id, image_url } = image; + + const sql = ` + INSERT INTO card_images (card_id, image_url) + VALUES (?, ?) + `; + await db.execute(sql, [card_id, image_url]); +} + +module.exports = { insertCardImage }; \ No newline at end of file diff --git a/src/models/cardModel.js b/src/models/cardModel.js index e69de29..9113eeb 100644 --- a/src/models/cardModel.js +++ b/src/models/cardModel.js @@ -0,0 +1,34 @@ +const db = require('../config/db'); + +async function upsertCard(card) { + const { + id, + name, + type, + frameType, + level, + race, + attribute, + linkval, + tcg_date, + ocg_date + } = card; + + const sql = ` + INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + name = VALUES(name), + card_type = VALUES(card_type), + frame_type = VALUES(frame_type), + level = VALUES(level), + race = VALUES(race), + attribute = VALUES(attribute), + link_val = VALUES(link_val), + tcg_date = VALUES(tcg_date), + ocg_date = VALUES(ocg_date) + `; + await db.execute(sql, [id, name, type, frameType, level, race, attribute, linkval, tcg_date, ocg_date]); +} + +module.exports = { upsertCard }; \ No newline at end of file diff --git a/src/models/cardSetRarityModel.js b/src/models/cardSetRarityModel.js index e69de29..5bf1a65 100644 --- a/src/models/cardSetRarityModel.js +++ b/src/models/cardSetRarityModel.js @@ -0,0 +1,13 @@ +const db = require('../config/db'); + +async function insertCardSetRarity(entry) { + const { card_id, set_id, rarity_id, card_set_code } = entry; + const sql = ` + INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code) + `; + await db.execute(sql, [card_id, set_id, rarity_id, card_set_code]); +} + +module.exports = { insertCardSetRarity }; \ No newline at end of file diff --git a/src/models/rarityModel.js b/src/models/rarityModel.js index e69de29..b9e57ce 100644 --- a/src/models/rarityModel.js +++ b/src/models/rarityModel.js @@ -0,0 +1,22 @@ +const db = require('../config/db'); + +async function upsertRarity(rarity) { + const { rarity_name, rarity_code } = rarity; + + const sql = ` + INSERT INTO rarities (rarity_name, rarity_code) + VALUES (?, ?) + ON DUPLICATE KEY UPDATE + rarity_code = VALUES(rarity_code) + `; + const [result] = await db.execute(sql, [rarity_name, rarity_code]); + return result.insertId; // returns id for junction table +} + +async function getRarityId(rarity_name) { + const sql = `SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1`; + const [rows] = await db.execute(sql, [rarity_name]); + return rows.length ? rows[0].id : null; +} + +module.exports = { upsertRarity, getRarityId }; \ No newline at end of file diff --git a/src/routes/importRoutes.js b/src/routes/importRoutes.js index 623c234..d3bff93 100644 --- a/src/routes/importRoutes.js +++ b/src/routes/importRoutes.js @@ -1,7 +1,8 @@ const express = require('express'); const router = express.Router(); -const { importSets } = require('../controllers/importController'); +const { importSets, importCards} = require('../controllers/importController'); router.post('/sets', importSets); +router.post('/cards', importCards); module.exports = router; \ No newline at end of file diff --git a/src/services/ygoproService.js b/src/services/ygoproService.js index 69ce51a..4e39aaa 100644 --- a/src/services/ygoproService.js +++ b/src/services/ygoproService.js @@ -19,4 +19,15 @@ async function fetchAllSets() { } } -module.exports = { fetchAllSets }; \ No newline at end of file +async function fetchAllCards() { + try { + const response = await axios.get(`${API_BASE}/cardinfo.php?misc=yes`); + return response.data.data; // array of card objects + } catch (err) { + console.error('Error fetching cards from YGOPRODeck:', err.message); + throw err; + } +} + + +module.exports = { fetchAllCards, fetchAllSets }; \ No newline at end of file