From e82a50458b32d10a2156764719e12d55c5b0e532 Mon Sep 17 00:00:00 2001 From: Syco21 Date: Fri, 15 May 2026 21:57:03 +0200 Subject: [PATCH] Optimize import: bulk INSERTs, pre-resolve rarities, BATCH_SIZE 50->500 --- src/controllers/importController.js | 201 ++++++++++++++-------------- 1 file changed, 102 insertions(+), 99 deletions(-) diff --git a/src/controllers/importController.js b/src/controllers/importController.js index dea07a4..2d76d68 100644 --- a/src/controllers/importController.js +++ b/src/controllers/importController.js @@ -1,33 +1,28 @@ -const { upsertCard } = require('../models/cardModel'); -const { upsertRarity } = require('../models/rarityModel'); -const { upsertSet } = require('../models/setModel'); -const { insertCardSetRarity } = require('../models/cardSetRarityModel'); -const { insertCardImage } = require('../models/cardImageModel'); const { getLocalDBVersion, setLocalDBVersion } = require('../models/dbVersionModel'); const { fetchAllCards, fetchAllSets, fetchDatabaseVersion } = require('../services/ygoproService'); const db = require('../config/db'); -const BATCH_SIZE = 50; +const BATCH_SIZE = 500; async function importSetsInternal() { const sets = await fetchAllSets(); - let added = 0; + if (!sets.length) return { added: 0, total: 0 }; - for (let i = 0; i < sets.length; i += BATCH_SIZE) { - const batch = sets.slice(i, i + BATCH_SIZE); - await Promise.all(batch.map(async (set) => { - await upsertSet(set); - added++; - })); - } + const values = sets.map(() => '(?, ?, ?, ?)').join(', '); + const params = sets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]); - return { - added, - total: sets.length - }; + await db.execute(` + INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date) + VALUES ${values} + ON DUPLICATE KEY UPDATE + set_name = VALUES(set_name), + num_of_cards = VALUES(num_of_cards), + tcg_date = VALUES(tcg_date) + `, params); + + return { added: sets.length, total: sets.length }; } -// Express handler for /sets endpoint async function importSets(req, res) { try { const result = await importSetsInternal(); @@ -43,114 +38,123 @@ async function importCardsInternal() { const cards = await fetchAllCards(); const totalCards = cards.length; - let addedCards = 0, addedImages = 0, addedRarities = 0; - - const rarityCache = {}; - const setCache = {}; - + // Pre-load set and rarity caches const [setsRows] = await db.execute('SELECT id, set_name FROM sets'); - for (const row of setsRows) { - setCache[row.set_name] = row.id; - } + const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id])); const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities'); - for (const row of rarityRows) { - rarityCache[row.rarity_name] = row.id; + const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id])); + + // Collect all unique rarities from the full card list and upsert them before the main loop + const unseenRarities = new Map(); + for (const card of cards) { + if (!Array.isArray(card.card_sets)) continue; + for (const set of card.card_sets) { + if (set.set_rarity && !rarityCache[set.set_rarity] && !unseenRarities.has(set.set_rarity)) { + unseenRarities.set(set.set_rarity, set.set_rarity_code); + } + } } + if (unseenRarities.size) { + const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', '); + const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]); + await db.execute(` + INSERT INTO rarities (rarity_name, rarity_code) + VALUES ${rarityValues} + ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code) + `, rarityParams); + + const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities'); + for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id; + } + + let addedCards = 0, addedImages = 0, addedSetRarities = 0; + for (let i = 0; i < totalCards; i += BATCH_SIZE) { const batch = cards.slice(i, i + BATCH_SIZE); + + const cardValues = [], cardParams = []; + const imageValues = [], imageParams = []; + const csrValues = [], csrParams = []; + + for (const card of batch) { + cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + cardParams.push( + card.id, card.name, card.type, card.frameType, + card.level || null, card.race || null, card.attribute || null, + card.linkval || null, card.tcg_date || null, card.ocg_date || null + ); + + if (Array.isArray(card.card_images)) { + for (const img of card.card_images) { + imageValues.push('(?, ?, ?)'); + imageParams.push(img.id, card.id, img.image_url); + } + } + + if (Array.isArray(card.card_sets)) { + for (const set of card.card_sets) { + const set_id = setCache[set.set_name]; + const rarity_id = rarityCache[set.set_rarity]; + if (set_id && rarity_id) { + csrValues.push('(?, ?, ?, ?)'); + csrParams.push(card.id, set_id, rarity_id, set.set_code); + } + } + } + } + const conn = await db.getConnection(); try { await conn.beginTransaction(); - await Promise.all(batch.map(async (card) => { + await conn.execute(` + INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) + VALUES ${cardValues.join(', ')} + ON DUPLICATE KEY UPDATE + name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type), + level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute), + link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date) + `, cardParams); + + if (imageValues.length) { await conn.execute(` - INSERT INTO cards - (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - name = VALUES(name), - card_type = VALUES(card_type), - frame_type = VALUES(frame_type), - level = VALUES(level), - race = VALUES(race), - attribute = VALUES(attribute), - link_val = VALUES(link_val), - tcg_date = VALUES(tcg_date), - ocg_date = VALUES(ocg_date) - `, [ - card.id, - card.name, - card.type, - card.frameType, - card.level || null, - card.race || null, - card.attribute || null, - card.linkval || null, - card.tcg_date || null, - card.ocg_date || null - ]); - addedCards++; + INSERT INTO card_images (image_id, card_id, image_url) + VALUES ${imageValues.join(', ')} + ON DUPLICATE KEY UPDATE image_url = VALUES(image_url) + `, imageParams); + } - if (Array.isArray(card.card_sets)) { - for (const set of card.card_sets) { - if (!rarityCache[set.set_rarity]) { - await conn.execute(` - INSERT INTO rarities (rarity_name, rarity_code) - VALUES (?, ?) - ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code) - `, [set.set_rarity, set.set_rarity_code]); - const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]); - rarityCache[set.set_rarity] = rows[0].id; - addedRarities++; - } - const rarity_id = rarityCache[set.set_rarity]; - const set_id = setCache[set.set_name]; - if (set_id) { - await conn.execute(` - INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) - VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code) - `, [card.id, set_id, rarity_id, set.set_code]); - } - } - } - - if (Array.isArray(card.card_images)) { - for (const img of card.card_images) { - await conn.execute(` - INSERT INTO card_images (image_id, card_id, image_url) - VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE image_url = VALUES(image_url) - `, [img.id, card.id, img.image_url]); - addedImages++; - } - } - })); + if (csrValues.length) { + await conn.execute(` + INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) + VALUES ${csrValues.join(', ')} + ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code) + `, csrParams); + } await conn.commit(); + addedCards += batch.length; + addedImages += imageValues.length; + addedSetRarities += csrValues.length; } catch (err) { await conn.rollback(); throw err; } finally { conn.release(); } - } - const durationSeconds = ((Date.now() - startTime) / 1000).toFixed(2); - return { total_cards: totalCards, cards_added: addedCards, images_added: addedImages, - rarities_added: addedRarities, - duration_seconds: parseFloat(durationSeconds) + set_rarities_added: addedSetRarities, + duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)), }; } -// Express handler for /cards async function importCards(req, res) { try { const result = await importCardsInternal(); @@ -161,7 +165,6 @@ async function importCards(req, res) { } } - async function importFullDatabase(req, res) { const startTime = Date.now(); try { @@ -189,4 +192,4 @@ async function importFullDatabase(req, res) { } } -module.exports = { importCards, importCardsInternal, importSets, importSetsInternal, importFullDatabase }; \ No newline at end of file +module.exports = { importCards, importCardsInternal, importSets, importSetsInternal, importFullDatabase };