Syco/YuGiOh-Database-API
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize import: bulk INSERTs, pre-resolve rarities, BATCH_SIZE 50->500
ci/woodpecker/push/woodpecker Pipeline was successful
Details

Browse Source
This commit is contained in:
Syco21
2026-05-15 21:57:03 +02:00
parent 09eea6d176
commit e82a50458b
1 changed files with 102 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/controllers/importController.js
+95 -92
View File
@@ -1,33 +1,28 @@
const { upsertCard } = require('../models/cardModel');
const { upsertRarity } = require('../models/rarityModel');
const { upsertSet } = require('../models/setModel');
const { insertCardSetRarity } = require('../models/cardSetRarityModel');
const { insertCardImage } = require('../models/cardImageModel');
const { getLocalDBVersion, setLocalDBVersion } = require('../models/dbVersionModel');
const { fetchAllCards, fetchAllSets, fetchDatabaseVersion } = require('../services/ygoproService');
const db = require('../config/db');
const BATCH_SIZE = 50;
const BATCH_SIZE = 500;
async function importSetsInternal() {
const sets = await fetchAllSets();
let added = 0;
if (!sets.length) return { added: 0, total: 0 };
for (let i = 0; i < sets.length; i += BATCH_SIZE) {
const batch = sets.slice(i, i + BATCH_SIZE);
await Promise.all(batch.map(async (set) => {
await upsertSet(set);
added++;
}));
}
const values = sets.map(() => '(?, ?, ?, ?)').join(', ');
const params = sets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]);
return {
added,
total: sets.length
};
await db.execute(`
INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date)
VALUES ${values}
ON DUPLICATE KEY UPDATE
set_name = VALUES(set_name),
num_of_cards = VALUES(num_of_cards),
tcg_date = VALUES(tcg_date)
`, params);
return { added: sets.length, total: sets.length };
}
// Express handler for /sets endpoint
async function importSets(req, res) {
try {
const result = await importSetsInternal();
@@ -43,114 +38,123 @@ async function importCardsInternal() {
const cards = await fetchAllCards();
const totalCards = cards.length;
let addedCards = 0, addedImages = 0, addedRarities = 0;
const rarityCache = {};
const setCache = {};
// Pre-load set and rarity caches
const [setsRows] = await db.execute('SELECT id, set_name FROM sets');
for (const row of setsRows) {
setCache[row.set_name] = row.id;
}
const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id]));
const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
for (const row of rarityRows) {
rarityCache[row.rarity_name] = row.id;
const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id]));
// Collect all unique rarities from the full card list and upsert them before the main loop
const unseenRarities = new Map();
for (const card of cards) {
if (!Array.isArray(card.card_sets)) continue;
for (const set of card.card_sets) {
if (set.set_rarity && !rarityCache[set.set_rarity] && !unseenRarities.has(set.set_rarity)) {
unseenRarities.set(set.set_rarity, set.set_rarity_code);
}
}
}
if (unseenRarities.size) {
const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', ');
const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]);
await db.execute(`
INSERT INTO rarities (rarity_name, rarity_code)
VALUES ${rarityValues}
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
`, rarityParams);
const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id;
}
let addedCards = 0, addedImages = 0, addedSetRarities = 0;
for (let i = 0; i < totalCards; i += BATCH_SIZE) {
const batch = cards.slice(i, i + BATCH_SIZE);
const cardValues = [], cardParams = [];
const imageValues = [], imageParams = [];
const csrValues = [], csrParams = [];
for (const card of batch) {
cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
cardParams.push(
card.id, card.name, card.type, card.frameType,
card.level || null, card.race || null, card.attribute || null,
card.linkval || null, card.tcg_date || null, card.ocg_date || null
);
if (Array.isArray(card.card_images)) {
for (const img of card.card_images) {
imageValues.push('(?, ?, ?)');
imageParams.push(img.id, card.id, img.image_url);
}
}
if (Array.isArray(card.card_sets)) {
for (const set of card.card_sets) {
const set_id = setCache[set.set_name];
const rarity_id = rarityCache[set.set_rarity];
if (set_id && rarity_id) {
csrValues.push('(?, ?, ?, ?)');
csrParams.push(card.id, set_id, rarity_id, set.set_code);
}
}
}
}
const conn = await db.getConnection();
try {
await conn.beginTransaction();
await Promise.all(batch.map(async (card) => {
await conn.execute(`
INSERT INTO cards
(id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
VALUES ${cardValues.join(', ')}
ON DUPLICATE KEY UPDATE
name = VALUES(name),
card_type = VALUES(card_type),
frame_type = VALUES(frame_type),
level = VALUES(level),
race = VALUES(race),
attribute = VALUES(attribute),
link_val = VALUES(link_val),
tcg_date = VALUES(tcg_date),
ocg_date = VALUES(ocg_date)
`, [
card.id,
card.name,
card.type,
card.frameType,
card.level || null,
card.race || null,
card.attribute || null,
card.linkval || null,
card.tcg_date || null,
card.ocg_date || null
]);
addedCards++;
name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type),
level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute),
link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date)
`, cardParams);
if (Array.isArray(card.card_sets)) {
for (const set of card.card_sets) {
if (!rarityCache[set.set_rarity]) {
await conn.execute(`
INSERT INTO rarities (rarity_name, rarity_code)
VALUES (?, ?)
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
`, [set.set_rarity, set.set_rarity_code]);
const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
rarityCache[set.set_rarity] = rows[0].id;
addedRarities++;
}
const rarity_id = rarityCache[set.set_rarity];
const set_id = setCache[set.set_name];
if (set_id) {
await conn.execute(`
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
`, [card.id, set_id, rarity_id, set.set_code]);
}
}
}
if (Array.isArray(card.card_images)) {
for (const img of card.card_images) {
if (imageValues.length) {
await conn.execute(`
INSERT INTO card_images (image_id, card_id, image_url)
VALUES (?, ?, ?)
VALUES ${imageValues.join(', ')}
ON DUPLICATE KEY UPDATE image_url = VALUES(image_url)
`, [img.id, card.id, img.image_url]);
addedImages++;
`, imageParams);
}
if (csrValues.length) {
await conn.execute(`
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
VALUES ${csrValues.join(', ')}
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
`, csrParams);
}
}));
await conn.commit();
addedCards += batch.length;
addedImages += imageValues.length;
addedSetRarities += csrValues.length;
} catch (err) {
await conn.rollback();
throw err;
} finally {
conn.release();
}
}
const durationSeconds = ((Date.now() - startTime) / 1000).toFixed(2);
return {
total_cards: totalCards,
cards_added: addedCards,
images_added: addedImages,
rarities_added: addedRarities,
duration_seconds: parseFloat(durationSeconds)
set_rarities_added: addedSetRarities,
duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)),
};
}
// Express handler for /cards
async function importCards(req, res) {
try {
const result = await importCardsInternal();
@@ -161,7 +165,6 @@ async function importCards(req, res) {
}
}
async function importFullDatabase(req, res) {
const startTime = Date.now();
try {