Card Import Works
This commit is contained in:
@@ -1,7 +1,16 @@
|
||||
const { fetchAllSets } = require('../services/ygoproService');
|
||||
const { upsertSet } = require('../models/setModel');
|
||||
const {
|
||||
upsertCard
|
||||
} = require('../models/cardModel');
|
||||
const {
|
||||
upsertRarity,
|
||||
getRarityId
|
||||
} = require('../models/rarityModel');
|
||||
const { insertCardSetRarity } = require('../models/cardSetRarityModel');
|
||||
const { insertCardImage } = require('../models/cardImageModel');
|
||||
const { fetchAllCards } = require('../services/ygoproService');
|
||||
const db = require('../config/db');
|
||||
|
||||
const BATCH_SIZE = 10;
|
||||
const BATCH_SIZE = 50;
|
||||
|
||||
async function importSets(req, res) {
|
||||
try {
|
||||
@@ -27,4 +36,137 @@ async function importSets(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { importSets };
|
||||
async function importCards(req, res) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const cards = await fetchAllCards();
|
||||
const totalCards = cards.length;
|
||||
|
||||
let addedCards = 0, addedImages = 0, addedRarities = 0;
|
||||
|
||||
// Cache sets and rarities to reduce repeated DB lookups
|
||||
const rarityCache = {};
|
||||
const setCache = {};
|
||||
|
||||
// Preload sets into cache
|
||||
const [setsRows] = await db.execute('SELECT id, set_code FROM sets');
|
||||
for (const row of setsRows) {
|
||||
setCache[row.set_code] = row.id;
|
||||
}
|
||||
|
||||
// Preload rarities into cache
|
||||
const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
|
||||
for (const row of rarityRows) {
|
||||
rarityCache[row.rarity_name] = row.id;
|
||||
}
|
||||
|
||||
// Process cards in batches
|
||||
for (let i = 0; i < totalCards; i += BATCH_SIZE) {
|
||||
const batch = cards.slice(i, i + BATCH_SIZE);
|
||||
|
||||
// Wrap batch in a transaction
|
||||
const conn = await db.getConnection();
|
||||
try {
|
||||
await conn.beginTransaction();
|
||||
|
||||
await Promise.all(batch.map(async (card) => {
|
||||
// Upsert card
|
||||
await conn.execute(`
|
||||
INSERT INTO cards
|
||||
(id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
card_type = VALUES(card_type),
|
||||
frame_type = VALUES(frame_type),
|
||||
level = VALUES(level),
|
||||
race = VALUES(race),
|
||||
attribute = VALUES(attribute),
|
||||
link_val = VALUES(link_val),
|
||||
tcg_date = VALUES(tcg_date),
|
||||
ocg_date = VALUES(ocg_date)
|
||||
`, [
|
||||
card.id,
|
||||
card.name,
|
||||
card.type,
|
||||
card.frameType,
|
||||
card.level || null,
|
||||
card.race || null,
|
||||
card.attribute || null,
|
||||
card.linkval || null,
|
||||
card.tcg_date || null,
|
||||
card.ocg_date || null
|
||||
]);
|
||||
addedCards++;
|
||||
|
||||
// Process rarities and card_sets_rarity
|
||||
if (Array.isArray(card.card_sets)) {
|
||||
for (const set of card.card_sets) {
|
||||
// Upsert rarity if not cached
|
||||
if (!rarityCache[set.set_rarity]) {
|
||||
await conn.execute(`
|
||||
INSERT INTO rarities (rarity_name, rarity_code)
|
||||
VALUES (?, ?)
|
||||
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
|
||||
`, [set.set_rarity, set.set_rarity_code]);
|
||||
const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
|
||||
rarityCache[set.set_rarity] = rows[0].id;
|
||||
addedRarities++;
|
||||
}
|
||||
const rarity_id = rarityCache[set.set_rarity];
|
||||
|
||||
// Get set_id from cache
|
||||
const set_id = setCache[set.set_code];
|
||||
if (set_id) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
|
||||
`, [card.id, set_id, rarity_id, set.set_code]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process card images
|
||||
if (Array.isArray(card.card_images)) {
|
||||
for (const img of card.card_images) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_images (card_id, image_url)
|
||||
VALUES (?, ?)
|
||||
`, [card.id, img.image_url]);
|
||||
addedImages++;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
await conn.commit();
|
||||
} catch (err) {
|
||||
await conn.rollback();
|
||||
throw err;
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
|
||||
// Release batch memory
|
||||
batch.length = 0;
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
const durationSeconds = ((endTime - startTime) / 1000).toFixed(2);
|
||||
|
||||
res.json({
|
||||
total_cards: totalCards,
|
||||
cards_added: addedCards,
|
||||
images_added: addedImages,
|
||||
rarities_added: addedRarities,
|
||||
duration_seconds: parseFloat(durationSeconds)
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('Error importing cards:', err);
|
||||
res.status(500).json({ error: 'Failed to import cards' });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { importCards, importSets };
|
||||
@@ -0,0 +1,13 @@
|
||||
const db = require('../config/db');
|
||||
|
||||
async function insertCardImage(image) {
|
||||
const { card_id, image_url } = image;
|
||||
|
||||
const sql = `
|
||||
INSERT INTO card_images (card_id, image_url)
|
||||
VALUES (?, ?)
|
||||
`;
|
||||
await db.execute(sql, [card_id, image_url]);
|
||||
}
|
||||
|
||||
module.exports = { insertCardImage };
|
||||
@@ -0,0 +1,34 @@
|
||||
const db = require('../config/db');
|
||||
|
||||
async function upsertCard(card) {
|
||||
const {
|
||||
id,
|
||||
name,
|
||||
type,
|
||||
frameType,
|
||||
level,
|
||||
race,
|
||||
attribute,
|
||||
linkval,
|
||||
tcg_date,
|
||||
ocg_date
|
||||
} = card;
|
||||
|
||||
const sql = `
|
||||
INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
card_type = VALUES(card_type),
|
||||
frame_type = VALUES(frame_type),
|
||||
level = VALUES(level),
|
||||
race = VALUES(race),
|
||||
attribute = VALUES(attribute),
|
||||
link_val = VALUES(link_val),
|
||||
tcg_date = VALUES(tcg_date),
|
||||
ocg_date = VALUES(ocg_date)
|
||||
`;
|
||||
await db.execute(sql, [id, name, type, frameType, level, race, attribute, linkval, tcg_date, ocg_date]);
|
||||
}
|
||||
|
||||
module.exports = { upsertCard };
|
||||
@@ -0,0 +1,13 @@
|
||||
const db = require('../config/db');
|
||||
|
||||
async function insertCardSetRarity(entry) {
|
||||
const { card_id, set_id, rarity_id, card_set_code } = entry;
|
||||
const sql = `
|
||||
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
|
||||
`;
|
||||
await db.execute(sql, [card_id, set_id, rarity_id, card_set_code]);
|
||||
}
|
||||
|
||||
module.exports = { insertCardSetRarity };
|
||||
@@ -0,0 +1,22 @@
|
||||
const db = require('../config/db');
|
||||
|
||||
async function upsertRarity(rarity) {
|
||||
const { rarity_name, rarity_code } = rarity;
|
||||
|
||||
const sql = `
|
||||
INSERT INTO rarities (rarity_name, rarity_code)
|
||||
VALUES (?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
rarity_code = VALUES(rarity_code)
|
||||
`;
|
||||
const [result] = await db.execute(sql, [rarity_name, rarity_code]);
|
||||
return result.insertId; // returns id for junction table
|
||||
}
|
||||
|
||||
async function getRarityId(rarity_name) {
|
||||
const sql = `SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1`;
|
||||
const [rows] = await db.execute(sql, [rarity_name]);
|
||||
return rows.length ? rows[0].id : null;
|
||||
}
|
||||
|
||||
module.exports = { upsertRarity, getRarityId };
|
||||
@@ -1,7 +1,8 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { importSets } = require('../controllers/importController');
|
||||
const { importSets, importCards} = require('../controllers/importController');
|
||||
|
||||
router.post('/sets', importSets);
|
||||
router.post('/cards', importCards);
|
||||
|
||||
module.exports = router;
|
||||
@@ -19,4 +19,15 @@ async function fetchAllSets() {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchAllSets };
|
||||
async function fetchAllCards() {
|
||||
try {
|
||||
const response = await axios.get(`${API_BASE}/cardinfo.php?misc=yes`);
|
||||
return response.data.data; // array of card objects
|
||||
} catch (err) {
|
||||
console.error('Error fetching cards from YGOPRODeck:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
module.exports = { fetchAllCards, fetchAllSets };
|
||||
Reference in New Issue
Block a user