parallelized set Import

This commit is contained in:
2026-03-13 18:33:31 +01:00
parent 64166cce0d
commit 9f378f9e53
2 changed files with 10 additions and 21 deletions
+10 -4
View File
@@ -1,14 +1,20 @@
const { fetchAllSets } = require('../services/ygoproService');
const { upsertSet } = require('../models/setModel');
const BATCH_SIZE = 10;
async function importSets(req, res) {
try {
const sets = await fetchAllSets();
let added = 0;
for (const set of sets) {
await upsertSet(set);
added++;
// Parallelized batch insert
for (let i = 0; i < sets.length; i += BATCH_SIZE) {
const batch = sets.slice(i, i + BATCH_SIZE);
await Promise.all(batch.map(async (set) => {
await upsertSet(set);
added++;
}));
}
res.json({
@@ -16,7 +22,7 @@ async function importSets(req, res) {
total: sets.length
});
} catch (err) {
console.error(err);
console.error('Error importing sets:', err);
res.status(500).json({ error: 'Failed to import sets' });
}
}