parallelized set Import
This commit is contained in:
@@ -1,14 +1,20 @@
|
||||
const { fetchAllSets } = require('../services/ygoproService');
|
||||
const { upsertSet } = require('../models/setModel');
|
||||
|
||||
const BATCH_SIZE = 10;
|
||||
|
||||
async function importSets(req, res) {
|
||||
try {
|
||||
const sets = await fetchAllSets();
|
||||
let added = 0;
|
||||
|
||||
for (const set of sets) {
|
||||
await upsertSet(set);
|
||||
added++;
|
||||
// Parallelized batch insert
|
||||
for (let i = 0; i < sets.length; i += BATCH_SIZE) {
|
||||
const batch = sets.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(batch.map(async (set) => {
|
||||
await upsertSet(set);
|
||||
added++;
|
||||
}));
|
||||
}
|
||||
|
||||
res.json({
|
||||
@@ -16,7 +22,7 @@ async function importSets(req, res) {
|
||||
total: sets.length
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
console.error('Error importing sets:', err);
|
||||
res.status(500).json({ error: 'Failed to import sets' });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user