diff --git a/app/admin/page.tsx b/app/admin/page.tsx index 9135916..822d205 100644 --- a/app/admin/page.tsx +++ b/app/admin/page.tsx @@ -134,19 +134,56 @@ export default function AdminPage() { setCategorizationResults(null); try { - const res = await fetch('/api/categorize', { - method: 'POST', + let offset = 0; + let hasMore = true; + let allResults: any[] = []; + let totalUncategorized = 0; + let totalProcessed = 0; + + // Process in batches + while (hasMore) { + const res = await fetch('/api/categorize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ offset }) + }); + + if (!res.ok) { + const error = await res.json(); + alert(`Categorization failed: ${error.error || 'Unknown error'}`); + break; + } + + const data = await res.json(); + totalUncategorized = data.totalUncategorized; + totalProcessed = data.processed; + hasMore = data.hasMore; + offset = data.nextOffset || 0; + + // Accumulate results + allResults = [...allResults, ...data.results]; + + // Update UI with progress + setCategorizationResults({ + message: `Processing: ${totalProcessed} / ${totalUncategorized} songs...`, + totalProcessed: totalUncategorized, + totalCategorized: allResults.length, + results: allResults, + inProgress: hasMore + }); + } + + // Final update + setCategorizationResults({ + message: `Completed! Processed ${totalUncategorized} songs, categorized ${allResults.length}`, + totalProcessed: totalUncategorized, + totalCategorized: allResults.length, + results: allResults, + inProgress: false }); - if (res.ok) { - const data = await res.json(); - setCategorizationResults(data); - fetchSongs(); // Refresh song list - fetchGenres(); // Refresh genre counts - } else { - const error = await res.json(); - alert(`Categorization failed: ${error.error || 'Unknown error'}`); - } + fetchSongs(); // Refresh song list + fetchGenres(); // Refresh genre counts } catch (error) { alert('Failed to categorize songs'); console.error(error); diff --git a/app/api/categorize/route.ts b/app/api/categorize/route.ts index f9c2bd7..da04137 100644 --- a/app/api/categorize/route.ts +++ b/app/api/categorize/route.ts @@ -6,6 +6,7 @@ const prisma = new PrismaClient(); const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY; const OPENROUTER_MODEL = 'anthropic/claude-3.5-haiku'; +const BATCH_SIZE = 20; // Process 20 songs at a time to avoid timeouts interface CategorizeResult { songId: number; @@ -14,7 +15,7 @@ interface CategorizeResult { assignedGenres: string[]; } -export async function POST() { +export async function POST(request: Request) { try { if (!OPENROUTER_API_KEY) { return Response.json( @@ -23,7 +24,30 @@ export async function POST() { ); } + // Get offset from request body (for batch processing) + const body = await request.json().catch(() => ({})); + const offset = body.offset || 0; + // Fetch all songs without genres + const totalUncategorized = await prisma.song.count({ + where: { + genres: { + none: {} + } + } + }); + + if (totalUncategorized === 0) { + return Response.json({ + message: 'No uncategorized songs found', + results: [], + hasMore: false, + totalUncategorized: 0, + processed: 0 + }); + } + + // Fetch batch of songs const uncategorizedSongs = await prisma.song.findMany({ where: { genres: { @@ -32,16 +56,11 @@ export async function POST() { }, include: { genres: true - } + }, + take: BATCH_SIZE, + skip: offset }); - if (uncategorizedSongs.length === 0) { - return Response.json({ - message: 'No uncategorized songs found', - results: [] - }); - } - // Fetch all available genres const allGenres = await prisma.genre.findMany({ orderBy: { name: 'asc' } @@ -56,7 +75,7 @@ export async function POST() { const results: CategorizeResult[] = []; - // Process each song + // Process each song in this batch for (const song of uncategorizedSongs) { try { const genreNames = allGenres.map(g => g.name); @@ -147,10 +166,15 @@ Your response:`; } } + const newOffset = offset + BATCH_SIZE; + const hasMore = newOffset < totalUncategorized; + return Response.json({ - message: `Processed ${uncategorizedSongs.length} songs, categorized ${results.length}`, - totalProcessed: uncategorizedSongs.length, - totalCategorized: results.length, + message: `Processed ${uncategorizedSongs.length} songs in this batch, categorized ${results.length}`, + totalUncategorized, + processed: Math.min(newOffset, totalUncategorized), + hasMore, + nextOffset: hasMore ? newOffset : null, results });