HASH 64848b109a84
DATE 2025-12-21
SUBJECT ookstats: remove skip logic
FILES 8 CHANGED
HASH 64848b109a84
DATE 2025-12-21
SUBJECT ookstats: remove skip logic
FILES 8 CHANGED
┌─ nix/pkgs/ookstats/src/cmd/build.go ───────────────────────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/cmd/build.go b/nix/pkgs/ookstats/src/cmd/build. │
│ go │
│ index c532b2c..23c31ba 100644 │
│ --- a/nix/pkgs/ookstats/src/cmd/build.go │
│ +++ b/nix/pkgs/ookstats/src/cmd/build.go │
│ @@ -44,6 +44,7 @@ var buildCmd = &cobra.Command{ │
│ skipProfiles, _ := cmd.Flags().GetBool("skip-profiles") │
│ periodsCSV, _ := cmd.Flags().GetString("periods") │
│ concurrency, _ := cmd.Flags().GetInt("concurrency") │
│ + workers, _ := cmd.Flags().GetInt("workers") │
│ │
│ // optional verbose logging propagated to API client │
│ verbose, _ := cmd.InheritedFlags().GetBool("verbose") │
│ @@ -187,7 +188,7 @@ var buildCmd = &cobra.Command{ │
│ │
│ // 9) Generate static API │
│ log.Info("generating static API") │
│ - if err := generateAllAPI(db, normalizedOut, pageSize, shardSize, regionsCSV); e │
│ rr != nil { │
│ + if err := generateAllAPI(db, normalizedOut, pageSize, shardSize, workers, regio │
│ nsCSV); err != nil { │
│ return err │
│ } │
│ │
│ @@ -326,7 +327,7 @@ func fetchProfilesOnce(db *sql.DB, client *blizzard.Client) er │
│ ror { │
│ } │
│ │
│ // generateAllAPI mirrors the behavior of `generate api` │
│ -func generateAllAPI(db *sql.DB, outParent string, pageSize, shardSize int, region │
│ sCSV string) error { │
│ +func generateAllAPI(db *sql.DB, outParent string, pageSize, shardSize, workers in │
│ t, regionsCSV string) error { │
│ base := filepath.Join(outParent, "api") │
│ if err := os.MkdirAll(base, 0o755); err != nil { │
│ return fmt.Errorf("mkdir: %w", err) │
│ @@ -347,10 +348,10 @@ func generateAllAPI(db *sql.DB, outParent string, pageSize, │
│ shardSize int, regio │
│ } │
│ } │
│ } │
│ - if err := generator.GenerateLeaderboards(db, filepath.Join(base, "leaderboard"), │
│ pageSize, regions); err != nil { │
│ + if err := generator.GenerateLeaderboards(db, filepath.Join(base, "leaderboard"), │
│ pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ - if err := generator.GeneratePlayerLeaderboards(db, filepath.Join(base, "leaderbo │
│ ard"), pageSize, regions); err != nil { │
│ + if err := generator.GeneratePlayerLeaderboards(db, filepath.Join(base, "leaderbo │
│ ard"), pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ │
│ @@ -509,4 +510,5 @@ func init() { │
│ buildCmd.Flags().Bool("skip-profiles", false, "Skip fetching player detailed pro │
│ files") │
│ buildCmd.Flags().String("periods", "", "Period specification: comma-separated li │
│ st or ranges (e.g., '1020-1036' or '1020,1025,1030-1036'). Default: fetch all peri │
│ ods from API") │
│ buildCmd.Flags().Int("concurrency", 20, "Max concurrent API requests") │
│ + buildCmd.Flags().Int("workers", 10, "Number of parallel workers for leaderboard │
│ generation") │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...kgs/ookstats/src/cmd/build.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/cmd/build │
│ .go b/nix/pkgs/ookstats/src/cmd/build.go │
│ index c532b2c..23c31ba 100644 │
│ --- a/nix/pkgs/ookstats/src/cmd/build.go │
│ +++ b/nix/pkgs/ookstats/src/cmd/build.go │
│ @@ -44,6 +44,7 @@ var buildCmd = &cobra.Comm │
│ and{ │
│ skipProfiles, _ := cmd.Flags().GetBool("s │
│ kip-profiles") │
│ periodsCSV, _ := cmd.Flags().GetString("p │
│ eriods") │
│ concurrency, _ := cmd.Flags().GetInt("con │
│ currency") │
│ + workers, _ := cmd.Flags().GetInt("workers │
│ ") │
│ │
│ // optional verbose logging propagated to │
│ API client │
│ verbose, _ := cmd.InheritedFlags().GetBoo │
│ l("verbose") │
│ @@ -187,7 +188,7 @@ var buildCmd = &cobra.Co │
│ mmand{ │
│ │
│ // 9) Generate static API │
│ log.Info("generating static API") │
│ - if err := generateAllAPI(db, normalizedOu │
│ t, pageSize, shardSize, regionsCSV); err != │
│ nil { │
│ + if err := generateAllAPI(db, normalizedOu │
│ t, pageSize, shardSize, workers, regionsCSV) │
│ ; err != nil { │
│ return err │
│ } │
│ │
│ @@ -326,7 +327,7 @@ func fetchProfilesOnce(d │
│ b *sql.DB, client *blizzard.Client) error { │
│ } │
│ │
│ // generateAllAPI mirrors the behavior of ` │
│ generate api` │
│ -func generateAllAPI(db *sql.DB, outParent s │
│ tring, pageSize, shardSize int, regionsCSV s │
│ tring) error { │
│ +func generateAllAPI(db *sql.DB, outParent s │
│ tring, pageSize, shardSize, workers int, reg │
│ ionsCSV string) error { │
│ base := filepath.Join(outParent, "api") │
│ if err := os.MkdirAll(base, 0o755); err != │
│ nil { │
│ return fmt.Errorf("mkdir: %w", err) │
│ @@ -347,10 +348,10 @@ func generateAllAPI(db │
│ *sql.DB, outParent string, pageSize, shardS │
│ ize int, regio │
│ } │
│ } │
│ } │
│ - if err := generator.GenerateLeaderboards(d │
│ b, filepath.Join(base, "leaderboard"), pageS │
│ ize, regions); err != nil { │
│ + if err := generator.GenerateLeaderboards(d │
│ b, filepath.Join(base, "leaderboard"), pageS │
│ ize, regions, workers); err != nil { │
│ return err │
│ } │
│ - if err := generator.GeneratePlayerLeaderbo │
│ ards(db, filepath.Join(base, "leaderboard"), │
│ pageSize, regions); err != nil { │
│ + if err := generator.GeneratePlayerLeaderbo │
│ ards(db, filepath.Join(base, "leaderboard"), │
│ pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ │
│ @@ -509,4 +510,5 @@ func init() { │
│ buildCmd.Flags().Bool("skip-profiles", fal │
│ se, "Skip fetching player detailed profiles" │
│ ) │
│ buildCmd.Flags().String("periods", "", "Pe │
│ riod specification: comma-separated list or │
│ ranges (e.g., '1020-1036' or '1020,1025,1030 │
│ -1036'). Default: fetch all periods from API │
│ ") │
│ buildCmd.Flags().Int("concurrency", 20, "M │
│ ax concurrent API requests") │
│ + buildCmd.Flags().Int("workers", 10, "Numbe │
│ r of parallel workers for leaderboard genera │
│ tion") │
│ } │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/cmd/generate.go ────────────────────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/cmd/generate.go b/nix/pkgs/ookstats/src/cmd/gen │
│ erate.go │
│ index 4b56e3f..179359f 100644 │
│ --- a/nix/pkgs/ookstats/src/cmd/generate.go │
│ +++ b/nix/pkgs/ookstats/src/cmd/generate.go │
│ @@ -32,6 +32,7 @@ var generateAPICmd = &cobra.Command{ │
│ pageSize, _ := cmd.Flags().GetInt("page-size") │
│ shardSize, _ := cmd.Flags().GetInt("shard-size") │
│ regionsCSV, _ := cmd.Flags().GetString("regions") │
│ + workers, _ := cmd.Flags().GetInt("workers") │
│ │
│ if strings.TrimSpace(outDir) == "" { │
│ return errors.New("--out is required") │
│ @@ -64,10 +65,10 @@ var generateAPICmd = &cobra.Command{ │
│ } │
│ } │
│ } │
│ - if err := generator.GenerateLeaderboards(db, filepath.Join(base, "leaderboard" │
│ ), pageSize, regions); err != nil { │
│ + if err := generator.GenerateLeaderboards(db, filepath.Join(base, "leaderboard" │
│ ), pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ - if err := generator.GeneratePlayerLeaderboards(db, filepath.Join(base, "leader │
│ board"), pageSize, regions); err != nil { │
│ + if err := generator.GeneratePlayerLeaderboards(db, filepath.Join(base, "leader │
│ board"), pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -100,4 +101,5 @@ func init() { │
│ generateAPICmd.Flags().Int("page-size", 25, "Leaderboard page size") │
│ generateAPICmd.Flags().Int("shard-size", 5000, "Search index shard size") │
│ generateAPICmd.Flags().String("regions", "us,eu,kr,tw", "Regions to include for │
│ regional leaderboards") │
│ + generateAPICmd.Flags().Int("workers", 10, "Number of parallel workers for leader │
│ board generation") │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ .../ookstats/src/cmd/generate.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/cmd/gener │
│ ate.go b/nix/pkgs/ookstats/src/cmd/generate. │
│ go │
│ index 4b56e3f..179359f 100644 │
│ --- a/nix/pkgs/ookstats/src/cmd/generate.go │
│ +++ b/nix/pkgs/ookstats/src/cmd/generate.go │
│ @@ -32,6 +32,7 @@ var generateAPICmd = &cobr │
│ a.Command{ │
│ pageSize, _ := cmd.Flags().GetInt("page-s │
│ ize") │
│ shardSize, _ := cmd.Flags().GetInt("shard │
│ -size") │
│ regionsCSV, _ := cmd.Flags().GetString("r │
│ egions") │
│ + workers, _ := cmd.Flags().GetInt("workers │
│ ") │
│ │
│ if strings.TrimSpace(outDir) == "" { │
│ return errors.New("--out is required") │
│ @@ -64,10 +65,10 @@ var generateAPICmd = &co │
│ bra.Command{ │
│ } │
│ } │
│ } │
│ - if err := generator.GenerateLeaderboards │
│ (db, filepath.Join(base, "leaderboard"), pag │
│ eSize, regions); err != nil { │
│ + if err := generator.GenerateLeaderboards │
│ (db, filepath.Join(base, "leaderboard"), pag │
│ eSize, regions, workers); err != nil { │
│ return err │
│ } │
│ - if err := generator.GeneratePlayerLeader │
│ boards(db, filepath.Join(base, "leaderboard" │
│ ), pageSize, regions); err != nil { │
│ + if err := generator.GeneratePlayerLeader │
│ boards(db, filepath.Join(base, "leaderboard" │
│ ), pageSize, regions, workers); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -100,4 +101,5 @@ func init() { │
│ generateAPICmd.Flags().Int("page-size", 25 │
│ , "Leaderboard page size") │
│ generateAPICmd.Flags().Int("shard-size", 5 │
│ 000, "Search index shard size") │
│ generateAPICmd.Flags().String("regions", " │
│ us,eu,kr,tw", "Regions to include for region │
│ al leaderboards") │
│ + generateAPICmd.Flags().Int("workers", 10, │
│ "Number of parallel workers for leaderboard │
│ generation") │
│ } │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/database/operations.go ────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/database/operations.go b/nix/pkgs/ooks │
│ tats/src/internal/database/operations.go │
│ index 51f547e..dfada77 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/database/operations.go │
│ +++ b/nix/pkgs/ookstats/src/internal/database/operations.go │
│ @@ -11,7 +11,6 @@ import ( │
│ "sort" │
│ "strconv" │
│ "strings" │
│ - "sync" │
│ "time" │
│ ) │
│ │
│ @@ -443,72 +442,9 @@ func (ds *DatabaseService) BatchProcessFetchResults(ctx conte │
│ xt.Context, results │
│ processedCount := 0 │
│ errorCount := 0 │
│ │
│ - // channel to collect results for batching │
│ - batchChan := make(chan blizzard.FetchResult, 50) // Buffer for batching │
│ - │
│ - // batch processor goroutine │
│ - var wg sync.WaitGroup │
│ - wg.Add(1) │
│ - │
│ - go func() { │
│ - defer wg.Done() │
│ - fmt.Printf("[INFO] Batch processor goroutine started\n") │
│ - │
│ - batch := make([]blizzard.FetchResult, 0, 10) // Process in batches of 10 │
│ - batchNumber := 0 │
│ - │
│ - for { │
│ - select { │
│ - case result, ok := <-batchChan: │
│ - if !ok { │
│ - fmt.Printf("[INFO] Batch channel closed, processing final batch...\n") │
│ - // channel closed, process remaining batch │
│ - if len(batch) > 0 { │
│ - batchNumber++ │
│ - fmt.Printf("[INFO] Processing final batch %d with %d items...\n", batchNumb │
│ er, len(batch)) │
│ - runs, players, err := ds.processBatch(batch) │
│ - if err != nil { │
│ - fmt.Printf("[ERROR] Final batch %d failed: %v\n", batchNumber, err) │
│ - } else { │
│ - totalRuns += runs │
│ - totalPlayers += players │
│ - if runs > 0 || players > 0 { │
│ - fmt.Printf("[OK] Final batch %d: +%d runs, +%d players (total: %d runs, % │
│ d players)\n", │
│ - batchNumber, runs, players, totalRuns, totalPlayers) │
│ - } │
│ - } │
│ - } │
│ - fmt.Printf("[INFO] Batch processor goroutine ending\n") │
│ - return │
│ - } │
│ - │
│ - batch = append(batch, result) │
│ - │
│ - // process batch when full │
│ - if len(batch) >= 10 { │
│ - batchNumber++ │
│ - runs, players, err := ds.processBatch(batch) │
│ - if err != nil { │
│ - fmt.Printf("[ERROR] Batch %d failed: %v\n", batchNumber, err) │
│ - } else { │
│ - totalRuns += runs │
│ - totalPlayers += players │
│ - if runs > 0 || players > 0 { │
│ - fmt.Printf("[INFO] Batch %d: +%d runs, +%d players (total: %d runs, %d pla │
│ yers)\n", │
│ - batchNumber, runs, players, totalRuns, totalPlayers) │
│ - } │
│ - } │
│ - batch = batch[:0] // Clear batch │
│ - } │
│ - │
│ - case <-ctx.Done(): │
│ - fmt.Printf("[WARN] Batch processor context cancelled\n") │
│ - return │
│ - } │
│ - } │
│ - }() │
│ + batch := make([]blizzard.FetchResult, 0, 10) │
│ + batchNumber := 0 │
│ │
│ - // forward results to batch processor with progress tracking │
│ fmt.Printf("[INFO] Starting to process API results...\n") │
│ for result := range results { │
│ processedCount++ │
│ @@ -525,27 +461,52 @@ func (ds *DatabaseService) BatchProcessFetchResults(ctx cont │
│ ext.Context, results │
│ continue │
│ } │
│ │
│ + batch = append(batch, result) │
│ + │
│ + // process batch when full │
│ + if len(batch) >= 10 { │
│ + batchNumber++ │
│ + runs, players, err := ds.processBatch(batch) │
│ + if err != nil { │
│ + fmt.Printf("[ERROR] Batch %d failed: %v\n", batchNumber, err) │
│ + } else { │
│ + totalRuns += runs │
│ + totalPlayers += players │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf("[INFO] Batch %d: +%d runs, +%d players (total: %d runs, %d playe │
│ rs)\n", │
│ + batchNumber, runs, players, totalRuns, totalPlayers) │
│ + } │
│ + } │
│ + batch = batch[:0] │
│ + } │
│ + │
│ // show periodic progress │
│ if processedCount%10 == 0 { │
│ - fmt.Printf("[INFO] Progress: %d requests processed, %d errors, queued for batc │
│ h processing...\n", processedCount, errorCount) │
│ + fmt.Printf("[INFO] Progress: %d requests processed, %d errors\n", processedCou │
│ nt, errorCount) │
│ } │
│ │
│ select { │
│ - case batchChan <- result: │
│ case <-ctx.Done(): │
│ fmt.Printf("[WARN] Context cancelled, stopping processing\n") │
│ - close(batchChan) │
│ - wg.Wait() │
│ - fmt.Printf("\n[INFO] Final stats: %d requests processed, %d errors, %d runs, % │
│ d players\n", │
│ - processedCount, errorCount, totalRuns, totalPlayers) │
│ return totalRuns, totalPlayers, ctx.Err() │
│ + default: │
│ } │
│ } │
│ │
│ - fmt.Printf("[INFO] Finished processing all API results, closing batch channel... │
│ \n") │
│ - │
│ - close(batchChan) │
│ - wg.Wait() │
│ + // process remaining batch │
│ + if len(batch) > 0 { │
│ + batchNumber++ │
│ + runs, players, err := ds.processBatch(batch) │
│ + if err != nil { │
│ + fmt.Printf("[ERROR] Final batch %d failed: %v\n", batchNumber, err) │
│ + } else { │
│ + totalRuns += runs │
│ + totalPlayers += players │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf("[INFO] Final batch %d: +%d runs, +%d players\n", batchNumber, run │
│ s, players) │
│ + } │
│ + } │
│ + } │
│ │
│ fmt.Printf("\n[INFO] Final stats: %d requests processed, %d errors, %d runs, %d │
│ players\n", │
│ processedCount, errorCount, totalRuns, totalPlayers) │
│ @@ -559,18 +520,12 @@ func (ds *DatabaseService) processBatch(batch []blizzard.Fet │
│ chResult) (int, int, │
│ return 0, 0, nil │
│ } │
│ │
│ - // Pre-scan batch to decide which items actually need writes (local diff -> inse │
│ rt-new) │
│ + // collect items with valid leaderboard data │
│ type batchItem struct { │
│ - idx int │
│ - r blizzard.RealmInfo │
│ - d blizzard.DungeonInfo │
│ - board *blizzard.LeaderboardResponse │
│ - realmID int │
│ - dungeonID int │
│ - existingMax int64 │
│ - marker int64 │
│ - maxCT int64 │
│ - needsWrite bool │
│ + idx int │
│ + r blizzard.RealmInfo │
│ + d blizzard.DungeonInfo │
│ + board *blizzard.LeaderboardResponse │
│ } │
│ │
│ items := make([]batchItem, 0, len(batch)) │
│ @@ -578,64 +533,19 @@ func (ds *DatabaseService) processBatch(batch []blizzard.Fet │
│ chResult) (int, int, │
│ if res.Leaderboard == nil || len(res.Leaderboard.LeadingGroups) == 0 { │
│ continue │
│ } │
│ - // resolve IDs (read-only) │
│ - realmID, err := ds.GetRealmIDByRegionAndSlug(res.RealmInfo.Region, res.RealmInf │
│ o.Slug) │
│ - if err != nil { │
│ - return 0, 0, fmt.Errorf("failed to resolve realm id: %w", err) │
│ - } │
│ - dungeonID, err := ds.GetDungeonID(res.Dungeon.Slug) │
│ - if err != nil { │
│ - return 0, 0, fmt.Errorf("failed to resolve dungeon id: %w", err) │
│ - } │
│ - // compute payload maxCT │
│ - maxCT := int64(0) │
│ - for _, run := range res.Leaderboard.LeadingGroups { │
│ - if run.CompletedTimestamp > maxCT { │
│ - maxCT = run.CompletedTimestamp │
│ - } │
│ - } │
│ - // read DB high-water and marker (read-only; outside TX) │
│ - var existingMax sql.NullInt64 │
│ - _ = ds.db.QueryRow(`SELECT MAX(completed_timestamp) FROM challenge_runs WHERE r │
│ ealm_id = ? AND dungeon_id = ?`, realmID, dungeonID).Scan(&existingMax) │
│ - var marker int64 │
│ - _ = ds.db.QueryRow(`SELECT last_completed_ts FROM api_fetch_markers WHERE realm │
│ _slug = ? AND dungeon_id = ? AND period_id = ?`, res.RealmInfo.Slug, dungeonID, re │
│ s.Leaderboard.Period).Scan(&marker) │
│ - minCT := marker │
│ - if existingMax.Valid && existingMax.Int64 > minCT { │
│ - minCT = existingMax.Int64 │
│ - } │
│ - needsWrite := maxCT > 0 && maxCT > minCT │
│ - │
│ - if !needsWrite { │
│ - fmt.Printf(" [SKIP] %s/%s: up-to-date (minCT=%d, maxCT=%d)\n", res.RealmInf │
│ o.Name, res.Dungeon.Name, minCT, maxCT) │
│ - } │
│ items = append(items, batchItem{ │
│ - idx: i + 1, │
│ - r: res.RealmInfo, │
│ - d: res.Dungeon, │
│ - board: res.Leaderboard, │
│ - realmID: realmID, │
│ - dungeonID: dungeonID, │
│ - existingMax: existingMax.Int64, │
│ - marker: marker, │
│ - maxCT: maxCT, │
│ - needsWrite: needsWrite, │
│ + idx: i + 1, │
│ + r: res.RealmInfo, │
│ + d: res.Dungeon, │
│ + board: res.Leaderboard, │
│ }) │
│ } │
│ │
│ - // filter to items needing writes │
│ - toWrite := make([]batchItem, 0, len(items)) │
│ - for _, it := range items { │
│ - if it.needsWrite { │
│ - toWrite = append(toWrite, it) │
│ - } │
│ - } │
│ - │
│ - if len(toWrite) == 0 { │
│ - // nothing to write; avoid starting a transaction │
│ + if len(items) == 0 { │
│ return 0, 0, nil │
│ } │
│ │
│ - // begin transaction for write set only │
│ + // begin transaction │
│ startBatch := time.Now() │
│ tx, err := ds.db.Begin() │
│ if err != nil { │
│ @@ -646,7 +556,7 @@ func (ds *DatabaseService) processBatch(batch []blizzard.Fetch │
│ Result) (int, int, │
│ totalRuns := 0 │
│ totalPlayers := 0 │
│ │
│ - for _, it := range toWrite { │
│ + for _, it := range items { │
│ itemStart := time.Now() │
│ runs, players, err := ds.insertLeaderboardDataTx(tx, it.board, it.r, it.d) │
│ if err != nil { │
│ @@ -654,8 +564,10 @@ func (ds *DatabaseService) processBatch(batch []blizzard.Fetc │
│ hResult) (int, int, │
│ } │
│ totalRuns += runs │
│ totalPlayers += players │
│ - fmt.Printf(" - Batch item %d: %s/%s -> +%d runs, +%d players in %dms\n", │
│ - it.idx, it.r.Name, it.d.Name, runs, players, time.Since(itemStart).Millisecond │
│ s()) │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf(" - Batch item %d: %s/%s -> +%d runs, +%d players in %dms\n", │
│ + it.idx, it.r.Name, it.d.Name, runs, players, time.Since(itemStart).Millisecon │
│ ds()) │
│ + } │
│ } │
│ │
│ if err := tx.Commit(); err != nil { │
│ @@ -699,38 +611,18 @@ func (ds *DatabaseService) insertLeaderboardDataTx(tx *sql.T │
│ x, leaderboard *bliz │
│ return 0, 0, fmt.Errorf("failed to get dungeon ID: %w", err) │
│ } │
│ │
│ - // Compute newest completed_timestamp in this leaderboard │
│ + // Compute newest completed_timestamp in this leaderboard (for marker update) │
│ maxCT := int64(0) │
│ for _, run := range leaderboard.LeadingGroups { │
│ if run.CompletedTimestamp > maxCT { │
│ maxCT = run.CompletedTimestamp │
│ } │
│ } │
│ - // Quick skip using existing DB max completed_timestamp for this realm+dungeon │
│ - var existingMax sql.NullInt64 │
│ - if err := tx.QueryRow(`SELECT MAX(completed_timestamp) FROM challenge_runs WHERE │
│ realm_id = ? AND dungeon_id = ?`, realmID, dungeonID).Scan(&existingMax); err == │
│ nil { │
│ - if existingMax.Valid && maxCT > 0 && existingMax.Int64 >= maxCT { │
│ - fmt.Printf(" [SKIP] %s/%s: existingMax=%d >= maxCT=%d (DB)\n", realmInfo.Na │
│ me, dungeon.Name, existingMax.Int64, maxCT) │
│ - return 0, 0, nil │
│ - } │
│ - } │
│ - var marker int64 │
│ - if err := tx.QueryRow(`SELECT last_completed_ts FROM api_fetch_markers WHERE rea │
│ lm_slug = ? AND dungeon_id = ? AND period_id = ?`, realmInfo.Slug, dungeonID, lead │
│ erboard.Period).Scan(&marker); err != nil && err != sql.ErrNoRows { │
│ - return 0, 0, fmt.Errorf("failed to read fetch marker: %w", err) │
│ - } │
│ - if marker >= maxCT && maxCT > 0 { │
│ - fmt.Printf(" [SKIP] %s/%s: up-to-date (marker=%d, maxCT=%d)\n", realmInfo.Na │
│ me, dungeon.Name, marker, maxCT) │
│ - return 0, 0, nil │
│ - } │
│ │
│ runsInserted := 0 │
│ playersInserted := 0 │
│ │
│ for _, run := range leaderboard.LeadingGroups { │
│ - // fast-skip duplicates using DB max and marker │
│ - if (existingMax.Valid && run.CompletedTimestamp <= existingMax.Int64) || (marke │
│ r > 0 && run.CompletedTimestamp <= marker) { │
│ - continue │
│ - } │
│ // extract player IDs for team signature │
│ var playerIDs []int │
│ for _, member := range run.Members { │
│ @@ -748,7 +640,7 @@ func (ds *DatabaseService) insertLeaderboardDataTx(tx *sql.Tx, │
│ leaderboard *bliz │
│ // determine season for this run │
│ var runSeasonID *int │
│ if run.CompletedTimestamp > 0 { │
│ - sid, err := ds.DetermineSeasonForRun(realmInfo.Region, run.CompletedTimestamp) │
│ + sid, err := ds.determineSeasonForRunTx(tx, realmInfo.Region, run.CompletedTime │
│ stamp) │
│ if err != nil { │
│ return 0, 0, fmt.Errorf("failed to determine season for run: %w", err) │
│ } │
│ @@ -876,8 +768,8 @@ func (ds *DatabaseService) insertLeaderboardDataTx(tx *sql.Tx, │
│ leaderboard *bliz │
│ } │
│ } │
│ │
│ - // update marker inside the transaction if we progressed │
│ - if maxCT > marker && maxCT > 0 { │
│ + // update marker inside the transaction │
│ + if maxCT > 0 { │
│ if _, err := tx.Exec(`INSERT INTO api_fetch_markers (realm_slug, dungeon_id, pe │
│ riod_id, last_completed_ts) │
│ VALUES (?, ?, ?, ?) │
│ ON CONFLICT(realm_slug, dungeon_id, period_id) │
│ @@ -1577,6 +1469,25 @@ func (ds *DatabaseService) DetermineSeasonForRun(region str │
│ ing, completedTimesta │
│ return seasonNumber, err │
│ } │
│ │
│ +// determineSeasonForRunTx is a transaction-aware version of DetermineSeasonForRu │
│ n │
│ +func (ds *DatabaseService) determineSeasonForRunTx(tx *sql.Tx, region string, com │
│ pletedTimestamp int64) (int, error) { │
│ + query := ` │
│ + SELECT season_number │
│ + FROM seasons │
│ + WHERE region = ? │
│ + AND start_timestamp <= ? │
│ + AND (end_timestamp IS NULL OR end_timestamp > ?) │
│ + ORDER BY start_timestamp DESC │
│ + LIMIT 1 │
│ + ` │
│ + var seasonNumber int │
│ + err := tx.QueryRow(query, region, completedTimestamp, completedTimestamp).Scan(& │
│ seasonNumber) │
│ + if err == sql.ErrNoRows { │
│ + return 0, nil │
│ + } │
│ + return seasonNumber, err │
│ +} │
│ + │
│ // AssignRunsToSeasons assigns season_id to all challenge_runs based on completed │
│ _timestamp │
│ // and realm region. This is used to backfill existing data. │
│ func (ds *DatabaseService) AssignRunsToSeasons() error { │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...ternal/database/operations.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ database/operations.go b/nix/pkgs/ookstats/s │
│ rc/internal/database/operations.go │
│ index 51f547e..dfada77 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/databas │
│ e/operations.go │
│ +++ b/nix/pkgs/ookstats/src/internal/databas │
│ e/operations.go │
│ @@ -11,7 +11,6 @@ import ( │
│ "sort" │
│ "strconv" │
│ "strings" │
│ - "sync" │
│ "time" │
│ ) │
│ │
│ @@ -443,72 +442,9 @@ func (ds *DatabaseServi │
│ ce) BatchProcessFetchResults(ctx context.Con │
│ text, results │
│ processedCount := 0 │
│ errorCount := 0 │
│ │
│ - // channel to collect results for batching │
│ - batchChan := make(chan blizzard.FetchResul │
│ t, 50) // Buffer for batching │
│ - │
│ - // batch processor goroutine │
│ - var wg sync.WaitGroup │
│ - wg.Add(1) │
│ - │
│ - go func() { │
│ - defer wg.Done() │
│ - fmt.Printf("[INFO] Batch processor gorout │
│ ine started\n") │
│ - │
│ - batch := make([]blizzard.FetchResult, 0, │
│ 10) // Process in batches of 10 │
│ - batchNumber := 0 │
│ - │
│ - for { │
│ - select { │
│ - case result, ok := <-batchChan: │
│ - if !ok { │
│ - fmt.Printf("[INFO] Batch channel close │
│ d, processing final batch...\n") │
│ - // channel closed, process remaining b │
│ atch │
│ - if len(batch) > 0 { │
│ - batchNumber++ │
│ - fmt.Printf("[INFO] Processing final b │
│ atch %d with %d items...\n", batchNumber, le │
│ n(batch)) │
│ - runs, players, err := ds.processBatch │
│ (batch) │
│ - if err != nil { │
│ - fmt.Printf("[ERROR] Final batch %d f │
│ ailed: %v\n", batchNumber, err) │
│ - } else { │
│ - totalRuns += runs │
│ - totalPlayers += players │
│ - if runs > 0 || players > 0 { │
│ - fmt.Printf("[OK] Final batch %d: +% │
│ d runs, +%d players (total: %d runs, %d play │
│ ers)\n", │
│ - batchNumber, runs, players, totalR │
│ uns, totalPlayers) │
│ - } │
│ - } │
│ - } │
│ - fmt.Printf("[INFO] Batch processor gor │
│ outine ending\n") │
│ - return │
│ - } │
│ - │
│ - batch = append(batch, result) │
│ - │
│ - // process batch when full │
│ - if len(batch) >= 10 { │
│ - batchNumber++ │
│ - runs, players, err := ds.processBatch( │
│ batch) │
│ - if err != nil { │
│ - fmt.Printf("[ERROR] Batch %d failed: │
│ %v\n", batchNumber, err) │
│ - } else { │
│ - totalRuns += runs │
│ - totalPlayers += players │
│ - if runs > 0 || players > 0 { │
│ - fmt.Printf("[INFO] Batch %d: +%d run │
│ s, +%d players (total: %d runs, %d players)\ │
│ n", │
│ - batchNumber, runs, players, totalRu │
│ ns, totalPlayers) │
│ - } │
│ - } │
│ - batch = batch[:0] // Clear batch │
│ - } │
│ - │
│ - case <-ctx.Done(): │
│ - fmt.Printf("[WARN] Batch processor cont │
│ ext cancelled\n") │
│ - return │
│ - } │
│ - } │
│ - }() │
│ + batch := make([]blizzard.FetchResult, 0, 1 │
│ 0) │
│ + batchNumber := 0 │
│ │
│ - // forward results to batch processor with │
│ progress tracking │
│ fmt.Printf("[INFO] Starting to process API │
│ results...\n") │
│ for result := range results { │
│ processedCount++ │
│ @@ -525,27 +461,52 @@ func (ds *DatabaseServ │
│ ice) BatchProcessFetchResults(ctx context.Co │
│ ntext, results │
│ continue │
│ } │
│ │
│ + batch = append(batch, result) │
│ + │
│ + // process batch when full │
│ + if len(batch) >= 10 { │
│ + batchNumber++ │
│ + runs, players, err := ds.processBatch(ba │
│ tch) │
│ + if err != nil { │
│ + fmt.Printf("[ERROR] Batch %d failed: %v │
│ \n", batchNumber, err) │
│ + } else { │
│ + totalRuns += runs │
│ + totalPlayers += players │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf("[INFO] Batch %d: +%d runs, │
│ +%d players (total: %d runs, %d players)\n" │
│ , │
│ + batchNumber, runs, players, totalRuns │
│ , totalPlayers) │
│ + } │
│ + } │
│ + batch = batch[:0] │
│ + } │
│ + │
│ // show periodic progress │
│ if processedCount%10 == 0 { │
│ - fmt.Printf("[INFO] Progress: %d requests │
│ processed, %d errors, queued for batch proc │
│ essing...\n", processedCount, errorCount) │
│ + fmt.Printf("[INFO] Progress: %d requests │
│ processed, %d errors\n", processedCount, er │
│ rorCount) │
│ } │
│ │
│ select { │
│ - case batchChan <- result: │
│ case <-ctx.Done(): │
│ fmt.Printf("[WARN] Context cancelled, st │
│ opping processing\n") │
│ - close(batchChan) │
│ - wg.Wait() │
│ - fmt.Printf("\n[INFO] Final stats: %d req │
│ uests processed, %d errors, %d runs, %d play │
│ ers\n", │
│ - processedCount, errorCount, totalRuns, │
│ totalPlayers) │
│ return totalRuns, totalPlayers, ctx.Err( │
│ ) │
│ + default: │
│ } │
│ } │
│ │
│ - fmt.Printf("[INFO] Finished processing all │
│ API results, closing batch channel...\n") │
│ - │
│ - close(batchChan) │
│ - wg.Wait() │
│ + // process remaining batch │
│ + if len(batch) > 0 { │
│ + batchNumber++ │
│ + runs, players, err := ds.processBatch(bat │
│ ch) │
│ + if err != nil { │
│ + fmt.Printf("[ERROR] Final batch %d faile │
│ d: %v\n", batchNumber, err) │
│ + } else { │
│ + totalRuns += runs │
│ + totalPlayers += players │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf("[INFO] Final batch %d: +%d │
│ runs, +%d players\n", batchNumber, runs, pla │
│ yers) │
│ + } │
│ + } │
│ + } │
│ │
│ fmt.Printf("\n[INFO] Final stats: %d reque │
│ sts processed, %d errors, %d runs, %d player │
│ s\n", │
│ processedCount, errorCount, totalRuns, to │
│ talPlayers) │
│ @@ -559,18 +520,12 @@ func (ds *DatabaseServ │
│ ice) processBatch(batch []blizzard.FetchResu │
│ lt) (int, int, │
│ return 0, 0, nil │
│ } │
│ │
│ - // Pre-scan batch to decide which items ac │
│ tually need writes (local diff -> insert-new │
│ ) │
│ + // collect items with valid leaderboard da │
│ ta │
│ type batchItem struct { │
│ - idx int │
│ - r blizzard.RealmInfo │
│ - d blizzard.DungeonInfo │
│ - board *blizzard.LeaderboardResponse │
│ - realmID int │
│ - dungeonID int │
│ - existingMax int64 │
│ - marker int64 │
│ - maxCT int64 │
│ - needsWrite bool │
│ + idx int │
│ + r blizzard.RealmInfo │
│ + d blizzard.DungeonInfo │
│ + board *blizzard.LeaderboardResponse │
│ } │
│ │
│ items := make([]batchItem, 0, len(batch)) │
│ @@ -578,64 +533,19 @@ func (ds *DatabaseServ │
│ ice) processBatch(batch []blizzard.FetchResu │
│ lt) (int, int, │
│ if res.Leaderboard == nil || len(res.Lead │
│ erboard.LeadingGroups) == 0 { │
│ continue │
│ } │
│ - // resolve IDs (read-only) │
│ - realmID, err := ds.GetRealmIDByRegionAndS │
│ lug(res.RealmInfo.Region, res.RealmInfo.Slug │
│ ) │
│ - if err != nil { │
│ - return 0, 0, fmt.Errorf("failed to resol │
│ ve realm id: %w", err) │
│ - } │
│ - dungeonID, err := ds.GetDungeonID(res.Dun │
│ geon.Slug) │
│ - if err != nil { │
│ - return 0, 0, fmt.Errorf("failed to resol │
│ ve dungeon id: %w", err) │
│ - } │
│ - // compute payload maxCT │
│ - maxCT := int64(0) │
│ - for _, run := range res.Leaderboard.Leadi │
│ ngGroups { │
│ - if run.CompletedTimestamp > maxCT { │
│ - maxCT = run.CompletedTimestamp │
│ - } │
│ - } │
│ - // read DB high-water and marker (read-on │
│ ly; outside TX) │
│ - var existingMax sql.NullInt64 │
│ - _ = ds.db.QueryRow(`SELECT MAX(completed_ │
│ timestamp) FROM challenge_runs WHERE realm_i │
│ d = ? AND dungeon_id = ?`, realmID, dungeonI │
│ D).Scan(&existingMax) │
│ - var marker int64 │
│ - _ = ds.db.QueryRow(`SELECT last_completed │
│ _ts FROM api_fetch_markers WHERE realm_slug │
│ = ? AND dungeon_id = ? AND period_id = ?`, r │
│ es.RealmInfo.Slug, dungeonID, res.Leaderboar │
│ d.Period).Scan(&marker) │
│ - minCT := marker │
│ - if existingMax.Valid && existingMax.Int64 │
│ > minCT { │
│ - minCT = existingMax.Int64 │
│ - } │
│ - needsWrite := maxCT > 0 && maxCT > minCT │
│ - │
│ - if !needsWrite { │
│ - fmt.Printf(" [SKIP] %s/%s: up-to-date │
│ (minCT=%d, maxCT=%d)\n", res.RealmInfo.Name │
│ , res.Dungeon.Name, minCT, maxCT) │
│ - } │
│ items = append(items, batchItem{ │
│ - idx: i + 1, │
│ - r: res.RealmInfo, │
│ - d: res.Dungeon, │
│ - board: res.Leaderboard, │
│ - realmID: realmID, │
│ - dungeonID: dungeonID, │
│ - existingMax: existingMax.Int64, │
│ - marker: marker, │
│ - maxCT: maxCT, │
│ - needsWrite: needsWrite, │
│ + idx: i + 1, │
│ + r: res.RealmInfo, │
│ + d: res.Dungeon, │
│ + board: res.Leaderboard, │
│ }) │
│ } │
│ │
│ - // filter to items needing writes │
│ - toWrite := make([]batchItem, 0, len(items) │
│ ) │
│ - for _, it := range items { │
│ - if it.needsWrite { │
│ - toWrite = append(toWrite, it) │
│ - } │
│ - } │
│ - │
│ - if len(toWrite) == 0 { │
│ - // nothing to write; avoid starting a tra │
│ nsaction │
│ + if len(items) == 0 { │
│ return 0, 0, nil │
│ } │
│ │
│ - // begin transaction for write set only │
│ + // begin transaction │
│ startBatch := time.Now() │
│ tx, err := ds.db.Begin() │
│ if err != nil { │
│ @@ -646,7 +556,7 @@ func (ds *DatabaseServic │
│ e) processBatch(batch []blizzard.FetchResult │
│ ) (int, int, │
│ totalRuns := 0 │
│ totalPlayers := 0 │
│ │
│ - for _, it := range toWrite { │
│ + for _, it := range items { │
│ itemStart := time.Now() │
│ runs, players, err := ds.insertLeaderboar │
│ dDataTx(tx, it.board, it.r, it.d) │
│ if err != nil { │
│ @@ -654,8 +564,10 @@ func (ds *DatabaseServi │
│ ce) processBatch(batch []blizzard.FetchResul │
│ t) (int, int, │
│ } │
│ totalRuns += runs │
│ totalPlayers += players │
│ - fmt.Printf(" - Batch item %d: %s/%s -> │
│ +%d runs, +%d players in %dms\n", │
│ - it.idx, it.r.Name, it.d.Name, runs, play │
│ ers, time.Since(itemStart).Milliseconds()) │
│ + if runs > 0 || players > 0 { │
│ + fmt.Printf(" - Batch item %d: %s/%s - │
│ > +%d runs, +%d players in %dms\n", │
│ + it.idx, it.r.Name, it.d.Name, runs, pla │
│ yers, time.Since(itemStart).Milliseconds()) │
│ + } │
│ } │
│ │
│ if err := tx.Commit(); err != nil { │
│ @@ -699,38 +611,18 @@ func (ds *DatabaseServ │
│ ice) insertLeaderboardDataTx(tx *sql.Tx, lea │
│ derboard *bliz │
│ return 0, 0, fmt.Errorf("failed to get du │
│ ngeon ID: %w", err) │
│ } │
│ │
│ - // Compute newest completed_timestamp in t │
│ his leaderboard │
│ + // Compute newest completed_timestamp in t │
│ his leaderboard (for marker update) │
│ maxCT := int64(0) │
│ for _, run := range leaderboard.LeadingGro │
│ ups { │
│ if run.CompletedTimestamp > maxCT { │
│ maxCT = run.CompletedTimestamp │
│ } │
│ } │
│ - // Quick skip using existing DB max comple │
│ ted_timestamp for this realm+dungeon │
│ - var existingMax sql.NullInt64 │
│ - if err := tx.QueryRow(`SELECT MAX(complete │
│ d_timestamp) FROM challenge_runs WHERE realm │
│ _id = ? AND dungeon_id = ?`, realmID, dungeo │
│ nID).Scan(&existingMax); err == nil { │
│ - if existingMax.Valid && maxCT > 0 && exis │
│ tingMax.Int64 >= maxCT { │
│ - fmt.Printf(" [SKIP] %s/%s: existingMa │
│ x=%d >= maxCT=%d (DB)\n", realmInfo.Name, du │
│ ngeon.Name, existingMax.Int64, maxCT) │
│ - return 0, 0, nil │
│ - } │
│ - } │
│ - var marker int64 │
│ - if err := tx.QueryRow(`SELECT last_complet │
│ ed_ts FROM api_fetch_markers WHERE realm_slu │
│ g = ? AND dungeon_id = ? AND period_id = ?`, │
│ realmInfo.Slug, dungeonID, leaderboard.Peri │
│ od).Scan(&marker); err != nil && err != sql. │
│ ErrNoRows { │
│ - return 0, 0, fmt.Errorf("failed to read f │
│ etch marker: %w", err) │
│ - } │
│ - if marker >= maxCT && maxCT > 0 { │
│ - fmt.Printf(" [SKIP] %s/%s: up-to-date │
│ (marker=%d, maxCT=%d)\n", realmInfo.Name, du │
│ ngeon.Name, marker, maxCT) │
│ - return 0, 0, nil │
│ - } │
│ │
│ runsInserted := 0 │
│ playersInserted := 0 │
│ │
│ for _, run := range leaderboard.LeadingGro │
│ ups { │
│ - // fast-skip duplicates using DB max and │
│ marker │
│ - if (existingMax.Valid && run.CompletedTim │
│ estamp <= existingMax.Int64) || (marker > 0 │
│ && run.CompletedTimestamp <= marker) { │
│ - continue │
│ - } │
│ // extract player IDs for team signature │
│ var playerIDs []int │
│ for _, member := range run.Members { │
│ @@ -748,7 +640,7 @@ func (ds *DatabaseServic │
│ e) insertLeaderboardDataTx(tx *sql.Tx, leade │
│ rboard *bliz │
│ // determine season for this run │
│ var runSeasonID *int │
│ if run.CompletedTimestamp > 0 { │
│ - sid, err := ds.DetermineSeasonForRun(rea │
│ lmInfo.Region, run.CompletedTimestamp) │
│ + sid, err := ds.determineSeasonForRunTx(t │
│ x, realmInfo.Region, run.CompletedTimestamp) │
│ if err != nil { │
│ return 0, 0, fmt.Errorf("failed to dete │
│ rmine season for run: %w", err) │
│ } │
│ @@ -876,8 +768,8 @@ func (ds *DatabaseServic │
│ e) insertLeaderboardDataTx(tx *sql.Tx, leade │
│ rboard *bliz │
│ } │
│ } │
│ │
│ - // update marker inside the transaction if │
│ we progressed │
│ - if maxCT > marker && maxCT > 0 { │
│ + // update marker inside the transaction │
│ + if maxCT > 0 { │
│ if _, err := tx.Exec(`INSERT INTO api_fet │
│ ch_markers (realm_slug, dungeon_id, period_i │
│ d, last_completed_ts) │
│ VALUES (?, ? │
│ , ?, ?) │
│ ON CONFLICT( │
│ realm_slug, dungeon_id, period_id) │
│ @@ -1577,6 +1469,25 @@ func (ds *DatabaseSer │
│ vice) DetermineSeasonForRun(region string, c │
│ ompletedTimesta │
│ return seasonNumber, err │
│ } │
│ │
│ +// determineSeasonForRunTx is a transaction │
│ -aware version of DetermineSeasonForRun │
│ +func (ds *DatabaseService) determineSeasonF │
│ orRunTx(tx *sql.Tx, region string, completed │
│ Timestamp int64) (int, error) { │
│ + query := ` │
│ + SELECT season_number │
│ + FROM seasons │
│ + WHERE region = ? │
│ + AND start_timestamp <= ? │
│ + AND (end_timestamp IS NULL OR end_times │
│ tamp > ?) │
│ + ORDER BY start_timestamp DESC │
│ + LIMIT 1 │
│ + ` │
│ + var seasonNumber int │
│ + err := tx.QueryRow(query, region, complete │
│ dTimestamp, completedTimestamp).Scan(&season │
│ Number) │
│ + if err == sql.ErrNoRows { │
│ + return 0, nil │
│ + } │
│ + return seasonNumber, err │
│ +} │
│ + │
│ // AssignRunsToSeasons assigns season_id to │
│ all challenge_runs based on completed_times │
│ tamp │
│ // and realm region. This is used to backfi │
│ ll existing data. │
│ func (ds *DatabaseService) AssignRunsToSeas │
│ ons() error { │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/generator/leaderboards.go ─────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/generator/leaderboards.go b/nix/pkgs/o │
│ okstats/src/internal/generator/leaderboards.go │
│ index 70dd0a6..d5069f7 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generator/leaderboards.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generator/leaderboards.go │
│ @@ -7,7 +7,8 @@ import ( │
│ "ookstats/internal/writer" │
│ "os" │
│ "path/filepath" │
│ - "strings" │
│ + "sync" │
│ + "sync/atomic" │
│ ) │
│ │
│ // LeaderboardPageJSON represents a paginated leaderboard response │
│ @@ -56,11 +57,26 @@ type dungeonInfo struct { │
│ Slug, Name string │
│ } │
│ │
│ +// leaderboardJob represents a single leaderboard generation task │
│ +type leaderboardJob struct { │
│ + seasonID int │
│ + seasonName string │
│ + dungeon dungeonInfo │
│ + region string // empty for global │
│ + realmSlug string // empty for global/regional │
│ + out string │
│ + pageSize int │
│ +} │
│ + │
│ // GenerateLeaderboards generates leaderboard JSON files for all scopes (global, │
│ regional, realm) per season │
│ -func GenerateLeaderboards(db *sql.DB, out string, pageSize int, regions []string) │
│ error { │
│ +// Uses a worker pool for parallel generation │
│ +func GenerateLeaderboards(db *sql.DB, out string, pageSize int, regions []string, │
│ workers int) error { │
│ if pageSize <= 0 { │
│ pageSize = 25 │
│ } │
│ + if workers <= 0 { │
│ + workers = 10 │
│ + } │
│ if err := os.MkdirAll(out, 0o755); err != nil { │
│ return err │
│ } │
│ @@ -87,47 +103,128 @@ func GenerateLeaderboards(db *sql.DB, out string, pageSize i │
│ nt, regions []string │
│ regions = []string{"us", "eu", "kr", "tw"} │
│ } │
│ │
│ - // Generate leaderboards for each season │
│ + // Pre-load all realm slugs per region │
│ + realmSlugs := make(map[string][]string) │
│ + for _, reg := range regions { │
│ + slugs, err := loadRealmSlugs(db, reg) │
│ + if err != nil { │
│ + return err │
│ + } │
│ + realmSlugs[reg] = slugs │
│ + } │
│ + │
│ + // Count total jobs for progress reporting │
│ + var totalJobs int │
│ + for range seasons { │
│ + totalJobs += len(dungeons) // global │
│ + for _, reg := range regions { │
│ + totalJobs += len(dungeons) // regional │
│ + totalJobs += len(realmSlugs[reg]) * len(dungeons) // realm │
│ + } │
│ + } │
│ + │
│ + fmt.Printf("Generating leaderboards with %d workers (%d total jobs)...\n", worke │
│ rs, totalJobs) │
│ + │
│ + // Create job channel and error channel │
│ + jobs := make(chan leaderboardJob, 100) │
│ + var firstErr atomic.Value │
│ + var completed atomic.Int64 │
│ + var wg sync.WaitGroup │
│ + │
│ + // Start workers │
│ + for i := 0; i < workers; i++ { │
│ + wg.Add(1) │
│ + go func() { │
│ + defer wg.Done() │
│ + for job := range jobs { │
│ + // Skip if we already have an error │
│ + if firstErr.Load() != nil { │
│ + continue │
│ + } │
│ + │
│ + var err error │
│ + if job.realmSlug != "" { │
│ + err = generateRealmLeaderboard(db, job.out, job.region, job.realmSlug, job.d │
│ ungeon, job.seasonID, job.pageSize) │
│ + } else if job.region != "" { │
│ + err = generateRegionalLeaderboard(db, job.out, job.region, job.dungeon, job. │
│ seasonID, job.pageSize) │
│ + } else { │
│ + err = generateGlobalLeaderboard(db, job.out, job.dungeon, job.seasonID, job. │
│ pageSize) │
│ + } │
│ + │
│ + if err != nil { │
│ + firstErr.CompareAndSwap(nil, err) │
│ + continue │
│ + } │
│ + │
│ + c := completed.Add(1) │
│ + if c%100 == 0 { │
│ + fmt.Printf(" ... %d/%d leaderboards generated\n", c, totalJobs) │
│ + } │
│ + } │
│ + }() │
│ + } │
│ + │
│ + // Queue all jobs │
│ for _, season := range seasons { │
│ - fmt.Printf("\n=== Season %d (%s) ===\n", season.ID, season.Name) │
│ + fmt.Printf("\n=== Queuing Season %d (%s) ===\n", season.ID, season.Name) │
│ seasonOut := filepath.Join(out, "season", fmt.Sprintf("%d", season.ID)) │
│ │
│ - // Process globals for this season │
│ - fmt.Println("Generating global leaderboards...") │
│ + // Global leaderboards │
│ for _, d := range dungeons { │
│ - if err := generateGlobalLeaderboard(db, seasonOut, d, season.ID, pageSize); er │
│ r != nil { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: "", │
│ + realmSlug: "", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ │
│ - // Regions for this season │
│ + // Regional leaderboards │
│ for _, reg := range regions { │
│ - fmt.Printf("Generating %s leaderboards...\n", strings.ToUpper(reg)) │
│ for _, d := range dungeons { │
│ - if err := generateRegionalLeaderboard(db, seasonOut, reg, d, season.ID, pageS │
│ ize); err != nil { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: reg, │
│ + realmSlug: "", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ } │
│ │
│ - // Realm leaderboards for each region+realm for this season │
│ + // Realm leaderboards │
│ for _, reg := range regions { │
│ - slugs, err := loadRealmSlugs(db, reg) │
│ - if err != nil { │
│ - return err │
│ - } │
│ - for _, rslug := range slugs { │
│ - fmt.Printf("Generating realm %s/%s leaderboards...\n", reg, rslug) │
│ + for _, rslug := range realmSlugs[reg] { │
│ for _, d := range dungeons { │
│ - if err := generateRealmLeaderboard(db, seasonOut, reg, rslug, d, season.ID, │
│ pageSize); err != nil { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: reg, │
│ + realmSlug: rslug, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ } │
│ } │
│ } │
│ + close(jobs) │
│ + │
│ + // Wait for all workers to complete │
│ + wg.Wait() │
│ + │
│ + // Check for errors │
│ + if err := firstErr.Load(); err != nil { │
│ + return err.(error) │
│ + } │
│ │
│ - fmt.Println("\n[OK] Season-scoped leaderboards generated") │
│ + fmt.Printf("\n[OK] Generated %d leaderboards\n", completed.Load()) │
│ return nil │
│ } │
│ │
│ @@ -225,7 +322,7 @@ func generateGlobalLeaderboard(db *sql.DB, out string, d dunge │
│ onInfo, seasonID, │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name, "", total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -262,7 +359,7 @@ func generateRegionalLeaderboard(db *sql.DB, out, region strin │
│ g, d dungeonInfo, │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name, "", total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -304,7 +401,7 @@ func generateRealmLeaderboard(db *sql.DB, out, region, realmSl │
│ ug string, d dunge │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name, realmName, total, pages, p, pageSize │
│ ) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...nal/generator/leaderboards.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ generator/leaderboards.go b/nix/pkgs/ookstat │
│ s/src/internal/generator/leaderboards.go │
│ index 70dd0a6..d5069f7 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generat │
│ or/leaderboards.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generat │
│ or/leaderboards.go │
│ @@ -7,7 +7,8 @@ import ( │
│ "ookstats/internal/writer" │
│ "os" │
│ "path/filepath" │
│ - "strings" │
│ + "sync" │
│ + "sync/atomic" │
│ ) │
│ │
│ // LeaderboardPageJSON represents a paginat │
│ ed leaderboard response │
│ @@ -56,11 +57,26 @@ type dungeonInfo struct │
│ { │
│ Slug, Name string │
│ } │
│ │
│ +// leaderboardJob represents a single leade │
│ rboard generation task │
│ +type leaderboardJob struct { │
│ + seasonID int │
│ + seasonName string │
│ + dungeon dungeonInfo │
│ + region string // empty for global │
│ + realmSlug string // empty for global/regi │
│ onal │
│ + out string │
│ + pageSize int │
│ +} │
│ + │
│ // GenerateLeaderboards generates leaderboa │
│ rd JSON files for all scopes (global, region │
│ al, realm) per season │
│ -func GenerateLeaderboards(db *sql.DB, out s │
│ tring, pageSize int, regions []string) error │
│ { │
│ +// Uses a worker pool for parallel generati │
│ on │
│ +func GenerateLeaderboards(db *sql.DB, out s │
│ tring, pageSize int, regions []string, worke │
│ rs int) error { │
│ if pageSize <= 0 { │
│ pageSize = 25 │
│ } │
│ + if workers <= 0 { │
│ + workers = 10 │
│ + } │
│ if err := os.MkdirAll(out, 0o755); err != │
│ nil { │
│ return err │
│ } │
│ @@ -87,47 +103,128 @@ func GenerateLeaderboa │
│ rds(db *sql.DB, out string, pageSize int, re │
│ gions []string │
│ regions = []string{"us", "eu", "kr", "tw" │
│ } │
│ } │
│ │
│ - // Generate leaderboards for each season │
│ + // Pre-load all realm slugs per region │
│ + realmSlugs := make(map[string][]string) │
│ + for _, reg := range regions { │
│ + slugs, err := loadRealmSlugs(db, reg) │
│ + if err != nil { │
│ + return err │
│ + } │
│ + realmSlugs[reg] = slugs │
│ + } │
│ + │
│ + // Count total jobs for progress reporting │
│ + var totalJobs int │
│ + for range seasons { │
│ + totalJobs += len(dungeons) // global │
│ + for _, reg := range regions { │
│ + totalJobs += len(dungeons) │
│ // regional │
│ + totalJobs += len(realmSlugs[reg]) * len( │
│ dungeons) // realm │
│ + } │
│ + } │
│ + │
│ + fmt.Printf("Generating leaderboards with % │
│ d workers (%d total jobs)...\n", workers, to │
│ talJobs) │
│ + │
│ + // Create job channel and error channel │
│ + jobs := make(chan leaderboardJob, 100) │
│ + var firstErr atomic.Value │
│ + var completed atomic.Int64 │
│ + var wg sync.WaitGroup │
│ + │
│ + // Start workers │
│ + for i := 0; i < workers; i++ { │
│ + wg.Add(1) │
│ + go func() { │
│ + defer wg.Done() │
│ + for job := range jobs { │
│ + // Skip if we already have an error │
│ + if firstErr.Load() != nil { │
│ + continue │
│ + } │
│ + │
│ + var err error │
│ + if job.realmSlug != "" { │
│ + err = generateRealmLeaderboard(db, job │
│ .out, job.region, job.realmSlug, job.dungeon │
│ , job.seasonID, job.pageSize) │
│ + } else if job.region != "" { │
│ + err = generateRegionalLeaderboard(db, │
│ job.out, job.region, job.dungeon, job.season │
│ ID, job.pageSize) │
│ + } else { │
│ + err = generateGlobalLeaderboard(db, jo │
│ b.out, job.dungeon, job.seasonID, job.pageSi │
│ ze) │
│ + } │
│ + │
│ + if err != nil { │
│ + firstErr.CompareAndSwap(nil, err) │
│ + continue │
│ + } │
│ + │
│ + c := completed.Add(1) │
│ + if c%100 == 0 { │
│ + fmt.Printf(" ... %d/%d leaderboards g │
│ enerated\n", c, totalJobs) │
│ + } │
│ + } │
│ + }() │
│ + } │
│ + │
│ + // Queue all jobs │
│ for _, season := range seasons { │
│ - fmt.Printf("\n=== Season %d (%s) ===\n", │
│ season.ID, season.Name) │
│ + fmt.Printf("\n=== Queuing Season %d (%s) │
│ ===\n", season.ID, season.Name) │
│ seasonOut := filepath.Join(out, "season", │
│ fmt.Sprintf("%d", season.ID)) │
│ │
│ - // Process globals for this season │
│ - fmt.Println("Generating global leaderboar │
│ ds...") │
│ + // Global leaderboards │
│ for _, d := range dungeons { │
│ - if err := generateGlobalLeaderboard(db, │
│ seasonOut, d, season.ID, pageSize); err != n │
│ il { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: "", │
│ + realmSlug: "", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ │
│ - // Regions for this season │
│ + // Regional leaderboards │
│ for _, reg := range regions { │
│ - fmt.Printf("Generating %s leaderboards.. │
│ .\n", strings.ToUpper(reg)) │
│ for _, d := range dungeons { │
│ - if err := generateRegionalLeaderboard(d │
│ b, seasonOut, reg, d, season.ID, pageSize); │
│ err != nil { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: reg, │
│ + realmSlug: "", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ } │
│ │
│ - // Realm leaderboards for each region+rea │
│ lm for this season │
│ + // Realm leaderboards │
│ for _, reg := range regions { │
│ - slugs, err := loadRealmSlugs(db, reg) │
│ - if err != nil { │
│ - return err │
│ - } │
│ - for _, rslug := range slugs { │
│ - fmt.Printf("Generating realm %s/%s lead │
│ erboards...\n", reg, rslug) │
│ + for _, rslug := range realmSlugs[reg] { │
│ for _, d := range dungeons { │
│ - if err := generateRealmLeaderboard(db, │
│ seasonOut, reg, rslug, d, season.ID, pageSi │
│ ze); err != nil { │
│ - return err │
│ + jobs <- leaderboardJob{ │
│ + seasonID: season.ID, │
│ + seasonName: season.Name, │
│ + dungeon: d, │
│ + region: reg, │
│ + realmSlug: rslug, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ } │
│ } │
│ } │
│ + close(jobs) │
│ + │
│ + // Wait for all workers to complete │
│ + wg.Wait() │
│ + │
│ + // Check for errors │
│ + if err := firstErr.Load(); err != nil { │
│ + return err.(error) │
│ + } │
│ │
│ - fmt.Println("\n[OK] Season-scoped leaderbo │
│ ards generated") │
│ + fmt.Printf("\n[OK] Generated %d leaderboar │
│ ds\n", completed.Load()) │
│ return nil │
│ } │
│ │
│ @@ -225,7 +322,7 @@ func generateGlobalLeade │
│ rboard(db *sql.DB, out string, d dungeonInfo │
│ , seasonID, │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name │
│ , "", total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -262,7 +359,7 @@ func generateRegionalLea │
│ derboard(db *sql.DB, out, region string, d d │
│ ungeonInfo, │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name │
│ , "", total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -304,7 +401,7 @@ func generateRealmLeader │
│ board(db *sql.DB, out, region, realmSlug str │
│ ing, d dunge │
│ } │
│ │
│ page := buildLeaderboardPage(rows, d.Name │
│ , realmName, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/generator/player_leaderboards.go ──────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/generator/player_leaderboards.go b/nix │
│ /pkgs/ookstats/src/internal/generator/player_leaderboards.go │
│ index 6d7811c..ae6f183 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generator/player_leaderboards.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generator/player_leaderboards.go │
│ @@ -8,14 +8,33 @@ import ( │
│ "os" │
│ "path/filepath" │
│ "strings" │
│ + "sync" │
│ + "sync/atomic" │
│ "time" │
│ ) │
│ │
│ +// playerLeaderboardJob represents a single player leaderboard generation task │
│ +type playerLeaderboardJob struct { │
│ + seasonID int │
│ + seasonName string │
│ + scope string // "global", "regional", "realm", "class" │
│ + region string // for regional/realm/class scopes │
│ + realmSlug string // for realm scope │
│ + classKey string // for class scope │
│ + out string │
│ + pageSize int │
│ + regions []string // for class scope │
│ +} │
│ + │
│ // GeneratePlayerLeaderboards generates player ranking JSON files for all scopes │
│ per season │
│ -func GeneratePlayerLeaderboards(db *sql.DB, out string, pageSize int, regions []s │
│ tring) error { │
│ +// Uses a worker pool for parallel generation │
│ +func GeneratePlayerLeaderboards(db *sql.DB, out string, pageSize int, regions []s │
│ tring, workers int) error { │
│ if pageSize <= 0 { │
│ pageSize = 25 │
│ } │
│ + if workers <= 0 { │
│ + workers = 10 │
│ + } │
│ │
│ // Default regions │
│ if len(regions) == 0 { │
│ @@ -33,39 +52,146 @@ func GeneratePlayerLeaderboards(db *sql.DB, out string, pageS │
│ ize int, regions [] │
│ return nil │
│ } │
│ │
│ - // Generate leaderboards for each season │
│ + // Pre-load realm slugs per region (only parent realms) │
│ + realmSlugs := make(map[string][]string) │
│ + for _, reg := range regions { │
│ + rrows, err := db.Query(` │
│ + SELECT slug │
│ + FROM realms │
│ + WHERE region = ? AND (parent_realm_slug IS NULL OR parent_realm_slug = '') │
│ + ORDER BY slug │
│ + `, reg) │
│ + if err != nil { │
│ + return fmt.Errorf("load realm slugs: %w", err) │
│ + } │
│ + var slugs []string │
│ + for rrows.Next() { │
│ + var s string │
│ + if err := rrows.Scan(&s); err != nil { │
│ + rrows.Close() │
│ + return err │
│ + } │
│ + slugs = append(slugs, s) │
│ + } │
│ + rrows.Close() │
│ + realmSlugs[reg] = slugs │
│ + } │
│ + │
│ + classKeys := []string{"death_knight", "druid", "hunter", "mage", "monk", "paladi │
│ n", "priest", "rogue", "shaman", "warlock", "warrior"} │
│ + │
│ + // Count total jobs │
│ + var totalJobs int │
│ + for range seasons { │
│ + totalJobs += 1 // global │
│ + totalJobs += len(regions) // regional │
│ + for _, reg := range regions { │
│ + totalJobs += len(realmSlugs[reg]) // realm │
│ + } │
│ + totalJobs += len(classKeys) // class (each handles all scopes internally) │
│ + } │
│ + │
│ + fmt.Printf("Generating player leaderboards with %d workers (%d total jobs)...\n" │
│ , workers, totalJobs) │
│ + │
│ + // Create job channel │
│ + jobs := make(chan playerLeaderboardJob, 100) │
│ + var firstErr atomic.Value │
│ + var completed atomic.Int64 │
│ + var wg sync.WaitGroup │
│ + │
│ + // Start workers │
│ + for i := 0; i < workers; i++ { │
│ + wg.Add(1) │
│ + go func() { │
│ + defer wg.Done() │
│ + for job := range jobs { │
│ + if firstErr.Load() != nil { │
│ + continue │
│ + } │
│ + │
│ + var err error │
│ + switch job.scope { │
│ + case "global": │
│ + err = generateGlobalPlayerLeaderboard(db, job.out, job.pageSize, job.seasonI │
│ D) │
│ + case "regional": │
│ + err = generateRegionalPlayerLeaderboard(db, job.out, job.region, job.pageSiz │
│ e, job.seasonID) │
│ + case "realm": │
│ + err = generateSingleRealmPlayerLeaderboard(db, job.out, job.region, job.real │
│ mSlug, job.pageSize, job.seasonID) │
│ + case "class": │
│ + err = generateClassPlayerLeaderboards(db, job.out, job.classKey, job.pageSiz │
│ e, job.regions, job.seasonID) │
│ + } │
│ + │
│ + if err != nil { │
│ + firstErr.CompareAndSwap(nil, err) │
│ + continue │
│ + } │
│ + │
│ + c := completed.Add(1) │
│ + if c%50 == 0 { │
│ + fmt.Printf(" ... %d/%d player leaderboards generated\n", c, totalJobs) │
│ + } │
│ + } │
│ + }() │
│ + } │
│ + │
│ + // Queue all jobs │
│ for _, season := range seasons { │
│ - fmt.Printf("\n=== Generating Player Leaderboards for Season %d (%s) ===\n", sea │
│ son.ID, season.Name) │
│ + fmt.Printf("\n=== Queuing Player Leaderboards for Season %d (%s) ===\n", season │
│ .ID, season.Name) │
│ seasonOut := filepath.Join(out, "season", fmt.Sprintf("%d", season.ID)) │
│ │
│ - // Generate global and regional scopes for this season │
│ - if err := generateGlobalPlayerLeaderboard(db, seasonOut, pageSize, season.ID); │
│ err != nil { │
│ - return err │
│ + // Global │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "global", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ │
│ + // Regional │
│ for _, reg := range regions { │
│ - if err := generateRegionalPlayerLeaderboard(db, seasonOut, reg, pageSize, seas │
│ on.ID); err != nil { │
│ - return err │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "regional", │
│ + region: reg, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ │
│ - // Generate realm scopes for this season │
│ + // Realm │
│ for _, reg := range regions { │
│ - if err := generateRealmPlayerLeaderboards(db, seasonOut, reg, pageSize, season │
│ .ID); err != nil { │
│ - return err │
│ + for _, rslug := range realmSlugs[reg] { │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "realm", │
│ + region: reg, │
│ + realmSlug: rslug, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ + } │
│ } │
│ } │
│ │
│ - // Generate class-filtered variants for this season │
│ - classKeys := []string{"death_knight", "druid", "hunter", "mage", "monk", "palad │
│ in", "priest", "rogue", "shaman", "warlock", "warrior"} │
│ + // Class (each handles global/regional/realm internally) │
│ for _, cls := range classKeys { │
│ - if err := generateClassPlayerLeaderboards(db, seasonOut, cls, pageSize, region │
│ s, season.ID); err != nil { │
│ - return err │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "class", │
│ + classKey: cls, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ + regions: regions, │
│ } │
│ } │
│ } │
│ + close(jobs) │
│ + │
│ + wg.Wait() │
│ + │
│ + if err := firstErr.Load(); err != nil { │
│ + return err.(error) │
│ + } │
│ │
│ - fmt.Println("[OK] Player leaderboards generated for all seasons") │
│ + fmt.Printf("\n[OK] Generated %d player leaderboards\n", completed.Load()) │
│ return nil │
│ } │
│ │
│ @@ -111,7 +237,7 @@ func generateGlobalPlayerLeaderboard(db *sql.DB, out string, p │
│ ageSize int, seaso │
│ } │
│ │
│ page := buildPlayerLeaderboardPage(list, "Global Player Rankings", total, pages │
│ , p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -163,88 +289,65 @@ func generateRegionalPlayerLeaderboard(db *sql.DB, out, regi │
│ on string, pageSize │
│ │
│ title := strings.ToUpper(region) + " Player Rankings" │
│ page := buildPlayerLeaderboardPage(list, title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
│ return nil │
│ } │
│ │
│ -// generateRealmPlayerLeaderboards generates realm-specific player rankings for a │
│ season │
│ -func generateRealmPlayerLeaderboards(db *sql.DB, out, region string, pageSize int │
│ , seasonID int) error { │
│ - // Only generate leaderboards for parent/independent realms (skip child realms) │
│ - rrows, err := db.Query(` │
│ - SELECT slug │
│ - FROM realms │
│ - WHERE region = ? AND (parent_realm_slug IS NULL OR parent_realm_slug = '') │
│ - ORDER BY slug │
│ - `, region) │
│ - if err != nil { │
│ - return fmt.Errorf("players realms list: %w", err) │
│ +// generateSingleRealmPlayerLeaderboard generates player rankings for a single re │
│ alm │
│ +func generateSingleRealmPlayerLeaderboard(db *sql.DB, out, region, rslug string, │
│ pageSize int, seasonID int) error { │
│ + dir := filepath.Join(out, "players", "realm", region, rslug) │
│ + if err := os.MkdirAll(dir, 0o755); err != nil { │
│ + return err │
│ } │
│ - defer rrows.Close() │
│ │
│ - var slugs []string │
│ - for rrows.Next() { │
│ - var s string │
│ - if err := rrows.Scan(&s); err != nil { │
│ - return err │
│ - } │
│ - slugs = append(slugs, s) │
│ + // Include players from entire pool (parent + all children) │
│ + var total int │
│ + if err := db.QueryRow(` │
│ + SELECT COUNT(*) │
│ + FROM players p │
│ + JOIN realms r ON p.realm_id = r.id │
│ + JOIN player_profiles pp ON p.id = pp.player_id │
│ + WHERE pp.season_id = ? AND r.region = ? │
│ + AND (r.slug = ? OR r.parent_realm_slug = ?) │
│ + AND pp.has_complete_coverage = 1 AND pp.combined_best_time IS NOT NULL │
│ + `, seasonID, region, rslug, rslug).Scan(&total); err != nil { │
│ + return fmt.Errorf("players total (realm, season %d): %w", seasonID, err) │
│ } │
│ │
│ - for _, rslug := range slugs { │
│ - dir := filepath.Join(out, "players", "realm", region, rslug) │
│ - if err := os.MkdirAll(dir, 0o755); err != nil { │
│ - return err │
│ - } │
│ - │
│ - // Include players from entire pool (parent + all children) │
│ - var total int │
│ - if err := db.QueryRow(` │
│ - SELECT COUNT(*) │
│ + pages := (total + pageSize - 1) / pageSize │
│ + for p := 1; p <= pages; p++ { │
│ + offset := (p - 1) * pageSize │
│ + rows, err := db.Query(` │
│ + SELECT p.id, p.name, r.slug, r.name, r.region, │
│ + COALESCE(pd.class_name,''), COALESCE(pd.active_spec_name,''), pp.main_spec │
│ _id, │
│ + pp.combined_best_time, pp.dungeons_completed, pp.total_runs, │
│ + COALESCE(pp.realm_ranking_bracket, '') │
│ FROM players p │
│ JOIN realms r ON p.realm_id = r.id │
│ JOIN player_profiles pp ON p.id = pp.player_id │
│ + LEFT JOIN player_details pd ON p.id = pd.player_id │
│ WHERE pp.season_id = ? AND r.region = ? │
│ AND (r.slug = ? OR r.parent_realm_slug = ?) │
│ AND pp.has_complete_coverage = 1 AND pp.combined_best_time IS NOT NULL │
│ - `, seasonID, region, rslug, rslug).Scan(&total); err != nil { │
│ - return fmt.Errorf("players total (realm, season %d): %w", seasonID, err) │
│ - } │
│ - │
│ - pages := (total + pageSize - 1) / pageSize │
│ - for p := 1; p <= pages; p++ { │
│ - offset := (p - 1) * pageSize │
│ - rows, err := db.Query(` │
│ - SELECT p.id, p.name, r.slug, r.name, r.region, │
│ - COALESCE(pd.class_name,''), COALESCE(pd.active_spec_name,''), pp.main_spe │
│ c_id, │
│ - pp.combined_best_time, pp.dungeons_completed, pp.total_runs, │
│ - COALESCE(pp.realm_ranking_bracket, '') │
│ - FROM players p │
│ - JOIN realms r ON p.realm_id = r.id │
│ - JOIN player_profiles pp ON p.id = pp.player_id │
│ - LEFT JOIN player_details pd ON p.id = pd.player_id │
│ - WHERE pp.season_id = ? AND r.region = ? │
│ - AND (r.slug = ? OR r.parent_realm_slug = ?) │
│ - AND pp.has_complete_coverage = 1 AND pp.combined_best_time IS NOT NULL │
│ - ORDER BY pp.combined_best_time ASC, p.name ASC │
│ - LIMIT ? OFFSET ? │
│ - `, seasonID, region, rslug, rslug, pageSize, offset) │
│ - if err != nil { │
│ - return err │
│ - } │
│ + ORDER BY pp.combined_best_time ASC, p.name ASC │
│ + LIMIT ? OFFSET ? │
│ + `, seasonID, region, rslug, rslug, pageSize, offset) │
│ + if err != nil { │
│ + return err │
│ + } │
│ │
│ - list, err := scanPlayerRows(rows) │
│ - if err != nil { │
│ - return err │
│ - } │
│ + list, err := scanPlayerRows(rows) │
│ + if err != nil { │
│ + return err │
│ + } │
│ │
│ - title := strings.ToUpper(region) + "/" + rslug + " Player Rankings" │
│ - page := buildPlayerLeaderboardPage(list, title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ - return err │
│ - } │
│ + title := strings.ToUpper(region) + "/" + rslug + " Player Rankings" │
│ + page := buildPlayerLeaderboardPage(list, title, total, pages, p, pageSize) │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ + return err │
│ } │
│ } │
│ return nil │
│ @@ -446,7 +549,7 @@ func generateClassScope(db *sql.DB, out, scope, region, realmS │
│ lug, classKey stri │
│ } │
│ │
│ page := buildPlayerLeaderboardPage(list, title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.Join(dir, fmt.Sprintf("%d.json", p)), p │
│ age); err != nil { │
│ + if err := writer.WriteJSONFileCompact(filepath.Join(dir, fmt.Sprintf("%d.json", │
│ p)), page); err != nil { │
│ return err │
│ } │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...erator/player_leaderboards.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ generator/player_leaderboards.go b/nix/pkgs/ │
│ ookstats/src/internal/generator/player_leade │
│ rboards.go │
│ index 6d7811c..ae6f183 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generat │
│ or/player_leaderboards.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generat │
│ or/player_leaderboards.go │
│ @@ -8,14 +8,33 @@ import ( │
│ "os" │
│ "path/filepath" │
│ "strings" │
│ + "sync" │
│ + "sync/atomic" │
│ "time" │
│ ) │
│ │
│ +// playerLeaderboardJob represents a single │
│ player leaderboard generation task │
│ +type playerLeaderboardJob struct { │
│ + seasonID int │
│ + seasonName string │
│ + scope string // "global", "regional", │
│ "realm", "class" │
│ + region string // for regional/realm/cl │
│ ass scopes │
│ + realmSlug string // for realm scope │
│ + classKey string // for class scope │
│ + out string │
│ + pageSize int │
│ + regions []string // for class scope │
│ +} │
│ + │
│ // GeneratePlayerLeaderboards generates pla │
│ yer ranking JSON files for all scopes per se │
│ ason │
│ -func GeneratePlayerLeaderboards(db *sql.DB, │
│ out string, pageSize int, regions []string) │
│ error { │
│ +// Uses a worker pool for parallel generati │
│ on │
│ +func GeneratePlayerLeaderboards(db *sql.DB, │
│ out string, pageSize int, regions []string, │
│ workers int) error { │
│ if pageSize <= 0 { │
│ pageSize = 25 │
│ } │
│ + if workers <= 0 { │
│ + workers = 10 │
│ + } │
│ │
│ // Default regions │
│ if len(regions) == 0 { │
│ @@ -33,39 +52,146 @@ func GeneratePlayerLead │
│ erboards(db *sql.DB, out string, pageSize in │
│ t, regions [] │
│ return nil │
│ } │
│ │
│ - // Generate leaderboards for each season │
│ + // Pre-load realm slugs per region (only p │
│ arent realms) │
│ + realmSlugs := make(map[string][]string) │
│ + for _, reg := range regions { │
│ + rrows, err := db.Query(` │
│ + SELECT slug │
│ + FROM realms │
│ + WHERE region = ? AND (parent_realm_slug │
│ IS NULL OR parent_realm_slug = '') │
│ + ORDER BY slug │
│ + `, reg) │
│ + if err != nil { │
│ + return fmt.Errorf("load realm slugs: %w" │
│ , err) │
│ + } │
│ + var slugs []string │
│ + for rrows.Next() { │
│ + var s string │
│ + if err := rrows.Scan(&s); err != nil { │
│ + rrows.Close() │
│ + return err │
│ + } │
│ + slugs = append(slugs, s) │
│ + } │
│ + rrows.Close() │
│ + realmSlugs[reg] = slugs │
│ + } │
│ + │
│ + classKeys := []string{"death_knight", "dru │
│ id", "hunter", "mage", "monk", "paladin", "p │
│ riest", "rogue", "shaman", "warlock", "warri │
│ or"} │
│ + │
│ + // Count total jobs │
│ + var totalJobs int │
│ + for range seasons { │
│ + totalJobs += 1 // global │
│ + totalJobs += len(regions) // regional │
│ + for _, reg := range regions { │
│ + totalJobs += len(realmSlugs[reg]) // rea │
│ lm │
│ + } │
│ + totalJobs += len(classKeys) // class (eac │
│ h handles all scopes internally) │
│ + } │
│ + │
│ + fmt.Printf("Generating player leaderboards │
│ with %d workers (%d total jobs)...\n", work │
│ ers, totalJobs) │
│ + │
│ + // Create job channel │
│ + jobs := make(chan playerLeaderboardJob, 10 │
│ 0) │
│ + var firstErr atomic.Value │
│ + var completed atomic.Int64 │
│ + var wg sync.WaitGroup │
│ + │
│ + // Start workers │
│ + for i := 0; i < workers; i++ { │
│ + wg.Add(1) │
│ + go func() { │
│ + defer wg.Done() │
│ + for job := range jobs { │
│ + if firstErr.Load() != nil { │
│ + continue │
│ + } │
│ + │
│ + var err error │
│ + switch job.scope { │
│ + case "global": │
│ + err = generateGlobalPlayerLeaderboard( │
│ db, job.out, job.pageSize, job.seasonID) │
│ + case "regional": │
│ + err = generateRegionalPlayerLeaderboar │
│ d(db, job.out, job.region, job.pageSize, job │
│ .seasonID) │
│ + case "realm": │
│ + err = generateSingleRealmPlayerLeaderb │
│ oard(db, job.out, job.region, job.realmSlug, │
│ job.pageSize, job.seasonID) │
│ + case "class": │
│ + err = generateClassPlayerLeaderboards( │
│ db, job.out, job.classKey, job.pageSize, job │
│ .regions, job.seasonID) │
│ + } │
│ + │
│ + if err != nil { │
│ + firstErr.CompareAndSwap(nil, err) │
│ + continue │
│ + } │
│ + │
│ + c := completed.Add(1) │
│ + if c%50 == 0 { │
│ + fmt.Printf(" ... %d/%d player leaderb │
│ oards generated\n", c, totalJobs) │
│ + } │
│ + } │
│ + }() │
│ + } │
│ + │
│ + // Queue all jobs │
│ for _, season := range seasons { │
│ - fmt.Printf("\n=== Generating Player Leade │
│ rboards for Season %d (%s) ===\n", season.ID │
│ , season.Name) │
│ + fmt.Printf("\n=== Queuing Player Leaderbo │
│ ards for Season %d (%s) ===\n", season.ID, s │
│ eason.Name) │
│ seasonOut := filepath.Join(out, "season", │
│ fmt.Sprintf("%d", season.ID)) │
│ │
│ - // Generate global and regional scopes fo │
│ r this season │
│ - if err := generateGlobalPlayerLeaderboard │
│ (db, seasonOut, pageSize, season.ID); err != │
│ nil { │
│ - return err │
│ + // Global │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "global", │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ │
│ + // Regional │
│ for _, reg := range regions { │
│ - if err := generateRegionalPlayerLeaderbo │
│ ard(db, seasonOut, reg, pageSize, season.ID) │
│ ; err != nil { │
│ - return err │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "regional", │
│ + region: reg, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ } │
│ } │
│ │
│ - // Generate realm scopes for this season │
│ + // Realm │
│ for _, reg := range regions { │
│ - if err := generateRealmPlayerLeaderboard │
│ s(db, seasonOut, reg, pageSize, season.ID); │
│ err != nil { │
│ - return err │
│ + for _, rslug := range realmSlugs[reg] { │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "realm", │
│ + region: reg, │
│ + realmSlug: rslug, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ + } │
│ } │
│ } │
│ │
│ - // Generate class-filtered variants for t │
│ his season │
│ - classKeys := []string{"death_knight", "dr │
│ uid", "hunter", "mage", "monk", "paladin", " │
│ priest", "rogue", "shaman", "warlock", "warr │
│ ior"} │
│ + // Class (each handles global/regional/re │
│ alm internally) │
│ for _, cls := range classKeys { │
│ - if err := generateClassPlayerLeaderboard │
│ s(db, seasonOut, cls, pageSize, regions, sea │
│ son.ID); err != nil { │
│ - return err │
│ + jobs <- playerLeaderboardJob{ │
│ + seasonID: season.ID, │
│ + scope: "class", │
│ + classKey: cls, │
│ + out: seasonOut, │
│ + pageSize: pageSize, │
│ + regions: regions, │
│ } │
│ } │
│ } │
│ + close(jobs) │
│ + │
│ + wg.Wait() │
│ + │
│ + if err := firstErr.Load(); err != nil { │
│ + return err.(error) │
│ + } │
│ │
│ - fmt.Println("[OK] Player leaderboards gene │
│ rated for all seasons") │
│ + fmt.Printf("\n[OK] Generated %d player lea │
│ derboards\n", completed.Load()) │
│ return nil │
│ } │
│ │
│ @@ -111,7 +237,7 @@ func generateGlobalPlaye │
│ rLeaderboard(db *sql.DB, out string, pageSiz │
│ e int, seaso │
│ } │
│ │
│ page := buildPlayerLeaderboardPage(list, │
│ "Global Player Rankings", total, pages, p, p │
│ ageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
│ @@ -163,88 +289,65 @@ func generateRegionalP │
│ layerLeaderboard(db *sql.DB, out, region str │
│ ing, pageSize │
│ │
│ title := strings.ToUpper(region) + " Play │
│ er Rankings" │
│ page := buildPlayerLeaderboardPage(list, │
│ title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
│ return nil │
│ } │
│ │
│ -// generateRealmPlayerLeaderboards generate │
│ s realm-specific player rankings for a seaso │
│ n │
│ -func generateRealmPlayerLeaderboards(db *sq │
│ l.DB, out, region string, pageSize int, seas │
│ onID int) error { │
│ - // Only generate leaderboards for parent/i │
│ ndependent realms (skip child realms) │
│ - rrows, err := db.Query(` │
│ - SELECT slug │
│ - FROM realms │
│ - WHERE region = ? AND (parent_realm_slug I │
│ S NULL OR parent_realm_slug = '') │
│ - ORDER BY slug │
│ - `, region) │
│ - if err != nil { │
│ - return fmt.Errorf("players realms list: % │
│ w", err) │
│ +// generateSingleRealmPlayerLeaderboard gen │
│ erates player rankings for a single realm │
│ +func generateSingleRealmPlayerLeaderboard(d │
│ b *sql.DB, out, region, rslug string, pageSi │
│ ze int, seasonID int) error { │
│ + dir := filepath.Join(out, "players", "real │
│ m", region, rslug) │
│ + if err := os.MkdirAll(dir, 0o755); err != │
│ nil { │
│ + return err │
│ } │
│ - defer rrows.Close() │
│ │
│ - var slugs []string │
│ - for rrows.Next() { │
│ - var s string │
│ - if err := rrows.Scan(&s); err != nil { │
│ - return err │
│ - } │
│ - slugs = append(slugs, s) │
│ + // Include players from entire pool (paren │
│ t + all children) │
│ + var total int │
│ + if err := db.QueryRow(` │
│ + SELECT COUNT(*) │
│ + FROM players p │
│ + JOIN realms r ON p.realm_id = r.id │
│ + JOIN player_profiles pp ON p.id = pp.play │
│ er_id │
│ + WHERE pp.season_id = ? AND r.region = ? │
│ + AND (r.slug = ? OR r.parent_realm_slug = │
│ ?) │
│ + AND pp.has_complete_coverage = 1 AND pp. │
│ combined_best_time IS NOT NULL │
│ + `, seasonID, region, rslug, rslug).Scan(&t │
│ otal); err != nil { │
│ + return fmt.Errorf("players total (realm, │
│ season %d): %w", seasonID, err) │
│ } │
│ │
│ - for _, rslug := range slugs { │
│ - dir := filepath.Join(out, "players", "rea │
│ lm", region, rslug) │
│ - if err := os.MkdirAll(dir, 0o755); err != │
│ nil { │
│ - return err │
│ - } │
│ - │
│ - // Include players from entire pool (pare │
│ nt + all children) │
│ - var total int │
│ - if err := db.QueryRow(` │
│ - SELECT COUNT(*) │
│ + pages := (total + pageSize - 1) / pageSize │
│ + for p := 1; p <= pages; p++ { │
│ + offset := (p - 1) * pageSize │
│ + rows, err := db.Query(` │
│ + SELECT p.id, p.name, r.slug, r.name, r.r │
│ egion, │
│ + COALESCE(pd.class_name,''), COALESCE │
│ (pd.active_spec_name,''), pp.main_spec_id, │
│ + pp.combined_best_time, pp.dungeons_c │
│ ompleted, pp.total_runs, │
│ + COALESCE(pp.realm_ranking_bracket, ' │
│ ') │
│ FROM players p │
│ JOIN realms r ON p.realm_id = r.id │
│ JOIN player_profiles pp ON p.id = pp.pla │
│ yer_id │
│ + LEFT JOIN player_details pd ON p.id = pd │
│ .player_id │
│ WHERE pp.season_id = ? AND r.region = ? │
│ AND (r.slug = ? OR r.parent_realm_slug │
│ = ?) │
│ AND pp.has_complete_coverage = 1 AND pp │
│ .combined_best_time IS NOT NULL │
│ - `, seasonID, region, rslug, rslug).Scan(& │
│ total); err != nil { │
│ - return fmt.Errorf("players total (realm, │
│ season %d): %w", seasonID, err) │
│ - } │
│ - │
│ - pages := (total + pageSize - 1) / pageSiz │
│ e │
│ - for p := 1; p <= pages; p++ { │
│ - offset := (p - 1) * pageSize │
│ - rows, err := db.Query(` │
│ - SELECT p.id, p.name, r.slug, r.name, r. │
│ region, │
│ - COALESCE(pd.class_name,''), COALESC │
│ E(pd.active_spec_name,''), pp.main_spec_id, │
│ - pp.combined_best_time, pp.dungeons_ │
│ completed, pp.total_runs, │
│ - COALESCE(pp.realm_ranking_bracket, │
│ '') │
│ - FROM players p │
│ - JOIN realms r ON p.realm_id = r.id │
│ - JOIN player_profiles pp ON p.id = pp.pl │
│ ayer_id │
│ - LEFT JOIN player_details pd ON p.id = p │
│ d.player_id │
│ - WHERE pp.season_id = ? AND r.region = ? │
│ - AND (r.slug = ? OR r.parent_realm_slug │
│ = ?) │
│ - AND pp.has_complete_coverage = 1 AND p │
│ p.combined_best_time IS NOT NULL │
│ - ORDER BY pp.combined_best_time ASC, p.n │
│ ame ASC │
│ - LIMIT ? OFFSET ? │
│ - `, seasonID, region, rslug, rslug, pageS │
│ ize, offset) │
│ - if err != nil { │
│ - return err │
│ - } │
│ + ORDER BY pp.combined_best_time ASC, p.na │
│ me ASC │
│ + LIMIT ? OFFSET ? │
│ + `, seasonID, region, rslug, rslug, pageSi │
│ ze, offset) │
│ + if err != nil { │
│ + return err │
│ + } │
│ │
│ - list, err := scanPlayerRows(rows) │
│ - if err != nil { │
│ - return err │
│ - } │
│ + list, err := scanPlayerRows(rows) │
│ + if err != nil { │
│ + return err │
│ + } │
│ │
│ - title := strings.ToUpper(region) + "/" + │
│ rslug + " Player Rankings" │
│ - page := buildPlayerLeaderboardPage(list, │
│ title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath. │
│ Join(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ - return err │
│ - } │
│ + title := strings.ToUpper(region) + "/" + │
│ rslug + " Player Rankings" │
│ + page := buildPlayerLeaderboardPage(list, │
│ title, total, pages, p, pageSize) │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ + return err │
│ } │
│ } │
│ return nil │
│ @@ -446,7 +549,7 @@ func generateClassScope( │
│ db *sql.DB, out, scope, region, realmSlug, c │
│ lassKey stri │
│ } │
│ │
│ page := buildPlayerLeaderboardPage(list, │
│ title, total, pages, p, pageSize) │
│ - if err := writer.WriteJSONFile(filepath.J │
│ oin(dir, fmt.Sprintf("%d.json", p)), page); │
│ err != nil { │
│ + if err := writer.WriteJSONFileCompact(fil │
│ epath.Join(dir, fmt.Sprintf("%d.json", p)), │
│ page); err != nil { │
│ return err │
│ } │
│ } │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/generator/players.go ──────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/generator/players.go b/nix/pkgs/ooksta │
│ ts/src/internal/generator/players.go │
│ index d2af49a..e8ecc2d 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generator/players.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generator/players.go │
│ @@ -392,5 +392,5 @@ func generateSinglePlayerJSON(player loader.PlayerData, player │
│ SeasonsMap map[int │
│ // Write file │
│ dir := filepath.Join(out, pj.Region, pj.RealmSlug) │
│ fname := filepath.Join(dir, utils.SafeSlugName(pj.Name)+".json") │
│ - return writer.WriteJSONFile(fname, page) │
│ + return writer.WriteJSONFileCompact(fname, page) │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...internal/generator/players.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ generator/players.go b/nix/pkgs/ookstats/src │
│ /internal/generator/players.go │
│ index d2af49a..e8ecc2d 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generat │
│ or/players.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generat │
│ or/players.go │
│ @@ -392,5 +392,5 @@ func generateSinglePlaye │
│ rJSON(player loader.PlayerData, playerSeason │
│ sMap map[int │
│ // Write file │
│ dir := filepath.Join(out, pj.Region, pj.Re │
│ almSlug) │
│ fname := filepath.Join(dir, utils.SafeSlug │
│ Name(pj.Name)+".json") │
│ - return writer.WriteJSONFile(fname, page) │
│ + return writer.WriteJSONFileCompact(fname, │
│ page) │
│ } │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/generator/search.go ───────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/generator/search.go b/nix/pkgs/ookstat │
│ s/src/internal/generator/search.go │
│ index 8b931c5..355ce08 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generator/search.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generator/search.go │
│ @@ -73,7 +73,7 @@ func GenerateSearchIndex(db *sql.DB, out string, shardSize int) │
│ error { │
│ "limit": shardSize, │
│ "last_updated": time.Now().Format(time.RFC3339), │
│ } │
│ - if err := writer.WriteJSONFile(path, map[string]any{"players": buf, "metadata": │
│ meta}); err != nil { │
│ + if err := writer.WriteJSONFileCompact(path, map[string]any{"players": buf, "met │
│ adata": meta}); err != nil { │
│ return err │
│ } │
│ shard++ │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ .../internal/generator/search.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ generator/search.go b/nix/pkgs/ookstats/src/ │
│ internal/generator/search.go │
│ index 8b931c5..355ce08 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/generat │
│ or/search.go │
│ +++ b/nix/pkgs/ookstats/src/internal/generat │
│ or/search.go │
│ @@ -73,7 +73,7 @@ func GenerateSearchIndex(d │
│ b *sql.DB, out string, shardSize int) error │
│ { │
│ "limit": shardSize, │
│ "last_updated": time.Now().Format(ti │
│ me.RFC3339), │
│ } │
│ - if err := writer.WriteJSONFile(path, map[ │
│ string]any{"players": buf, "metadata": meta} │
│ ); err != nil { │
│ + if err := writer.WriteJSONFileCompact(pat │
│ h, map[string]any{"players": buf, "metadata" │
│ : meta}); err != nil { │
│ return err │
│ } │
│ shard++ │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/loader/equipment.go ───────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/loader/equipment.go b/nix/pkgs/ookstat │
│ s/src/internal/loader/equipment.go │
│ index 6b2d4ee..cecf405 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/loader/equipment.go │
│ +++ b/nix/pkgs/ookstats/src/internal/loader/equipment.go │
│ @@ -74,39 +74,54 @@ func LoadAllEquipment(db *sql.DB, playerIDs []int64) (map[int6 │
│ 4]map[int64][]Equi │
│ return make(map[int64]map[int64][]EquipmentData), make(map[int64][]EnchantmentD │
│ ata), nil │
│ } │
│ │
│ - // Load equipment for latest timestamps │
│ + // Load equipment for latest timestamps in a single batched query │
│ equipmentMap := make(map[int64]map[int64][]EquipmentData) │
│ var allEquipmentIDs []int64 │
│ │
│ + // Build VALUES clause for all player/timestamp pairs │
│ + valuesBuilder := strings.Builder{} │
│ + eqArgs := make([]any, 0, len(playerTimestamps)*2) │
│ + first := true │
│ for playerID, timestamp := range playerTimestamps { │
│ - rows, err := db.Query(` │
│ - SELECT e.id, e.slot_type, e.item_id, e.upgrade_id, e.quality, e.item_ │
│ name, e.snapshot_timestamp, │
│ - i.icon AS item_icon_slug, i.type AS item_type │
│ - FROM player_equipment e │
│ - LEFT JOIN items i ON e.item_id = i.id │
│ - WHERE e.player_id = ? AND e.snapshot_timestamp = ? │
│ - ORDER BY e.slot_type │
│ - `, playerID, timestamp) │
│ - if err != nil { │
│ - return nil, nil, err │
│ + if !first { │
│ + valuesBuilder.WriteString(",") │
│ } │
│ + valuesBuilder.WriteString("(?,?)") │
│ + eqArgs = append(eqArgs, playerID, timestamp) │
│ + first = false │
│ + } │
│ │
│ - if equipmentMap[playerID] == nil { │
│ - equipmentMap[playerID] = make(map[int64][]EquipmentData) │
│ + // Single query to load all equipment │
│ + eqQuery := fmt.Sprintf(` │
│ + WITH pairs(player_id, ts) AS (VALUES %s) │
│ + SELECT e.player_id, e.id, e.slot_type, e.item_id, e.upgrade_id, e.quality, e.it │
│ em_name, e.snapshot_timestamp, │
│ + i.icon AS item_icon_slug, i.type AS item_type │
│ + FROM pairs │
│ + JOIN player_equipment e ON e.player_id = pairs.player_id AND e.snapshot_timesta │
│ mp = pairs.ts │
│ + LEFT JOIN items i ON e.item_id = i.id │
│ + ORDER BY e.player_id, e.slot_type │
│ + `, valuesBuilder.String()) │
│ + │
│ + eqRows, err := db.Query(eqQuery, eqArgs...) │
│ + if err != nil { │
│ + return nil, nil, fmt.Errorf("batch equipment query: %w", err) │
│ + } │
│ + defer eqRows.Close() │
│ + │
│ + for eqRows.Next() { │
│ + var playerID int64 │
│ + var eq EquipmentData │
│ + if err := eqRows.Scan( │
│ + &playerID, &eq.ID, &eq.SlotType, &eq.ItemID, &eq.UpgradeID, &eq.Quality, &eq.I │
│ temName, &eq.SnapshotTs, │
│ + &eq.ItemIcon, &eq.ItemType); err != nil { │
│ + return nil, nil, fmt.Errorf("scan equipment: %w", err) │
│ } │
│ │
│ - for rows.Next() { │
│ - var eq EquipmentData │
│ - if err := rows.Scan( │
│ - &eq.ID, &eq.SlotType, &eq.ItemID, &eq.UpgradeID, &eq.Quality, &eq.ItemName, & │
│ eq.SnapshotTs, │
│ - &eq.ItemIcon, &eq.ItemType); err != nil { │
│ - rows.Close() │
│ - return nil, nil, fmt.Errorf("scan equipment: %w", err) │
│ - } │
│ - equipmentMap[playerID][timestamp] = append(equipmentMap[playerID][timestamp], │
│ eq) │
│ - allEquipmentIDs = append(allEquipmentIDs, eq.ID) │
│ + if equipmentMap[playerID] == nil { │
│ + equipmentMap[playerID] = make(map[int64][]EquipmentData) │
│ } │
│ - rows.Close() │
│ + equipmentMap[playerID][eq.SnapshotTs] = append(equipmentMap[playerID][eq.Snapsh │
│ otTs], eq) │
│ + allEquipmentIDs = append(allEquipmentIDs, eq.ID) │
│ } │
│ │
│ // Load enchantments in batches │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ .../internal/loader/equipment.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ loader/equipment.go b/nix/pkgs/ookstats/src/ │
│ internal/loader/equipment.go │
│ index 6b2d4ee..cecf405 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/loader/ │
│ equipment.go │
│ +++ b/nix/pkgs/ookstats/src/internal/loader/ │
│ equipment.go │
│ @@ -74,39 +74,54 @@ func LoadAllEquipment(db │
│ *sql.DB, playerIDs []int64) (map[int64]map[ │
│ int64][]Equi │
│ return make(map[int64]map[int64][]Equipme │
│ ntData), make(map[int64][]EnchantmentData), │
│ nil │
│ } │
│ │
│ - // Load equipment for latest timestamps │
│ + // Load equipment for latest timestamps in │
│ a single batched query │
│ equipmentMap := make(map[int64]map[int64][ │
│ ]EquipmentData) │
│ var allEquipmentIDs []int64 │
│ │
│ + // Build VALUES clause for all player/time │
│ stamp pairs │
│ + valuesBuilder := strings.Builder{} │
│ + eqArgs := make([]any, 0, len(playerTimesta │
│ mps)*2) │
│ + first := true │
│ for playerID, timestamp := range playerTim │
│ estamps { │
│ - rows, err := db.Query(` │
│ - SELECT e.id, e.slot_type, e.ite │
│ m_id, e.upgrade_id, e.quality, e.item_name, │
│ e.snapshot_timestamp, │
│ - i.icon AS item_icon_slug │
│ , i.type AS item_type │
│ - FROM player_equipment e │
│ - LEFT JOIN items i ON e.item_id │
│ = i.id │
│ - WHERE e.player_id = ? AND e.sna │
│ pshot_timestamp = ? │
│ - ORDER BY e.slot_type │
│ - `, playerID, timestamp) │
│ - if err != nil { │
│ - return nil, nil, err │
│ + if !first { │
│ + valuesBuilder.WriteString(",") │
│ } │
│ + valuesBuilder.WriteString("(?,?)") │
│ + eqArgs = append(eqArgs, playerID, timesta │
│ mp) │
│ + first = false │
│ + } │
│ │
│ - if equipmentMap[playerID] == nil { │
│ - equipmentMap[playerID] = make(map[int64] │
│ []EquipmentData) │
│ + // Single query to load all equipment │
│ + eqQuery := fmt.Sprintf(` │
│ + WITH pairs(player_id, ts) AS (VALUES %s) │
│ + SELECT e.player_id, e.id, e.slot_type, e. │
│ item_id, e.upgrade_id, e.quality, e.item_nam │
│ e, e.snapshot_timestamp, │
│ + i.icon AS item_icon_slug, i.type A │
│ S item_type │
│ + FROM pairs │
│ + JOIN player_equipment e ON e.player_id = │
│ pairs.player_id AND e.snapshot_timestamp = p │
│ airs.ts │
│ + LEFT JOIN items i ON e.item_id = i.id │
│ + ORDER BY e.player_id, e.slot_type │
│ + `, valuesBuilder.String()) │
│ + │
│ + eqRows, err := db.Query(eqQuery, eqArgs... │
│ ) │
│ + if err != nil { │
│ + return nil, nil, fmt.Errorf("batch equipm │
│ ent query: %w", err) │
│ + } │
│ + defer eqRows.Close() │
│ + │
│ + for eqRows.Next() { │
│ + var playerID int64 │
│ + var eq EquipmentData │
│ + if err := eqRows.Scan( │
│ + &playerID, &eq.ID, &eq.SlotType, &eq.Ite │
│ mID, &eq.UpgradeID, &eq.Quality, &eq.ItemNam │
│ e, &eq.SnapshotTs, │
│ + &eq.ItemIcon, &eq.ItemType); err != nil │
│ { │
│ + return nil, nil, fmt.Errorf("scan equipm │
│ ent: %w", err) │
│ } │
│ │
│ - for rows.Next() { │
│ - var eq EquipmentData │
│ - if err := rows.Scan( │
│ - &eq.ID, &eq.SlotType, &eq.ItemID, &eq.U │
│ pgradeID, &eq.Quality, &eq.ItemName, &eq.Sna │
│ pshotTs, │
│ - &eq.ItemIcon, &eq.ItemType); err != nil │
│ { │
│ - rows.Close() │
│ - return nil, nil, fmt.Errorf("scan equip │
│ ment: %w", err) │
│ - } │
│ - equipmentMap[playerID][timestamp] = appe │
│ nd(equipmentMap[playerID][timestamp], eq) │
│ - allEquipmentIDs = append(allEquipmentIDs │
│ , eq.ID) │
│ + if equipmentMap[playerID] == nil { │
│ + equipmentMap[playerID] = make(map[int64] │
│ []EquipmentData) │
│ } │
│ - rows.Close() │
│ + equipmentMap[playerID][eq.SnapshotTs] = a │
│ ppend(equipmentMap[playerID][eq.SnapshotTs], │
│ eq) │
│ + allEquipmentIDs = append(allEquipmentIDs, │
│ eq.ID) │
│ } │
│ │
│ // Load enchantments in batches │
└──────────────────────────────────────────────┘
[ BACK TO LOG ]
──────────────────────────────────────────────────────────────────────────────────────
OOKNET
────────────────────────────────────────────────
OOKNET