HASH d5832dcfbeba
DATE 2026-07-16
SUBJECT ookstats: cooldown failed account fingerprint backfills
FILES 4 CHANGED
HASH d5832dcfbeba
DATE 2026-07-16
SUBJECT ookstats: cooldown failed account
fingerprint backfills
FILES 4 CHANGED
┌─ nix/pkgs/ookstats/src/internal/database/accounts.go ──────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/database/accounts.go b/nix/pkgs/ooksta │
│ ts/src/internal/database/accounts.go │
│ index 5d81d44..cec819e 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/database/accounts.go │
│ +++ b/nix/pkgs/ookstats/src/internal/database/accounts.go │
│ @@ -56,7 +56,7 @@ type AccountBackfillCandidate struct { │
│ Region string │
│ } │
│ │
│ -func (ds *DatabaseService) PlayersMissingAccountFingerprint() ([]AccountBackfillC │
│ andidate, error) { │
│ +func (ds *DatabaseService) PlayersMissingAccountFingerprint(retryBefore int64) ([ │
│ ]AccountBackfillCandidate, error) { │
│ rows, err := ds.db.Query(` │
│ SELECT pf.player_id, p.name, r.slug, r.region │
│ FROM player_fingerprints pf │
│ @@ -65,7 +65,8 @@ func (ds *DatabaseService) PlayersMissingAccountFingerprint() ([ │
│ ]AccountBackfill │
│ LEFT JOIN player_account_fingerprint paf ON paf.player_id = pf.player_id │
│ WHERE paf.player_id IS NULL │
│ AND p.is_valid = 1 │
│ - `) │
│ + AND (p.account_fp_attempted_at IS NULL OR p.account_fp_attempted_at < ?) │
│ + `, retryBefore) │
│ if err != nil { │
│ return nil, err │
│ } │
│ @@ -138,6 +139,31 @@ func (ds *DatabaseService) RebuildAccounts(groups []AccountGr │
│ oup, computedAt int │
│ return totalChars, nil │
│ } │
│ │
│ +// MarkAccountFingerprintAttempts stamps candidates so failed fetches wait out th │
│ e retry cooldown │
│ +func (ds *DatabaseService) MarkAccountFingerprintAttempts(playerIDs []int64, ts i │
│ nt64) error { │
│ + if len(playerIDs) == 0 { │
│ + return nil │
│ + } │
│ + return retryOnBusy(func() error { │
│ + tx, err := ds.db.Begin() │
│ + if err != nil { │
│ + return err │
│ + } │
│ + defer tx.Rollback() │
│ + stmt, err := tx.Prepare(`UPDATE players SET account_fp_attempted_at = ? WHERE i │
│ d = ?`) │
│ + if err != nil { │
│ + return err │
│ + } │
│ + defer stmt.Close() │
│ + for _, id := range playerIDs { │
│ + if _, err := stmt.Exec(ts, id); err != nil { │
│ + return err │
│ + } │
│ + } │
│ + return tx.Commit() │
│ + }) │
│ +} │
│ + │
│ func (ds *DatabaseService) CountPlayerAccountFingerprints() (int, error) { │
│ var n int │
│ err := ds.db.QueryRow(`SELECT COUNT(*) FROM player_account_fingerprint`).Scan(&n │
│ ) │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...internal/database/accounts.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ database/accounts.go b/nix/pkgs/ookstats/src │
│ /internal/database/accounts.go │
│ index 5d81d44..cec819e 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/databas │
│ e/accounts.go │
│ +++ b/nix/pkgs/ookstats/src/internal/databas │
│ e/accounts.go │
│ @@ -56,7 +56,7 @@ type AccountBackfillCandid │
│ ate struct { │
│ Region string │
│ } │
│ │
│ -func (ds *DatabaseService) PlayersMissingAc │
│ countFingerprint() ([]AccountBackfillCandida │
│ te, error) { │
│ +func (ds *DatabaseService) PlayersMissingAc │
│ countFingerprint(retryBefore int64) ([]Accou │
│ ntBackfillCandidate, error) { │
│ rows, err := ds.db.Query(` │
│ SELECT pf.player_id, p.name, r.slug, r.re │
│ gion │
│ FROM player_fingerprints pf │
│ @@ -65,7 +65,8 @@ func (ds *DatabaseService) │
│ PlayersMissingAccountFingerprint() ([]Accou │
│ ntBackfill │
│ LEFT JOIN player_account_fingerprint paf │
│ ON paf.player_id = pf.player_id │
│ WHERE paf.player_id IS NULL │
│ AND p.is_valid = 1 │
│ - `) │
│ + AND (p.account_fp_attempted_at IS NULL │
│ OR p.account_fp_attempted_at < ?) │
│ + `, retryBefore) │
│ if err != nil { │
│ return nil, err │
│ } │
│ @@ -138,6 +139,31 @@ func (ds *DatabaseServi │
│ ce) RebuildAccounts(groups []AccountGroup, c │
│ omputedAt int │
│ return totalChars, nil │
│ } │
│ │
│ +// MarkAccountFingerprintAttempts stamps ca │
│ ndidates so failed fetches wait out the retr │
│ y cooldown │
│ +func (ds *DatabaseService) MarkAccountFinge │
│ rprintAttempts(playerIDs []int64, ts int64) │
│ error { │
│ + if len(playerIDs) == 0 { │
│ + return nil │
│ + } │
│ + return retryOnBusy(func() error { │
│ + tx, err := ds.db.Begin() │
│ + if err != nil { │
│ + return err │
│ + } │
│ + defer tx.Rollback() │
│ + stmt, err := tx.Prepare(`UPDATE players S │
│ ET account_fp_attempted_at = ? WHERE id = ?` │
│ ) │
│ + if err != nil { │
│ + return err │
│ + } │
│ + defer stmt.Close() │
│ + for _, id := range playerIDs { │
│ + if _, err := stmt.Exec(ts, id); err != n │
│ il { │
│ + return err │
│ + } │
│ + } │
│ + return tx.Commit() │
│ + }) │
│ +} │
│ + │
│ func (ds *DatabaseService) CountPlayerAccou │
│ ntFingerprints() (int, error) { │
│ var n int │
│ err := ds.db.QueryRow(`SELECT COUNT(*) FRO │
│ M player_account_fingerprint`).Scan(&n) │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/database/migrations.go ────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/database/migrations.go b/nix/pkgs/ooks │
│ tats/src/internal/database/migrations.go │
│ index 0a04670..7900c66 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/database/migrations.go │
│ +++ b/nix/pkgs/ookstats/src/internal/database/migrations.go │
│ @@ -172,6 +172,16 @@ func migratePlayersIdentityColumns(db *sql.DB) error { │
│ return fmt.Errorf("add status_checked_at: %w", err) │
│ } │
│ } │
│ + │
│ + hasFpAttempted, err := columnExists(db, "players", "account_fp_attempted_at") │
│ + if err != nil { │
│ + return err │
│ + } │
│ + if !hasFpAttempted { │
│ + if _, err := db.Exec(`ALTER TABLE players ADD COLUMN account_fp_attempted_at IN │
│ TEGER`); err != nil { │
│ + return fmt.Errorf("add account_fp_attempted_at: %w", err) │
│ + } │
│ + } │
│ return nil │
│ } │
│ │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...ternal/database/migrations.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ database/migrations.go b/nix/pkgs/ookstats/s │
│ rc/internal/database/migrations.go │
│ index 0a04670..7900c66 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/databas │
│ e/migrations.go │
│ +++ b/nix/pkgs/ookstats/src/internal/databas │
│ e/migrations.go │
│ @@ -172,6 +172,16 @@ func migratePlayersIden │
│ tityColumns(db *sql.DB) error { │
│ return fmt.Errorf("add status_checked_at │
│ : %w", err) │
│ } │
│ } │
│ + │
│ + hasFpAttempted, err := columnExists(db, "p │
│ layers", "account_fp_attempted_at") │
│ + if err != nil { │
│ + return err │
│ + } │
│ + if !hasFpAttempted { │
│ + if _, err := db.Exec(`ALTER TABLE players │
│ ADD COLUMN account_fp_attempted_at INTEGER` │
│ ); err != nil { │
│ + return fmt.Errorf("add account_fp_attemp │
│ ted_at: %w", err) │
│ + } │
│ + } │
│ return nil │
│ } │
│ │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/database/schema.go ────────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/database/schema.go b/nix/pkgs/ookstats │
│ /src/internal/database/schema.go │
│ index 70a8778..067f6ff 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/database/schema.go │
│ +++ b/nix/pkgs/ookstats/src/internal/database/schema.go │
│ @@ -51,7 +51,8 @@ func EnsureCompleteSchema(db *sql.DB) error { │
│ name_lower TEXT, │
│ realm_id INTEGER, │
│ is_valid INTEGER DEFAULT 1, │
│ - status_checked_at INTEGER │
│ + status_checked_at INTEGER, │
│ + account_fp_attempted_at INTEGER │
│ )`, │
│ │
│ `CREATE TABLE IF NOT EXISTS run_members ( │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...c/internal/database/schema.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ database/schema.go b/nix/pkgs/ookstats/src/i │
│ nternal/database/schema.go │
│ index 70a8778..067f6ff 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/databas │
│ e/schema.go │
│ +++ b/nix/pkgs/ookstats/src/internal/databas │
│ e/schema.go │
│ @@ -51,7 +51,8 @@ func EnsureCompleteSchema( │
│ db *sql.DB) error { │
│ name_lower TEXT, │
│ realm_id INTEGER, │
│ is_valid INTEGER DEFAULT 1, │
│ - status_checked_at INTEGER │
│ + status_checked_at INTEGER, │
│ + account_fp_attempted_at INTEGER │
│ )`, │
│ │
│ `CREATE TABLE IF NOT EXISTS run_members ( │
└──────────────────────────────────────────────┘
┌─ nix/pkgs/ookstats/src/internal/pipeline/accounts.go ──────────────────────┐
│ diff --git a/nix/pkgs/ookstats/src/internal/pipeline/accounts.go b/nix/pkgs/ooksta │
│ ts/src/internal/pipeline/accounts.go │
│ index bf64371..410eeba 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/pipeline/accounts.go │
│ +++ b/nix/pkgs/ookstats/src/internal/pipeline/accounts.go │
│ @@ -93,8 +93,12 @@ func ProcessAccounts(db *database.DatabaseService, client *bliz │
│ zard.Client, opts │
│ return res, nil │
│ } │
│ │
│ +// deleted or private characters fail every fetch; wait out a cooldown instead of │
│ retrying each build │
│ +const accountBackfillRetryAfter = 7 * 24 * time.Hour │
│ + │
│ func backfillAccountFingerprints(db *database.DatabaseService, client *blizzard.C │
│ lient, opts AccountGroupingOptions, logger *log.Logger) (int, int, error) { │
│ - candidates, err := db.PlayersMissingAccountFingerprint() │
│ + retryBefore := time.Now().Add(-accountBackfillRetryAfter).UnixMilli() │
│ + candidates, err := db.PlayersMissingAccountFingerprint(retryBefore) │
│ if err != nil { │
│ return 0, 0, err │
│ } │
│ @@ -132,7 +136,7 @@ func backfillAccountFingerprints(db *database.DatabaseService, │
│ client *blizzard. │
│ } │
│ tuples := playerid.ExtractTrustedTuples(resp) │
│ if len(tuples) == 0 { │
│ - // skip rather than insert empty so next pass can retry │
│ + // skip rather than insert empty; the attempt stamp defers retry to the next │
│ cooldown │
│ continue │
│ } │
│ j, err := json.Marshal(tuples) │
│ @@ -159,6 +163,15 @@ func backfillAccountFingerprints(db *database.DatabaseService │
│ , client *blizzard. │
│ } │
│ close(jobs) │
│ wg.Wait() │
│ + │
│ + ids := make([]int64, len(candidates)) │
│ + for i, c := range candidates { │
│ + ids[i] = c.PlayerID │
│ + } │
│ + if err := db.MarkAccountFingerprintAttempts(ids, time.Now().UnixMilli()); err != │
│ nil { │
│ + logger.Error("mark backfill attempts", "error", err) │
│ + } │
│ + │
│ logger.Info("backfill complete", "success", success, "errors", fail) │
│ return success, fail, nil │
│ } │
└────────────────────────────────────────────────────────────────────────────────────┘
┌─ ...internal/pipeline/accounts.go ───┐
│ diff --git a/nix/pkgs/ookstats/src/internal/ │
│ pipeline/accounts.go b/nix/pkgs/ookstats/src │
│ /internal/pipeline/accounts.go │
│ index bf64371..410eeba 100644 │
│ --- a/nix/pkgs/ookstats/src/internal/pipelin │
│ e/accounts.go │
│ +++ b/nix/pkgs/ookstats/src/internal/pipelin │
│ e/accounts.go │
│ @@ -93,8 +93,12 @@ func ProcessAccounts(db * │
│ database.DatabaseService, client *blizzard.C │
│ lient, opts │
│ return res, nil │
│ } │
│ │
│ +// deleted or private characters fail every │
│ fetch; wait out a cooldown instead of retry │
│ ing each build │
│ +const accountBackfillRetryAfter = 7 * 24 * │
│ time.Hour │
│ + │
│ func backfillAccountFingerprints(db *databa │
│ se.DatabaseService, client *blizzard.Client, │
│ opts AccountGroupingOptions, logger *log.Lo │
│ gger) (int, int, error) { │
│ - candidates, err := db.PlayersMissingAccoun │
│ tFingerprint() │
│ + retryBefore := time.Now().Add(-accountBack │
│ fillRetryAfter).UnixMilli() │
│ + candidates, err := db.PlayersMissingAccoun │
│ tFingerprint(retryBefore) │
│ if err != nil { │
│ return 0, 0, err │
│ } │
│ @@ -132,7 +136,7 @@ func backfillAccountFing │
│ erprints(db *database.DatabaseService, clien │
│ t *blizzard. │
│ } │
│ tuples := playerid.ExtractTrustedTuples │
│ (resp) │
│ if len(tuples) == 0 { │
│ - // skip rather than insert empty so ne │
│ xt pass can retry │
│ + // skip rather than insert empty; the │
│ attempt stamp defers retry to the next coold │
│ own │
│ continue │
│ } │
│ j, err := json.Marshal(tuples) │
│ @@ -159,6 +163,15 @@ func backfillAccountFin │
│ gerprints(db *database.DatabaseService, clie │
│ nt *blizzard. │
│ } │
│ close(jobs) │
│ wg.Wait() │
│ + │
│ + ids := make([]int64, len(candidates)) │
│ + for i, c := range candidates { │
│ + ids[i] = c.PlayerID │
│ + } │
│ + if err := db.MarkAccountFingerprintAttempt │
│ s(ids, time.Now().UnixMilli()); err != nil { │
│ + logger.Error("mark backfill attempts", "e │
│ rror", err) │
│ + } │
│ + │
│ logger.Info("backfill complete", "success" │
│ , success, "errors", fail) │
│ return success, fail, nil │
│ } │
└──────────────────────────────────────────────┘
[ BACK TO LOG ]
──────────────────────────────────────────────────────────────────────────────────────
OOKNET
────────────────────────────────────────────────
OOKNET