fix: foreign_tickers selalu include semua ticker dengan foreign data (bukan hanya saat watchlist>60)

.gitignore

@@ -49,3 +49,4 @@ task_plan.md
 findings.md
 progress.md
 backend/universe-sweep
+/backend/foreign-backfill

backend/cmd/foreign-backfill/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"os"
+	"time"
+
+	"flowsight/internal/sectors"
+	"flowsight/internal/store"
+)
+
+// one-shot foreign-flow backfill for the top-N universe tickers (by close),
+// foreign-only (1 credit each) so the dashboard chart picker has data for
+// hundreds of tickers immediately. Usage:
+// BWS_PROJECT_ID=... sh scripts/bws-run.sh go run ./cmd/foreign-backfill <N>
+func main() {
+	n := 400
+	if len(os.Args) > 1 {
+		if v := os.Args[1]; v != "" {
+			var t int
+			if _, err := fmtSscan(v, &t); err == nil && t > 0 {
+				n = t
+			}
+		}
+	}
+	db, err := store.Open(os.Getenv("DB_PATH"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	c := sectors.New("https://api.sectors.app/v2/", os.Getenv("SECTORS_API_KEY"))
+	ctx, cancel := context.WithTimeout(context.Background(), 400*time.Second)
+	defer cancel()
+	c.OnSpend(func(endpoint string, calls, credits int) { logf("spend %d credits %s\n", credits, endpoint) })
+
+	rows, err := db.Query(`SELECT ticker FROM universe ORDER BY close DESC LIMIT ?`, n)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var tickers []string
+	for rows.Next() {
+		var t string
+		if err := rows.Scan(&t); err != nil {
+			log.Fatal(err)
+		}
+		tickers = append(tickers, t)
+	}
+	rows.Close()
+	logf("backfilling foreign flow for %d tickers\n", len(tickers))
+
+	done := 0
+	now := time.Now().UTC()
+	end := now.Format("2006-01-02")
+	start := now.AddDate(0, 0, -30).Format("2006-01-02")
+	for _, t := range tickers {
+		ff, err := c.ForeignFlow(ctx, t, start, end)
+		if err != nil {
+			logf("  %s err: %v\n", t, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		if raw, err := json.Marshal(ff); err == nil && len(ff.Data) > 0 {
+			_ = db.SaveSnapshot(t, end, "foreign-flow", string(raw))
+			for _, d := range ff.Data {
+				_ = db.UpsertForeignFlow(t, d.Date, float64(d.NetForeignInflow))
+			}
+			done++
+		}
+		time.Sleep(1200 * time.Millisecond) // stay under the 429 pace
+	}
+	logf("done: %d/%d tickers have foreign data now\n", done, len(tickers))
+}
+
+func logf(f string, a ...any) { log.Printf(f, a...) }
+
+func fmtSscan(s string, v *int) (int, error) {
+	n := 0
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			return 0, errBad
+		}
+		n = n*10 + int(r-'0')
+	}
+	*v = n
+	return 1, nil
+}
+
+var errBad = errT("bad number")
+
+type errT string
+
+func (e errT) Error() string { return string(e) }

backend/internal/api/flow.go

@@ -69,12 +69,10 @@ func (s *Server) FlowSummary(w http.ResponseWriter, r *http.Request) {
 	}
 	// Rest of the universe: still list them as chart-picker options (foreign
 	// data may exist even if not scanned this request).
-	if len(wl) > 60 {
-		extra, _ := s.DB.TickersWithForeign()
-		for _, tk := range wl {
-			if !scanned[tk] && extra[tk] {
-				fkTickers = append(fkTickers, tk)
-			}
+	extra, _ := s.DB.TickersWithForeign()
+	for _, tk := range wl {
+		if !scanned[tk] && extra[tk] {
+			fkTickers = append(fkTickers, tk)
 		}
 	}
 	sort.Slice(accs, func(i, j int) bool { return accs[i].NetSum > accs[j].NetSum })

backend/internal/scheduler/scheduler.go

@@ -324,9 +324,12 @@ func (s *Scheduler) market(ctx context.Context) error {
 }
 
 // tickerDepth pulls broker-summary/top + foreign-flow + daily per ticker.
+// Dates use time.Now().UTC() — the Sectors API rejects end dates in the
+// future relative to its UTC clock (WIB midnight > UTC previous day).
 func (s *Scheduler) tickerDepth(ctx context.Context, ticker string) error {
-	end := time.Now().Format("2006-01-02")
-	start5 := time.Now().AddDate(0, 0, -6).Format("2006-01-02")
+	now := time.Now().UTC()
+	end := now.Format("2006-01-02")
+	start5 := now.AddDate(0, 0, -6).Format("2006-01-02")
 	if top, err := s.Sectors.BrokerSummaryTop(ctx, ticker, start5, end, 10, "", ""); err == nil {
 		if raw, err := json.Marshal(top); err == nil {
 			_ = s.DB.SaveSnapshot(ticker, end, "broker-summary-top", string(raw))
@@ -343,7 +346,7 @@ func (s *Scheduler) tickerDepth(ctx context.Context, ticker string) error {
 		return err
 	}
 	if ff, err := s.Sectors.ForeignFlow(ctx, ticker,
-		time.Now().AddDate(0, 0, -7).Format("2006-01-02"), end); err == nil {
+		now.AddDate(0, 0, -7).Format("2006-01-02"), end); err == nil {
 		if raw, err := json.Marshal(ff); err == nil {
 			_ = s.DB.SaveSnapshot(ticker, end, "foreign-flow", string(raw))
 			for _, d := range ff.Data {