diff --git a/backend/data/alert_darwin_api.go b/backend/data/alert_darwin_api.go index fc226a7..e40dd8a 100644 --- a/backend/data/alert_darwin_api.go +++ b/backend/data/alert_darwin_api.go @@ -34,7 +34,7 @@ func NewAlertWindowsApi(AppID string, Title string, Content string, Icon string) } func (a AlertWindowsApi) SendNotification() bool { - if getConfig().LocalPushEnable == false { + if GetConfig().LocalPushEnable == false { logger.SugaredLogger.Error("本地推送未开启") return false } diff --git a/backend/data/alert_windows_api.go b/backend/data/alert_windows_api.go index 76ff0e3..d0b468f 100644 --- a/backend/data/alert_windows_api.go +++ b/backend/data/alert_windows_api.go @@ -31,7 +31,7 @@ func NewAlertWindowsApi(AppID string, Title string, Content string, Icon string) } func (a AlertWindowsApi) SendNotification() bool { - if getConfig().LocalPushEnable == false { + if GetConfig().LocalPushEnable == false { logger.SugaredLogger.Error("本地推送未开启") return false } diff --git a/backend/data/crawler_api.go b/backend/data/crawler_api.go index 41dc1e0..90d83f4 100644 --- a/backend/data/crawler_api.go +++ b/backend/data/crawler_api.go @@ -15,6 +15,7 @@ import ( type CrawlerApi struct { crawlerCtx context.Context crawlerBaseInfo CrawlerBaseInfo + pool *BrowserPool } func (c *CrawlerApi) NewTimeOutCrawler(timeout int, crawlerBaseInfo CrawlerBaseInfo) CrawlerApi { @@ -26,12 +27,19 @@ func (c *CrawlerApi) NewCrawler(ctx context.Context, crawlerBaseInfo CrawlerBase return CrawlerApi{ crawlerCtx: ctx, crawlerBaseInfo: crawlerBaseInfo, + pool: NewBrowserPool(GetConfig().BrowserPoolSize), } } - func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bool) { + page, err := c.pool.FetchPage(url, waitVisible) + if err != nil { + return "", false + } + return page, true +} +func (c *CrawlerApi) GetHtml_old(url, waitVisible string, headless bool) (string, bool) { htmlContent := "" - path := getConfig().BrowserPath + path := GetConfig().BrowserPath //logger.SugaredLogger.Infof("Browser path:%s", path) if path != "" { pctx, pcancel := chromedp.NewExecAllocator( @@ -94,7 +102,7 @@ func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bo func (c *CrawlerApi) GetHtmlWithNoCancel(url, waitVisible string, headless bool) (html string, ok bool, parent context.CancelFunc, child context.CancelFunc) { htmlContent := "" - path := getConfig().BrowserPath + path := GetConfig().BrowserPath //logger.SugaredLogger.Infof("BrowserPath :%s", path) var parentCancel context.CancelFunc var childCancel context.CancelFunc @@ -162,7 +170,7 @@ func (c *CrawlerApi) GetHtmlWithActions(actions *[]chromedp.Action, headless boo htmlContent := "" *actions = append(*actions, chromedp.InnerHTML("body", &htmlContent)) - path := getConfig().BrowserPath + path := GetConfig().BrowserPath //logger.SugaredLogger.Infof("GetHtmlWithActions path:%s", path) if path != "" { pctx, pcancel := chromedp.NewExecAllocator( diff --git a/backend/data/dingding_api.go b/backend/data/dingding_api.go index 60c3afa..37b4d7a 100644 --- a/backend/data/dingding_api.go +++ b/backend/data/dingding_api.go @@ -21,7 +21,7 @@ func NewDingDingAPI() *DingDingAPI { } func (DingDingAPI) SendDingDingMessage(message string) string { - if getConfig().DingPushEnable == false { + if GetConfig().DingPushEnable == false { //logger.SugaredLogger.Info("钉钉推送未开启") return "钉钉推送未开启" } @@ -37,11 +37,11 @@ func (DingDingAPI) SendDingDingMessage(message string) string { logger.SugaredLogger.Infof("send dingding message: %s", resp.String()) return "发送钉钉消息成功" } -func getConfig() *Settings { +func GetConfig() *Settings { return NewSettingsApi(&Settings{}).GetConfig() } func getApiURL() string { - return getConfig().DingRobot + return GetConfig().DingRobot } func (DingDingAPI) SendToDingDing(title, message string) string { diff --git a/backend/data/fund_data_api.go b/backend/data/fund_data_api.go index 7a649f6..dc652ab 100644 --- a/backend/data/fund_data_api.go +++ b/backend/data/fund_data_api.go @@ -26,7 +26,7 @@ type FundApi struct { func NewFundApi() *FundApi { return &FundApi{ client: resty.New(), - config: getConfig(), + config: GetConfig(), } } diff --git a/backend/data/openai_api.go b/backend/data/openai_api.go index a021b98..d6d3be8 100644 --- a/backend/data/openai_api.go +++ b/backend/data/openai_api.go @@ -38,7 +38,7 @@ type OpenAi struct { } func NewDeepSeekOpenAi(ctx context.Context) *OpenAi { - config := getConfig() + config := GetConfig() if config.OpenAiEnable { if config.OpenAiApiTimeOut <= 0 { config.OpenAiApiTimeOut = 60 * 5 @@ -186,7 +186,7 @@ func (o OpenAi) NewChatStream(stock, stockCode, userQuestion string, sysPromptId if strutil.HasPrefixAny(stockCode, []string{"hk", "sz", "sh"}) { code = ConvertStockCodeToTushareCode(stockCode) } - K := NewTushareApi(getConfig()).GetDaily(code, startDate, endDate, o.CrawlTimeOut) + K := NewTushareApi(GetConfig()).GetDaily(code, startDate, endDate, o.CrawlTimeOut) msg = append(msg, map[string]interface{}{ "role": "user", "content": stock + "日K数据", diff --git a/backend/data/pool.go b/backend/data/pool.go new file mode 100644 index 0000000..8f7a59a --- /dev/null +++ b/backend/data/pool.go @@ -0,0 +1,115 @@ +package data + +import ( + "context" + "go-stock/backend/logger" + "sync" + "time" + + "github.com/chromedp/chromedp" +) + +// BrowserPool 浏览器池结构 +type BrowserPool struct { + pool chan *context.Context + mu sync.Mutex + size int +} + +// NewBrowserPool 创建新的浏览器池 +func NewBrowserPool(size int) *BrowserPool { + pool := make(chan *context.Context, size) + for i := 0; i < size; i++ { + path := GetConfig().BrowserPath + crawlTimeOut := GetConfig().CrawlTimeOut + if crawlTimeOut < 15 { + crawlTimeOut = 30 + } + if path != "" { + ctx, _ := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second) + ctx, _ = chromedp.NewExecAllocator( + ctx, + chromedp.ExecPath(path), + chromedp.Flag("headless", true), + chromedp.Flag("blink-settings", "imagesEnabled=false"), + chromedp.Flag("disable-javascript", false), + chromedp.Flag("disable-gpu", true), + //chromedp.UserAgent(""), + chromedp.Flag("disable-background-networking", true), + chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"), + chromedp.Flag("disable-background-timer-throttling", true), + chromedp.Flag("disable-backgrounding-occluded-windows", true), + chromedp.Flag("disable-breakpad", true), + chromedp.Flag("disable-client-side-phishing-detection", true), + chromedp.Flag("disable-default-apps", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("disable-extensions", true), + chromedp.Flag("disable-features", "site-per-process,Translate,BlinkGenPropertyTrees"), + chromedp.Flag("disable-hang-monitor", true), + chromedp.Flag("disable-ipc-flooding-protection", true), + chromedp.Flag("disable-popup-blocking", true), + chromedp.Flag("disable-prompt-on-repost", true), + chromedp.Flag("disable-renderer-backgrounding", true), + chromedp.Flag("disable-sync", true), + chromedp.Flag("force-color-profile", "srgb"), + chromedp.Flag("metrics-recording-only", true), + chromedp.Flag("safebrowsing-disable-auto-update", true), + chromedp.Flag("enable-automation", true), + chromedp.Flag("password-store", "basic"), + chromedp.Flag("use-mock-keychain", true), + ) + ctx, _ = chromedp.NewContext(ctx, chromedp.WithLogf(logger.SugaredLogger.Infof)) + pool <- &ctx + } + } + return &BrowserPool{ + pool: pool, + size: size, + } +} + +// Get 从池中获取浏览器实例 +func (pool *BrowserPool) Get() *context.Context { + return <-pool.pool +} + +// Put 将浏览器实例放回池中 +func (pool *BrowserPool) Put(ctx *context.Context) { + pool.mu.Lock() + defer pool.mu.Unlock() + // 检查池是否已满 + if len(pool.pool) >= pool.size { + // 池已满，关闭并丢弃这个实例 + chromedp.Cancel(*ctx) + return + } + chromedp.Cancel(*ctx) + pool.pool <- ctx +} + +// Close 关闭池中的所有浏览器实例 +func (pool *BrowserPool) Close() { + close(pool.pool) + for ctx := range pool.pool { + chromedp.Cancel(*ctx) + } +} + +// FetchPage 使用浏览器池获取页面内容 +func (pool *BrowserPool) FetchPage(url, waitVisible string) (string, error) { + // 从池中获取浏览器实例 + ctx := pool.Get() + defer pool.Put(ctx) // 使用完毕后放回池中 + var htmlContent string + err := chromedp.Run(*ctx, + chromedp.Navigate(url), + chromedp.WaitVisible(waitVisible, chromedp.ByQuery), // 确保 元素可见 + chromedp.WaitReady(waitVisible, chromedp.ByQuery), // 确保 元素准备好 + chromedp.InnerHTML("body", &htmlContent), + chromedp.Evaluate(`window.close()`, nil), + ) + if err != nil { + return "", err + } + return htmlContent, nil +} diff --git a/backend/data/pool_test.go b/backend/data/pool_test.go new file mode 100644 index 0000000..8b4c27b --- /dev/null +++ b/backend/data/pool_test.go @@ -0,0 +1,18 @@ +package data + +import ( + "go-stock/backend/db" + "testing" +) + +func TestPool(t *testing.T) { + db.Init("../../data/stock.db") + + pool := NewBrowserPool(1) + go pool.FetchPage("https://fund.eastmoney.com/016533.html", "body") + go pool.FetchPage("https://fund.eastmoney.com/217021.html", "body") + go pool.FetchPage("https://fund.eastmoney.com/001125.html", "body") + + select {} + +} diff --git a/backend/data/settings_api.go b/backend/data/settings_api.go index 098701f..62ef945 100644 --- a/backend/data/settings_api.go +++ b/backend/data/settings_api.go @@ -32,6 +32,7 @@ type Settings struct { BrowserPath string `json:"browserPath"` EnableNews bool `json:"enableNews"` DarkTheme bool `json:"darkTheme"` + BrowserPoolSize int `json:"browserPoolSize"` } func (receiver Settings) TableName() string { @@ -123,7 +124,9 @@ func (s SettingsApi) GetConfig() *Settings { if settings.BrowserPath == "" { settings.BrowserPath, _ = CheckBrowserOnWindows() } - + if settings.BrowserPoolSize <= 0 { + settings.BrowserPoolSize = 3 + } return &settings } diff --git a/backend/data/stock_data_api.go b/backend/data/stock_data_api.go index 3cfc23a..ea40f06 100644 --- a/backend/data/stock_data_api.go +++ b/backend/data/stock_data_api.go @@ -188,7 +188,7 @@ func (receiver StockBasic) TableName() string { func NewStockDataApi() *StockDataApi { return &StockDataApi{ client: resty.New(), - config: getConfig(), + config: GetConfig(), } } diff --git a/backend/data/tushare_data_api_test.go b/backend/data/tushare_data_api_test.go index 627e1bf..c68320b 100644 --- a/backend/data/tushare_data_api_test.go +++ b/backend/data/tushare_data_api_test.go @@ -11,7 +11,7 @@ import ( // ----------------------------------------------------------------------------------- func TestGetDaily(t *testing.T) { db.Init("../../data/stock.db") - tushareApi := NewTushareApi(getConfig()) + tushareApi := NewTushareApi(GetConfig()) res := tushareApi.GetDaily("00927.HK", "20250101", "20250217", 30) t.Log(res) @@ -19,7 +19,7 @@ func TestGetDaily(t *testing.T) { func TestGetUSDaily(t *testing.T) { db.Init("../../data/stock.db") - tushareApi := NewTushareApi(getConfig()) + tushareApi := NewTushareApi(GetConfig()) res := tushareApi.GetDaily("gb_AAPL", "20250101", "20250217", 30) t.Log(res) diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 23f2c99..cf15b91 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -228,6 +228,7 @@ export namespace data { browserPath: string; enableNews: boolean; darkTheme: boolean; + browserPoolSize: number; static createFrom(source: any = {}) { return new Settings(source); @@ -261,6 +262,7 @@ export namespace data { this.browserPath = source["browserPath"]; this.enableNews = source["enableNews"]; this.darkTheme = source["darkTheme"]; + this.browserPoolSize = source["browserPoolSize"]; } convertValues(a: any, classs: any, asMap: boolean = false): any {