third_codes/go-stock
1
0
Fork 0
mirror of https://github.com/ArvinLovegood/go-stock.git synced 2025-07-19 00:00:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(data):实现浏览器实例池化

Browse Source 
- 新增 BrowserPool 结构和相关方法，用于管理和复用浏览器实例
- 在 CrawlerApi 中集成浏览器池，使用 FetchPage 方法获取页面内容
-优化了配置获取方式，统一使用 GetConfig() 函数
-修复了一些代码中的小问题，如错误处理和日志记录
This commit is contained in:
ArvinLovegood 2025-03-31 23:08:09 +08:00
parent 51aae0539c
commit 3402f0d296
12 changed files with 162 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/data/alert_darwin_api.go
View File

@ -34,7 +34,7 @@ func NewAlertWindowsApi(AppID string, Title string, Content string, Icon string)
}
func (a AlertWindowsApi) SendNotification() bool {
if getConfig().LocalPushEnable == false {
if GetConfig().LocalPushEnable == false {
logger.SugaredLogger.Error("本地推送未开启")
return false
}

2
backend/data/alert_windows_api.go
View File

@ -31,7 +31,7 @@ func NewAlertWindowsApi(AppID string, Title string, Content string, Icon string)
}
func (a AlertWindowsApi) SendNotification() bool {
if getConfig().LocalPushEnable == false {
if GetConfig().LocalPushEnable == false {
logger.SugaredLogger.Error("本地推送未开启")
return false
}

16
backend/data/crawler_api.go
View File

@ -15,6 +15,7 @@ import (
type CrawlerApi struct {
crawlerCtx context.Context
crawlerBaseInfo CrawlerBaseInfo
pool *BrowserPool
}
func (c *CrawlerApi) NewTimeOutCrawler(timeout int, crawlerBaseInfo CrawlerBaseInfo) CrawlerApi {
@ -26,12 +27,19 @@ func (c *CrawlerApi) NewCrawler(ctx context.Context, crawlerBaseInfo CrawlerBase
return CrawlerApi{
crawlerCtx: ctx,
crawlerBaseInfo: crawlerBaseInfo,
pool: NewBrowserPool(GetConfig().BrowserPoolSize),
}
}
func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bool) {
page, err := c.pool.FetchPage(url, waitVisible)
if err != nil {
return "", false
}
return page, true
}
func (c *CrawlerApi) GetHtml_old(url, waitVisible string, headless bool) (string, bool) {
htmlContent := ""
path := getConfig().BrowserPath
path := GetConfig().BrowserPath
//logger.SugaredLogger.Infof("Browser path:%s", path)
if path != "" {
pctx, pcancel := chromedp.NewExecAllocator(
@ -94,7 +102,7 @@ func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bo
func (c *CrawlerApi) GetHtmlWithNoCancel(url, waitVisible string, headless bool) (html string, ok bool, parent context.CancelFunc, child context.CancelFunc) {
htmlContent := ""
path := getConfig().BrowserPath
path := GetConfig().BrowserPath
//logger.SugaredLogger.Infof("BrowserPath :%s", path)
var parentCancel context.CancelFunc
var childCancel context.CancelFunc
@ -162,7 +170,7 @@ func (c *CrawlerApi) GetHtmlWithActions(actions *[]chromedp.Action, headless boo
htmlContent := ""
*actions = append(*actions, chromedp.InnerHTML("body", &htmlContent))
path := getConfig().BrowserPath
path := GetConfig().BrowserPath
//logger.SugaredLogger.Infof("GetHtmlWithActions path:%s", path)
if path != "" {
pctx, pcancel := chromedp.NewExecAllocator(

6
backend/data/dingding_api.go
View File

@ -21,7 +21,7 @@ func NewDingDingAPI() *DingDingAPI {
}
func (DingDingAPI) SendDingDingMessage(message string) string {
if getConfig().DingPushEnable == false {
if GetConfig().DingPushEnable == false {
//logger.SugaredLogger.Info("钉钉推送未开启")
return "钉钉推送未开启"
}
@ -37,11 +37,11 @@ func (DingDingAPI) SendDingDingMessage(message string) string {
logger.SugaredLogger.Infof("send dingding message: %s", resp.String())
return "发送钉钉消息成功"
}
func getConfig() *Settings {
func GetConfig() *Settings {
return NewSettingsApi(&Settings{}).GetConfig()
}
func getApiURL() string {
return getConfig().DingRobot
return GetConfig().DingRobot
}
func (DingDingAPI) SendToDingDing(title, message string) string {

2
backend/data/fund_data_api.go
View File

@ -26,7 +26,7 @@ type FundApi struct {
func NewFundApi() *FundApi {
return &FundApi{
client: resty.New(),
config: getConfig(),
config: GetConfig(),
}
}

4
backend/data/openai_api.go
View File

@ -38,7 +38,7 @@ type OpenAi struct {
}
func NewDeepSeekOpenAi(ctx context.Context) *OpenAi {
config := getConfig()
config := GetConfig()
if config.OpenAiEnable {
if config.OpenAiApiTimeOut <= 0 {
config.OpenAiApiTimeOut = 60 * 5
@ -186,7 +186,7 @@ func (o OpenAi) NewChatStream(stock, stockCode, userQuestion string, sysPromptId
if strutil.HasPrefixAny(stockCode, []string{"hk", "sz", "sh"}) {
code = ConvertStockCodeToTushareCode(stockCode)
}
K := NewTushareApi(getConfig()).GetDaily(code, startDate, endDate, o.CrawlTimeOut)
K := NewTushareApi(GetConfig()).GetDaily(code, startDate, endDate, o.CrawlTimeOut)
msg = append(msg, map[string]interface{}{
"role": "user",
"content": stock + "日K数据",

115
backend/data/pool.go Normal file
View File

@ -0,0 +1,115 @@
package data
import (
"context"
"go-stock/backend/logger"
"sync"
"time"
"github.com/chromedp/chromedp"
)
// BrowserPool 浏览器池结构
type BrowserPool struct {
pool chan *context.Context
mu sync.Mutex
size int
}
// NewBrowserPool 创建新的浏览器池
func NewBrowserPool(size int) *BrowserPool {
pool := make(chan *context.Context, size)
for i := 0; i < size; i++ {
path := GetConfig().BrowserPath
crawlTimeOut := GetConfig().CrawlTimeOut
if crawlTimeOut < 15 {
crawlTimeOut = 30
}
if path != "" {
ctx, _ := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second)
ctx, _ = chromedp.NewExecAllocator(
ctx,
chromedp.ExecPath(path),
chromedp.Flag("headless", true),
chromedp.Flag("blink-settings", "imagesEnabled=false"),
chromedp.Flag("disable-javascript", false),
chromedp.Flag("disable-gpu", true),
//chromedp.UserAgent(""),
chromedp.Flag("disable-background-networking", true),
chromedp.Flag("enable-features", "NetworkService,NetworkServiceInProcess"),
chromedp.Flag("disable-background-timer-throttling", true),
chromedp.Flag("disable-backgrounding-occluded-windows", true),
chromedp.Flag("disable-breakpad", true),
chromedp.Flag("disable-client-side-phishing-detection", true),
chromedp.Flag("disable-default-apps", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-extensions", true),
chromedp.Flag("disable-features", "site-per-process,Translate,BlinkGenPropertyTrees"),
chromedp.Flag("disable-hang-monitor", true),
chromedp.Flag("disable-ipc-flooding-protection", true),
chromedp.Flag("disable-popup-blocking", true),
chromedp.Flag("disable-prompt-on-repost", true),
chromedp.Flag("disable-renderer-backgrounding", true),
chromedp.Flag("disable-sync", true),
chromedp.Flag("force-color-profile", "srgb"),
chromedp.Flag("metrics-recording-only", true),
chromedp.Flag("safebrowsing-disable-auto-update", true),
chromedp.Flag("enable-automation", true),
chromedp.Flag("password-store", "basic"),
chromedp.Flag("use-mock-keychain", true),
)
ctx, _ = chromedp.NewContext(ctx, chromedp.WithLogf(logger.SugaredLogger.Infof))
pool <- &ctx
}
}
return &BrowserPool{
pool: pool,
size: size,
}
}
// Get 从池中获取浏览器实例
func (pool *BrowserPool) Get() *context.Context {
return <-pool.pool
}
// Put 将浏览器实例放回池中
func (pool *BrowserPool) Put(ctx *context.Context) {
pool.mu.Lock()
defer pool.mu.Unlock()
// 检查池是否已满
if len(pool.pool) >= pool.size {
// 池已满，关闭并丢弃这个实例
chromedp.Cancel(*ctx)
return
}
chromedp.Cancel(*ctx)
pool.pool <- ctx
}
// Close 关闭池中的所有浏览器实例
func (pool *BrowserPool) Close() {
close(pool.pool)
for ctx := range pool.pool {
chromedp.Cancel(*ctx)
}
}
// FetchPage 使用浏览器池获取页面内容
func (pool *BrowserPool) FetchPage(url, waitVisible string) (string, error) {
// 从池中获取浏览器实例
ctx := pool.Get()
defer pool.Put(ctx) // 使用完毕后放回池中
var htmlContent string
err := chromedp.Run(*ctx,
chromedp.Navigate(url),
chromedp.WaitVisible(waitVisible, chromedp.ByQuery), // 确保 元素可见
chromedp.WaitReady(waitVisible, chromedp.ByQuery), // 确保 元素准备好
chromedp.InnerHTML("body", &htmlContent),
chromedp.Evaluate(`window.close()`, nil),
)
if err != nil {
return "", err
}
return htmlContent, nil
}

18
backend/data/pool_test.go Normal file
View File

@ -0,0 +1,18 @@
package data
import (
"go-stock/backend/db"
"testing"
)
func TestPool(t *testing.T) {
db.Init("../../data/stock.db")
pool := NewBrowserPool(1)
go pool.FetchPage("https://fund.eastmoney.com/016533.html", "body")
go pool.FetchPage("https://fund.eastmoney.com/217021.html", "body")
go pool.FetchPage("https://fund.eastmoney.com/001125.html", "body")
select {}
}

5
backend/data/settings_api.go
View File

@ -32,6 +32,7 @@ type Settings struct {
BrowserPath string `json:"browserPath"`
EnableNews bool `json:"enableNews"`
DarkTheme bool `json:"darkTheme"`
BrowserPoolSize int `json:"browserPoolSize"`
}
func (receiver Settings) TableName() string {
@ -123,7 +124,9 @@ func (s SettingsApi) GetConfig() *Settings {
if settings.BrowserPath == "" {
settings.BrowserPath, _ = CheckBrowserOnWindows()
}
if settings.BrowserPoolSize <= 0 {
settings.BrowserPoolSize = 3
}
return &settings
}

2
backend/data/stock_data_api.go
View File

@ -188,7 +188,7 @@ func (receiver StockBasic) TableName() string {
func NewStockDataApi() *StockDataApi {
return &StockDataApi{
client: resty.New(),
config: getConfig(),
config: GetConfig(),
}
}

4
backend/data/tushare_data_api_test.go
View File

@ -11,7 +11,7 @@ import (
// -----------------------------------------------------------------------------------
func TestGetDaily(t *testing.T) {
db.Init("../../data/stock.db")
tushareApi := NewTushareApi(getConfig())
tushareApi := NewTushareApi(GetConfig())
res := tushareApi.GetDaily("00927.HK", "20250101", "20250217", 30)
t.Log(res)
@ -19,7 +19,7 @@ func TestGetDaily(t *testing.T) {
func TestGetUSDaily(t *testing.T) {
db.Init("../../data/stock.db")
tushareApi := NewTushareApi(getConfig())
tushareApi := NewTushareApi(GetConfig())
res := tushareApi.GetDaily("gb_AAPL", "20250101", "20250217", 30)
t.Log(res)

2
frontend/wailsjs/go/models.ts
View File

@ -228,6 +228,7 @@ export namespace data {
browserPath: string;
enableNews: boolean;
darkTheme: boolean;
browserPoolSize: number;
static createFrom(source: any = {}) {
return new Settings(source);
@ -261,6 +262,7 @@ export namespace data {
this.browserPath = source["browserPath"];
this.enableNews = source["enableNews"];
this.darkTheme = source["darkTheme"];
this.browserPoolSize = source["browserPoolSize"];
}
convertValues(a: any, classs: any, asMap: boolean = false): any {