mirror of
https://github.com/ArvinLovegood/go-stock.git
synced 2025-07-19 00:00:09 +08:00
feat(settings):添加浏览器路径配置并优化爬虫功能
- 在前端和后端的设置中添加浏览器路径配置项 - 修改爬虫相关函数,使用配置的浏览器路径替代自动检测 - 优化日志输出,统一使用"BrowserPath"字段 - 重构部分代码,提高可维护性
This commit is contained in:
parent
1d4ede336c
commit
4c3fa36d4f
@ -31,9 +31,9 @@ func (c *CrawlerApi) NewCrawler(ctx context.Context, crawlerBaseInfo CrawlerBase
|
||||
|
||||
func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bool) {
|
||||
htmlContent := ""
|
||||
path, e := checkBrowserOnWindows()
|
||||
logger.SugaredLogger.Infof("GetHtml path:%s", path)
|
||||
if e {
|
||||
path := getConfig().BrowserPath
|
||||
logger.SugaredLogger.Infof("Browser path:%s", path)
|
||||
if path != "" {
|
||||
pctx, pcancel := chromedp.NewExecAllocator(
|
||||
c.crawlerCtx,
|
||||
chromedp.ExecPath(path),
|
||||
@ -92,14 +92,14 @@ func (c *CrawlerApi) GetHtml(url, waitVisible string, headless bool) (string, bo
|
||||
|
||||
func (c *CrawlerApi) GetHtmlWithNoCancel(url, waitVisible string, headless bool) (html string, ok bool, parent context.CancelFunc, child context.CancelFunc) {
|
||||
htmlContent := ""
|
||||
path, e := checkBrowserOnWindows()
|
||||
logger.SugaredLogger.Infof("GetHtml path:%s", path)
|
||||
path := getConfig().BrowserPath
|
||||
logger.SugaredLogger.Infof("BrowserPath :%s", path)
|
||||
var parentCancel context.CancelFunc
|
||||
var childCancel context.CancelFunc
|
||||
var pctx context.Context
|
||||
var cctx context.Context
|
||||
|
||||
if e {
|
||||
if path != "" {
|
||||
pctx, parentCancel = chromedp.NewExecAllocator(
|
||||
c.crawlerCtx,
|
||||
chromedp.ExecPath(path),
|
||||
@ -160,9 +160,9 @@ func (c *CrawlerApi) GetHtmlWithActions(actions *[]chromedp.Action, headless boo
|
||||
htmlContent := ""
|
||||
*actions = append(*actions, chromedp.InnerHTML("body", &htmlContent))
|
||||
|
||||
path, e := checkBrowserOnWindows()
|
||||
path := getConfig().BrowserPath
|
||||
logger.SugaredLogger.Infof("GetHtmlWithActions path:%s", path)
|
||||
if e {
|
||||
if path != "" {
|
||||
pctx, pcancel := chromedp.NewExecAllocator(
|
||||
c.crawlerCtx,
|
||||
chromedp.ExecPath(path),
|
||||
|
@ -34,6 +34,7 @@ type OpenAi struct {
|
||||
QuestionTemplate string `json:"question_template"`
|
||||
CrawlTimeOut int64 `json:"crawl_time_out"`
|
||||
KDays int64 `json:"kDays"`
|
||||
BrowserPath string `json:"browser_path"`
|
||||
}
|
||||
|
||||
func NewDeepSeekOpenAi(ctx context.Context) *OpenAi {
|
||||
@ -61,6 +62,7 @@ func NewDeepSeekOpenAi(ctx context.Context) *OpenAi {
|
||||
QuestionTemplate: config.QuestionTemplate,
|
||||
CrawlTimeOut: config.CrawlTimeOut,
|
||||
KDays: config.KDays,
|
||||
BrowserPath: config.BrowserPath,
|
||||
}
|
||||
}
|
||||
|
||||
@ -500,10 +502,10 @@ func GetFinancialReports(stockCode string, crawlTimeOut int64) *[]string {
|
||||
defer timeoutCtxCancel()
|
||||
var ctx context.Context
|
||||
var cancel context.CancelFunc
|
||||
path, e := checkBrowserOnWindows()
|
||||
path := getConfig().BrowserPath
|
||||
logger.SugaredLogger.Infof("GetFinancialReports path:%s", path)
|
||||
|
||||
if e {
|
||||
if path != "" {
|
||||
pctx, pcancel := chromedp.NewExecAllocator(
|
||||
timeoutCtx,
|
||||
chromedp.ExecPath(path),
|
||||
|
@ -9,7 +9,7 @@ import (
|
||||
func TestNewDeepSeekOpenAiConfig(t *testing.T) {
|
||||
db.Init("../../data/stock.db")
|
||||
ai := NewDeepSeekOpenAi(context.TODO())
|
||||
res := ai.NewChatStream("北京文化", "sz000802", "")
|
||||
res := ai.NewChatStream("上海贝岭", "sh600171", "分析以上股票资金流入信息,找出适合买入的股票,给出具体操作建议")
|
||||
for {
|
||||
select {
|
||||
case msg := <-res:
|
||||
|
@ -29,6 +29,7 @@ type Settings struct {
|
||||
CrawlTimeOut int64 `json:"crawlTimeOut"`
|
||||
KDays int64 `json:"kDays"`
|
||||
EnableDanmu bool `json:"enableDanmu"`
|
||||
BrowserPath string `json:"browserPath"`
|
||||
}
|
||||
|
||||
func (receiver Settings) TableName() string {
|
||||
@ -69,6 +70,7 @@ func (s SettingsApi) UpdateConfig() string {
|
||||
"crawl_time_out": s.Config.CrawlTimeOut,
|
||||
"k_days": s.Config.KDays,
|
||||
"enable_danmu": s.Config.EnableDanmu,
|
||||
"browser_path": s.Config.BrowserPath,
|
||||
})
|
||||
} else {
|
||||
logger.SugaredLogger.Infof("未找到配置,创建默认配置:%+v", s.Config)
|
||||
@ -92,6 +94,7 @@ func (s SettingsApi) UpdateConfig() string {
|
||||
CrawlTimeOut: s.Config.CrawlTimeOut,
|
||||
KDays: s.Config.KDays,
|
||||
EnableDanmu: s.Config.EnableDanmu,
|
||||
BrowserPath: s.Config.BrowserPath,
|
||||
})
|
||||
}
|
||||
return "保存成功!"
|
||||
@ -111,6 +114,10 @@ func (s SettingsApi) GetConfig() *Settings {
|
||||
settings.KDays = 120
|
||||
}
|
||||
}
|
||||
if settings.BrowserPath == "" {
|
||||
settings.BrowserPath, _ = CheckBrowserOnWindows()
|
||||
}
|
||||
|
||||
return &settings
|
||||
}
|
||||
|
||||
|
@ -872,9 +872,9 @@ func getSHSZStockPriceInfo(stockCode string, crawlTimeOut int64) *[]string {
|
||||
defer timeoutCtxCancel()
|
||||
var ctx context.Context
|
||||
var cancel context.CancelFunc
|
||||
path, e := checkBrowserOnWindows()
|
||||
logger.SugaredLogger.Infof("SearchStockPriceInfo path:%s", path)
|
||||
if e {
|
||||
path := getConfig().BrowserPath
|
||||
logger.SugaredLogger.Infof("SearchStockPriceInfo BrowserPath:%s", path)
|
||||
if path != "" {
|
||||
pctx, pcancel := chromedp.NewExecAllocator(
|
||||
timeoutCtx,
|
||||
chromedp.ExecPath(path),
|
||||
@ -1042,8 +1042,8 @@ func checkChromeOnWindows() (string, bool) {
|
||||
return path + "\\chrome.exe", true
|
||||
}
|
||||
|
||||
// checkBrowserOnWindows 在 Windows 系统上检查Edge浏览器是否安装,并返回安装路径
|
||||
func checkBrowserOnWindows() (string, bool) {
|
||||
// CheckBrowserOnWindows 在 Windows 系统上检查Edge浏览器是否安装,并返回安装路径
|
||||
func CheckBrowserOnWindows() (string, bool) {
|
||||
if path, ok := checkChromeOnWindows(); ok {
|
||||
return path, true
|
||||
}
|
||||
|
@ -34,6 +34,7 @@ const formValue = ref({
|
||||
kDays:30,
|
||||
},
|
||||
enableDanmu:false,
|
||||
browserPath: '',
|
||||
})
|
||||
|
||||
onMounted(()=>{
|
||||
@ -63,6 +64,7 @@ onMounted(()=>{
|
||||
kDays:res.kDays,
|
||||
}
|
||||
formValue.value.enableDanmu = res.enableDanmu
|
||||
formValue.value.browserPath = res.browserPath
|
||||
console.log(res)
|
||||
})
|
||||
//message.info("加载完成")
|
||||
@ -89,7 +91,8 @@ function saveConfig(){
|
||||
questionTemplate:formValue.value.openAI.questionTemplate,
|
||||
crawlTimeOut:formValue.value.openAI.crawlTimeOut,
|
||||
kDays:formValue.value.openAI.kDays,
|
||||
enableDanmu:formValue.value.enableDanmu
|
||||
enableDanmu:formValue.value.enableDanmu,
|
||||
browserPath:formValue.value.browserPath
|
||||
})
|
||||
|
||||
//console.log("Settings",config)
|
||||
@ -161,6 +164,7 @@ function importConfig(){
|
||||
kDays:config.kDays
|
||||
}
|
||||
formValue.value.enableDanmu = config.enableDanmu
|
||||
formValue.value.browserPath = config.browserPath
|
||||
// formRef.value.resetFields()
|
||||
};
|
||||
reader.readAsText(file);
|
||||
@ -205,6 +209,9 @@ window.onerror = function (event, source, lineno, colno, error) {
|
||||
</template>
|
||||
</n-input-number>
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="22" label="浏览器路径:" path="browserPath" >
|
||||
<n-input type="text" placeholder="浏览器路径" v-model:value="formValue.browserPath" clearable />
|
||||
</n-form-item-gi>
|
||||
</n-grid>
|
||||
|
||||
<n-grid :cols="24" :x-gap="24" style="text-align: left">
|
||||
|
@ -170,6 +170,7 @@ export namespace data {
|
||||
crawlTimeOut: number;
|
||||
kDays: number;
|
||||
enableDanmu: boolean;
|
||||
browserPath: string;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new Settings(source);
|
||||
@ -200,6 +201,7 @@ export namespace data {
|
||||
this.crawlTimeOut = source["crawlTimeOut"];
|
||||
this.kDays = source["kDays"];
|
||||
this.enableDanmu = source["enableDanmu"];
|
||||
this.browserPath = source["browserPath"];
|
||||
}
|
||||
|
||||
convertValues(a: any, classs: any, asMap: boolean = false): any {
|
||||
|
Loading…
x
Reference in New Issue
Block a user