feat(backend):添加资讯采集超时设置并优化相关功能

- 在 OpenAi 结构中添加 CrawlTimeOut 字段,用于设置资讯采集超时时间
- 修改相关函数以支持新的超时设置,包括 GetFinancialReports、GetTelegraphList、GetTopNewsList等
- 在前端设置页面添加 Crawler Timeout 设置项
- 优化浏览器检查逻辑,优先检查 Chrome 浏览器
This commit is contained in:
spark 2025-02-12 17:03:25 +08:00
parent d46872ffbd
commit d27bcbd334
6 changed files with 78 additions and 34 deletions

View File

@ -32,10 +32,19 @@ type OpenAi struct {
Prompt string `json:"prompt"` Prompt string `json:"prompt"`
TimeOut int `json:"time_out"` TimeOut int `json:"time_out"`
QuestionTemplate string `json:"question_template"` QuestionTemplate string `json:"question_template"`
CrawlTimeOut int64 `json:"crawl_time_out"`
} }
func NewDeepSeekOpenAi(ctx context.Context) *OpenAi { func NewDeepSeekOpenAi(ctx context.Context) *OpenAi {
config := getConfig() config := getConfig()
if config.OpenAiEnable {
if config.OpenAiApiTimeOut <= 0 {
config.OpenAiApiTimeOut = 60 * 5
}
if config.CrawlTimeOut <= 0 {
config.CrawlTimeOut = 60
}
}
return &OpenAi{ return &OpenAi{
ctx: ctx, ctx: ctx,
BaseUrl: config.OpenAiBaseUrl, BaseUrl: config.OpenAiBaseUrl,
@ -46,6 +55,7 @@ func NewDeepSeekOpenAi(ctx context.Context) *OpenAi {
Prompt: config.Prompt, Prompt: config.Prompt,
TimeOut: config.OpenAiApiTimeOut, TimeOut: config.OpenAiApiTimeOut,
QuestionTemplate: config.QuestionTemplate, QuestionTemplate: config.QuestionTemplate,
CrawlTimeOut: config.CrawlTimeOut,
} }
} }
@ -117,7 +127,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
wg.Add(5) wg.Add(5)
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := SearchStockPriceInfo(stockCode) messages := SearchStockPriceInfo(stockCode, o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取股票价格失败") logger.SugaredLogger.Error("获取股票价格失败")
ch <- "***❗获取股票价格失败,分析结果可能不准确***<hr>" ch <- "***❗获取股票价格失败,分析结果可能不准确***<hr>"
@ -136,7 +146,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := GetFinancialReports(stockCode) messages := GetFinancialReports(stockCode, o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取股票财报失败") logger.SugaredLogger.Error("获取股票财报失败")
ch <- "***❗获取股票财报失败,分析结果可能不准确***<hr>" ch <- "***❗获取股票财报失败,分析结果可能不准确***<hr>"
@ -153,7 +163,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := GetTelegraphList() messages := GetTelegraphList(o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取市场资讯失败") logger.SugaredLogger.Error("获取市场资讯失败")
ch <- "***❗获取市场资讯失败,分析结果可能不准确***<hr>" ch <- "***❗获取市场资讯失败,分析结果可能不准确***<hr>"
@ -166,7 +176,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
"content": message, "content": message,
}) })
} }
messages = GetTopNewsList() messages = GetTopNewsList(o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取新闻资讯失败") logger.SugaredLogger.Error("获取新闻资讯失败")
ch <- "***❗获取新闻资讯失败,分析结果可能不准确***<hr>" ch <- "***❗获取新闻资讯失败,分析结果可能不准确***<hr>"
@ -183,7 +193,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := SearchStockInfo(stock, "depth") messages := SearchStockInfo(stock, "depth", o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取股票资讯失败") logger.SugaredLogger.Error("获取股票资讯失败")
ch <- "***❗获取股票资讯失败,分析结果可能不准确***<hr>" ch <- "***❗获取股票资讯失败,分析结果可能不准确***<hr>"
@ -199,7 +209,7 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
}() }()
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := SearchStockInfo(stock, "telegram") messages := SearchStockInfo(stock, "telegram", o.CrawlTimeOut)
if messages == nil || len(*messages) == 0 { if messages == nil || len(*messages) == 0 {
logger.SugaredLogger.Error("获取股票电报资讯失败") logger.SugaredLogger.Error("获取股票电报资讯失败")
ch <- "***❗获取股票电报资讯失败,分析结果可能不准确***<hr>" ch <- "***❗获取股票电报资讯失败,分析结果可能不准确***<hr>"
@ -298,13 +308,13 @@ func (o OpenAi) NewChatStream(stock, stockCode string) <-chan string {
return ch return ch
} }
func GetFinancialReports(stockCode string) *[]string { func GetFinancialReports(stockCode string, crawlTimeOut int64) *[]string {
// 创建一个 chromedp 上下文 // 创建一个 chromedp 上下文
timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), 30*time.Second) timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second)
defer timeoutCtxCancel() defer timeoutCtxCancel()
var ctx context.Context var ctx context.Context
var cancel context.CancelFunc var cancel context.CancelFunc
path, e := checkEdgeOnWindows() path, e := checkBrowserOnWindows()
logger.SugaredLogger.Infof("GetFinancialReports path:%s", path) logger.SugaredLogger.Infof("GetFinancialReports path:%s", path)
if e { if e {
@ -388,7 +398,7 @@ func (o OpenAi) NewCommonChatStream(stock, stockCode, apiURL, apiKey, Model stri
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
messages := SearchStockPriceInfo(stockCode) messages := SearchStockPriceInfo(stockCode, o.CrawlTimeOut)
price := "" price := ""
for _, message := range *messages { for _, message := range *messages {
price += message + ";" price += message + ";"
@ -477,9 +487,9 @@ func (o OpenAi) NewCommonChatStream(stock, stockCode, apiURL, apiKey, Model stri
return ch return ch
} }
func GetTelegraphList() *[]string { func GetTelegraphList(crawlTimeOut int64) *[]string {
url := "https://www.cls.cn/telegraph" url := "https://www.cls.cn/telegraph"
response, err := resty.New().R(). response, err := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R().
SetHeader("Referer", "https://www.cls.cn/"). SetHeader("Referer", "https://www.cls.cn/").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(fmt.Sprintf(url)) Get(fmt.Sprintf(url))
@ -499,9 +509,9 @@ func GetTelegraphList() *[]string {
return &telegraph return &telegraph
} }
func GetTopNewsList() *[]string { func GetTopNewsList(crawlTimeOut int64) *[]string {
url := "https://www.cls.cn" url := "https://www.cls.cn"
response, err := resty.New().R(). response, err := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R().
SetHeader("Referer", "https://www.cls.cn/"). SetHeader("Referer", "https://www.cls.cn/").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(fmt.Sprintf(url)) Get(fmt.Sprintf(url))

View File

@ -22,5 +22,5 @@ func TestNewDeepSeekOpenAiConfig(t *testing.T) {
} }
func TestGetTopNewsList(t *testing.T) { func TestGetTopNewsList(t *testing.T) {
GetTopNewsList() GetTopNewsList(30)
} }

View File

@ -26,6 +26,7 @@ type Settings struct {
Prompt string `json:"prompt"` Prompt string `json:"prompt"`
CheckUpdate bool `json:"checkUpdate"` CheckUpdate bool `json:"checkUpdate"`
QuestionTemplate string `json:"questionTemplate"` QuestionTemplate string `json:"questionTemplate"`
CrawlTimeOut int64 `json:"crawlTimeOut"`
} }
func (receiver Settings) TableName() string { func (receiver Settings) TableName() string {
@ -63,6 +64,7 @@ func (s SettingsApi) UpdateConfig() string {
"check_update": s.Config.CheckUpdate, "check_update": s.Config.CheckUpdate,
"open_ai_api_time_out": s.Config.OpenAiApiTimeOut, "open_ai_api_time_out": s.Config.OpenAiApiTimeOut,
"question_template": s.Config.QuestionTemplate, "question_template": s.Config.QuestionTemplate,
"crawl_time_out": s.Config.CrawlTimeOut,
}) })
} else { } else {
logger.SugaredLogger.Infof("未找到配置,创建默认配置:%+v", s.Config) logger.SugaredLogger.Infof("未找到配置,创建默认配置:%+v", s.Config)
@ -83,6 +85,7 @@ func (s SettingsApi) UpdateConfig() string {
CheckUpdate: s.Config.CheckUpdate, CheckUpdate: s.Config.CheckUpdate,
OpenAiApiTimeOut: s.Config.OpenAiApiTimeOut, OpenAiApiTimeOut: s.Config.OpenAiApiTimeOut,
QuestionTemplate: s.Config.QuestionTemplate, QuestionTemplate: s.Config.QuestionTemplate,
CrawlTimeOut: s.Config.CrawlTimeOut,
}) })
} }
return "保存成功!" return "保存成功!"

View File

@ -516,15 +516,15 @@ func (IndexBasic) TableName() string {
return "tushare_index_basic" return "tushare_index_basic"
} }
func SearchStockPriceInfo(stockCode string) *[]string { func SearchStockPriceInfo(stockCode string, crawlTimeOut int64) *[]string {
url := "https://www.cls.cn/stock?code=" + stockCode url := "https://www.cls.cn/stock?code=" + stockCode
// 创建一个 chromedp 上下文 // 创建一个 chromedp 上下文
timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), 30*time.Second) timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second)
defer timeoutCtxCancel() defer timeoutCtxCancel()
var ctx context.Context var ctx context.Context
var cancel context.CancelFunc var cancel context.CancelFunc
path, e := checkEdgeOnWindows() path, e := checkBrowserOnWindows()
logger.SugaredLogger.Infof("SearchStockPriceInfo path:%s", path) logger.SugaredLogger.Infof("SearchStockPriceInfo path:%s", path)
if e { if e {
pctx, pcancel := chromedp.NewExecAllocator( pctx, pcancel := chromedp.NewExecAllocator(
@ -608,13 +608,13 @@ func FetchPrice(ctx context.Context) (string, error) {
} }
} }
} }
func SearchStockInfo(stock, msgType string) *[]string { func SearchStockInfo(stock, msgType string, crawlTimeOut int64) *[]string {
// 创建一个 chromedp 上下文 // 创建一个 chromedp 上下文
timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), 30*time.Second) timeoutCtx, timeoutCtxCancel := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second)
defer timeoutCtxCancel() defer timeoutCtxCancel()
var ctx context.Context var ctx context.Context
var cancel context.CancelFunc var cancel context.CancelFunc
path, e := checkEdgeOnWindows() path, e := checkBrowserOnWindows()
logger.SugaredLogger.Infof("SearchStockInfo path:%s", path) logger.SugaredLogger.Infof("SearchStockInfo path:%s", path)
if e { if e {
@ -712,8 +712,32 @@ func SearchStockInfoByCode(stock string) *[]string {
return &messages return &messages
} }
// checkEdgeOnWindows 在 Windows 系统上检查Edge浏览器是否安装并返回安装路径 // checkChromeOnWindows 在 Windows 系统上检查谷歌浏览器是否安装
func checkEdgeOnWindows() (string, bool) { func checkChromeOnWindows() (string, bool) {
key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe`, registry.QUERY_VALUE)
if err != nil {
// 尝试在 WOW6432Node 中查找(适用于 64 位系统上的 32 位程序)
key, err = registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe`, registry.QUERY_VALUE)
if err != nil {
return "", false
}
defer key.Close()
}
defer key.Close()
path, _, err := key.GetStringValue("Path")
logger.SugaredLogger.Infof("Chrome安装路径%s", path)
if err != nil {
return "", false
}
return path + "\\chrome.exe", true
}
// checkBrowserOnWindows 在 Windows 系统上检查Edge浏览器是否安装并返回安装路径
func checkBrowserOnWindows() (string, bool) {
if path, ok := checkChromeOnWindows(); ok {
return path, true
}
key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\msedge.exe`, registry.QUERY_VALUE) key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\msedge.exe`, registry.QUERY_VALUE)
if err != nil { if err != nil {
// 尝试在 WOW6432Node 中查找(适用于 64 位系统上的 32 位程序) // 尝试在 WOW6432Node 中查找(适用于 64 位系统上的 32 位程序)

View File

@ -29,7 +29,8 @@ const formValue = ref({
maxTokens: 1024, maxTokens: 1024,
prompt:"", prompt:"",
timeout: 5, timeout: 5,
questionTemplate: "{{stockName}}分析和总结" questionTemplate: "{{stockName}}分析和总结",
crawlTimeOut:30,
} }
}) })
@ -56,6 +57,7 @@ onMounted(()=>{
prompt:res.prompt, prompt:res.prompt,
timeout:res.openAiApiTimeOut, timeout:res.openAiApiTimeOut,
questionTemplate:res.questionTemplate?res.questionTemplate:'{{stockName}}分析和总结', questionTemplate:res.questionTemplate?res.questionTemplate:'{{stockName}}分析和总结',
crawlTimeOut:res.crawlTimeOut,
} }
console.log(res) console.log(res)
}) })
@ -80,7 +82,8 @@ function saveConfig(){
tushareToken:formValue.value.tushareToken, tushareToken:formValue.value.tushareToken,
prompt:formValue.value.openAI.prompt, prompt:formValue.value.openAI.prompt,
openAiApiTimeOut:formValue.value.openAI.timeout, openAiApiTimeOut:formValue.value.openAI.timeout,
questionTemplate:formValue.value.openAI.questionTemplate questionTemplate:formValue.value.openAI.questionTemplate,
crawlTimeOut:formValue.value.openAI.crawlTimeOut,
}) })
//console.log("Settings",config) //console.log("Settings",config)
@ -148,6 +151,7 @@ function importConfig(){
prompt:config.prompt, prompt:config.prompt,
timeout:config.openAiApiTimeOut, timeout:config.openAiApiTimeOut,
questionTemplate:config.questionTemplate, questionTemplate:config.questionTemplate,
crawlTimeOut:config.crawlTimeOut,
} }
// formRef.value.resetFields() // formRef.value.resetFields()
}; };
@ -215,14 +219,17 @@ window.onerror = function (event, source, lineno, colno, error) {
<n-gi :span="24"> <n-gi :span="24">
<n-text type="default" style="font-size: 25px;font-weight: bold">OpenAI设置</n-text> <n-text type="default" style="font-size: 25px;font-weight: bold">OpenAI设置</n-text>
</n-gi> </n-gi>
<n-form-item-gi :span="6" label="是否启用AI诊股" path="openAI.enable" > <n-form-item-gi :span="3" label="是否启用AI诊股" path="openAI.enable" >
<n-switch v-model:value="formValue.openAI.enable" /> <n-switch v-model:value="formValue.openAI.enable" />
</n-form-item-gi> </n-form-item-gi>
<n-form-item-gi :span="11" v-if="formValue.openAI.enable" label="openAI 接口地址:" path="openAI.baseUrl"> <n-form-item-gi :span="9" v-if="formValue.openAI.enable" label="openAI 接口地址:" path="openAI.baseUrl">
<n-input type="text" placeholder="AI接口地址" v-model:value="formValue.openAI.baseUrl" clearable /> <n-input type="text" placeholder="AI接口地址" v-model:value="formValue.openAI.baseUrl" clearable />
</n-form-item-gi> </n-form-item-gi>
<n-form-item-gi :span="5" v-if="formValue.openAI.enable" label="请求超时时间(秒)" path="openAI.timeout"> <n-form-item-gi :span="5" v-if="formValue.openAI.enable" label="AI Timeout(秒)" title="AI请求超时时间(秒)" path="openAI.timeout">
<n-input-number min="1" step="1" placeholder="请求超时时间(秒)" v-model:value="formValue.openAI.timeout" /> <n-input-number min="60" step="1" placeholder="AI请求超时时间(秒)" v-model:value="formValue.openAI.timeout" />
</n-form-item-gi>
<n-form-item-gi :span="5" v-if="formValue.openAI.enable" label="Crawler Timeout(秒)" title="资讯采集超时时间(秒)" path="openAI.crawlTimeOut">
<n-input-number min="30" step="1" placeholder="资讯采集超时时间(秒)" v-model:value="formValue.openAI.crawlTimeOut" />
</n-form-item-gi> </n-form-item-gi>
<n-form-item-gi :span="12" v-if="formValue.openAI.enable" label="openAI 令牌(apiKey)" path="openAI.apiKey"> <n-form-item-gi :span="12" v-if="formValue.openAI.enable" label="openAI 令牌(apiKey)" path="openAI.apiKey">
<n-input type="text" placeholder="apiKey" v-model:value="formValue.openAI.apiKey" clearable /> <n-input type="text" placeholder="apiKey" v-model:value="formValue.openAI.apiKey" clearable />
@ -253,15 +260,14 @@ window.onerror = function (event, source, lineno, colno, error) {
:show-count="true" :show-count="true"
placeholder="请输入用户prompt:例如{{stockName}}[{{stockCode}}]分析和总结" placeholder="请输入用户prompt:例如{{stockName}}[{{stockCode}}]分析和总结"
:autosize="{ :autosize="{
minRows: 5, minRows: 2,
maxRows: 8 maxRows: 5
}" }"
/> />
</n-form-item-gi> </n-form-item-gi>
</n-grid> </n-grid>
<n-gi :span="24"> <n-gi :span="24">
<div style="display: flex; justify-content: center"> <n-space justify="center">
<n-space>
<n-button type="primary" @click="saveConfig"> <n-button type="primary" @click="saveConfig">
保存 保存
</n-button> </n-button>
@ -272,7 +278,6 @@ window.onerror = function (event, source, lineno, colno, error) {
导入 导入
</n-button> </n-button>
</n-space> </n-space>
</div>
</n-gi> </n-gi>
</n-form> </n-form>
</n-flex> </n-flex>

View File

@ -77,6 +77,7 @@ export namespace data {
prompt: string; prompt: string;
checkUpdate: boolean; checkUpdate: boolean;
questionTemplate: string; questionTemplate: string;
crawlTimeOut: number;
static createFrom(source: any = {}) { static createFrom(source: any = {}) {
return new Settings(source); return new Settings(source);
@ -104,6 +105,7 @@ export namespace data {
this.prompt = source["prompt"]; this.prompt = source["prompt"];
this.checkUpdate = source["checkUpdate"]; this.checkUpdate = source["checkUpdate"];
this.questionTemplate = source["questionTemplate"]; this.questionTemplate = source["questionTemplate"];
this.crawlTimeOut = source["crawlTimeOut"];
} }
convertValues(a: any, classs: any, asMap: boolean = false): any { convertValues(a: any, classs: any, asMap: boolean = false): any {