diff --git a/backend/data/openai_api.go b/backend/data/openai_api.go index 42b7fa5..4845d5f 100644 --- a/backend/data/openai_api.go +++ b/backend/data/openai_api.go @@ -285,7 +285,7 @@ func (o OpenAi) NewChatStream(stock, stockCode, userQuestion string, sysPromptId return } - messages := GetFinancialReports(stockCode, o.CrawlTimeOut) + messages := GetFinancialReportsByXUEQIU(stockCode, o.CrawlTimeOut) if messages == nil || len(*messages) == 0 { logger.SugaredLogger.Error("获取股票财报失败") // "***❗获取股票财报失败,分析结果可能不准确***
" @@ -608,7 +608,41 @@ func SearchGuShiTongStockInfo(stock string, crawlTimeOut int64) *[]string { } return &messages } +func GetFinancialReportsByXUEQIU(stockCode string, crawlTimeOut int64) *[]string { + if strutil.HasPrefixAny(stockCode, []string{"HK", "hk"}) { + stockCode = strings.ReplaceAll(stockCode, "hk", "") + stockCode = strings.ReplaceAll(stockCode, "HK", "") + } + if strutil.HasPrefixAny(stockCode, []string{"us", "gb_"}) { + stockCode = strings.ReplaceAll(stockCode, "us", "") + stockCode = strings.ReplaceAll(stockCode, "gb_", "") + } + url := fmt.Sprintf("https://xueqiu.com/snowman/S/%s/detail#/ZYCWZB", stockCode) + waitVisible := "div.tab-table-responsive table" + crawlerAPI := CrawlerApi{} + crawlerBaseInfo := CrawlerBaseInfo{ + Name: "TestCrawler", + Description: "Test Crawler Description", + BaseUrl: "https://xueqiu.com", + Headers: map[string]string{"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0"}, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(crawlTimeOut)*time.Second) + defer cancel() + crawlerAPI = crawlerAPI.NewCrawler(ctx, crawlerBaseInfo) + var markdown strings.Builder + markdown.WriteString("\n## 财务数据:\n") + html, ok := crawlerAPI.GetHtml(url, waitVisible, true) + if !ok { + return &[]string{""} + } + document, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + logger.SugaredLogger.Error(err.Error()) + } + GetTableMarkdown(document, waitVisible, &markdown) + return &[]string{markdown.String()} +} func GetFinancialReports(stockCode string, crawlTimeOut int64) *[]string { url := "https://emweb.securities.eastmoney.com/pc_hsf10/pages/index.html?type=web&code=" + stockCode + "#/cwfx" waitVisible := "div.report_table table" diff --git a/backend/data/stock_data_api_test.go b/backend/data/stock_data_api_test.go index 7acc573..e318c0a 100644 --- a/backend/data/stock_data_api_test.go +++ b/backend/data/stock_data_api_test.go @@ -26,9 +26,13 @@ func TestGetTelegraph(t *testing.T) { } func TestGetFinancialReports(t *testing.T) { + db.Init("../../data/stock.db") //GetFinancialReports("sz000802", 30) //GetFinancialReports("hk00927", 30) - GetFinancialReports("gb_aapl", 30) + //GetFinancialReports("gb_aapl", 30) + GetFinancialReportsByXUEQIU("sz000802", 30) + GetFinancialReportsByXUEQIU("gb_aapl", 30) + GetFinancialReportsByXUEQIU("hk00927", 30) } @@ -191,7 +195,7 @@ func TestReadFile(t *testing.T) { func TestFollowedList(t *testing.T) { db.Init("../../data/stock.db") stockDataApi := NewStockDataApi() - stockDataApi.GetFollowList() + stockDataApi.GetFollowList(1) } diff --git a/backend/data/utils.go b/backend/data/utils.go index 6ffbb4a..cceac90 100644 --- a/backend/data/utils.go +++ b/backend/data/utils.go @@ -64,7 +64,13 @@ func GetTableMarkdown(document *goquery.Document, waitVisible string, markdown * document.Find(waitVisible).First().Find("tr").Each(func(index int, item *goquery.Selection) { row := "" item.Find("th, td").Each(func(i int, cell *goquery.Selection) { - text := cell.Text() + text := cell.Children().FilterFunction(func(i int, s *goquery.Selection) bool { + return isVisible(s) + }).Text() + if text == "" { + text = cell.Text() + } + row += "|" + text }) row += "|" @@ -85,3 +91,21 @@ func GetTableMarkdown(document *goquery.Document, waitVisible string, markdown * }) logger.SugaredLogger.Infof("\n%s", markdown.String()) } + +// isVisible 函数用于判断元素是否可见 +func isVisible(s *goquery.Selection) bool { + // 检查 display 属性 + display, _ := s.Attr("style") + if strings.Contains(strings.ToLower(display), "display: none") { + return false + } + // 检查 visibility 属性 + if strings.Contains(strings.ToLower(display), "visibility: hidden") { + return false + } + // 检查 opacity 属性 + if strings.Contains(strings.ToLower(display), "opacity: 0") { + return false + } + return true +}