go-stock/backend/data/market_news_api.go
ArvinLovegood ad9bea4c24 feat(研报):增加个股研报搜索功能
- 修改 App.d.ts 和 App.js,为 StockResearchReport 函数添加股票代码参数
- 更新 app_common.go,将 StockResearchReport 方法改为接收股票代码参数
- 修改 market_news_api.go,实现根据股票代码查询研报的逻辑
- 更新 market_news_api_test.go,添加针对具体股票代码的测试用例
- 在前端 StockResearchReportList 组件中增加股票代码搜索功能
2025-06-16 14:45:59 +08:00

436 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package data
import (
"bytes"
"encoding/json"
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/duke-git/lancet/v2/convertor"
"github.com/duke-git/lancet/v2/strutil"
"github.com/go-resty/resty/v2"
"github.com/robertkrimen/otto"
"github.com/samber/lo"
"github.com/tidwall/gjson"
"go-stock/backend/db"
"go-stock/backend/logger"
"go-stock/backend/models"
"strconv"
"strings"
"time"
)
// @Author spark
// @Date 2025/4/23 14:54
// @Desc
// -----------------------------------------------------------------------------------
type MarketNewsApi struct {
}
func NewMarketNewsApi() *MarketNewsApi {
return &MarketNewsApi{}
}
func (m MarketNewsApi) GetNewTelegraph(crawlTimeOut int64) *[]models.Telegraph {
url := "https://www.cls.cn/telegraph"
response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R().
SetHeader("Referer", "https://www.cls.cn/").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(fmt.Sprintf(url))
var telegraphs []models.Telegraph
//logger.SugaredLogger.Info(string(response.Body()))
document, _ := goquery.NewDocumentFromReader(strings.NewReader(string(response.Body())))
document.Find(".telegraph-content-box").Each(func(i int, selection *goquery.Selection) {
//logger.SugaredLogger.Info(selection.Text())
telegraph := models.Telegraph{Source: "财联社电报"}
spans := selection.Find("div.telegraph-content-box span")
if spans.Length() == 2 {
telegraph.Time = spans.First().Text()
telegraph.Content = spans.Last().Text()
if spans.Last().HasClass("c-de0422") {
telegraph.IsRed = true
}
}
labels := selection.Find("div a.label-item")
labels.Each(func(i int, selection *goquery.Selection) {
if selection.HasClass("link-label-item") {
telegraph.Url = selection.AttrOr("href", "")
} else {
tag := &models.Tags{
Name: selection.Text(),
Type: "subject",
}
db.Dao.Model(tag).Where("name=? and type=?", selection.Text(), "subject").FirstOrCreate(&tag)
telegraph.SubjectTags = append(telegraph.SubjectTags, selection.Text())
}
})
stocks := selection.Find("div.telegraph-stock-plate-box a")
stocks.Each(func(i int, selection *goquery.Selection) {
telegraph.StocksTags = append(telegraph.StocksTags, selection.Text())
})
//telegraph = append(telegraph, ReplaceSensitiveWords(selection.Text()))
if telegraph.Content != "" {
cnt := int64(0)
db.Dao.Model(telegraph).Where("time=? and source=?", telegraph.Time, telegraph.Source).Count(&cnt)
if cnt == 0 {
db.Dao.Create(&telegraph)
telegraphs = append(telegraphs, telegraph)
for _, tag := range telegraph.SubjectTags {
tagInfo := &models.Tags{}
db.Dao.Model(models.Tags{}).Where("name=? and type=?", tag, "subject").First(&tagInfo)
if tagInfo.ID > 0 {
db.Dao.Model(models.TelegraphTags{}).Where("telegraph_id=? and tag_id=?", telegraph.ID, tagInfo.ID).FirstOrCreate(&models.TelegraphTags{
TelegraphId: telegraph.ID,
TagId: tagInfo.ID,
})
}
}
}
}
})
return &telegraphs
}
func (m MarketNewsApi) GetNewsList(source string, limit int) *[]*models.Telegraph {
news := &[]*models.Telegraph{}
if source != "" {
db.Dao.Model(news).Preload("TelegraphTags").Where("source=?", source).Order("id desc").Limit(limit).Find(news)
} else {
db.Dao.Model(news).Preload("TelegraphTags").Order("id desc").Limit(limit).Find(news)
}
for _, item := range *news {
tags := &[]models.Tags{}
db.Dao.Model(&models.Tags{}).Where("id in ?", lo.Map(item.TelegraphTags, func(item models.TelegraphTags, index int) uint {
return item.TagId
})).Find(&tags)
tagNames := lo.Map(*tags, func(item models.Tags, index int) string {
return item.Name
})
item.SubjectTags = tagNames
logger.SugaredLogger.Infof("tagNames %v SubjectTags%s", tagNames, item.SubjectTags)
}
return news
}
func (m MarketNewsApi) GetTelegraphList(source string) *[]*models.Telegraph {
news := &[]*models.Telegraph{}
if source != "" {
db.Dao.Model(news).Preload("TelegraphTags").Where("source=?", source).Order("id desc").Limit(20).Find(news)
} else {
db.Dao.Model(news).Preload("TelegraphTags").Order("id desc").Limit(20).Find(news)
}
for _, item := range *news {
tags := &[]models.Tags{}
db.Dao.Model(&models.Tags{}).Where("id in ?", lo.Map(item.TelegraphTags, func(item models.TelegraphTags, index int) uint {
return item.TagId
})).Find(&tags)
tagNames := lo.Map(*tags, func(item models.Tags, index int) string {
return item.Name
})
item.SubjectTags = tagNames
logger.SugaredLogger.Infof("tagNames %v SubjectTags%s", tagNames, item.SubjectTags)
}
return news
}
func (m MarketNewsApi) GetSinaNews(crawlTimeOut uint) *[]models.Telegraph {
news := &[]models.Telegraph{}
response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R().
SetHeader("Referer", "https://finance.sina.com.cn").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get("https://zhibo.sina.com.cn/api/zhibo/feed?callback=callback&page=1&page_size=20&zhibo_id=152&tag_id=0&dire=f&dpc=1&pagesize=20&id=4161089&type=0&_=" + strconv.FormatInt(time.Now().Unix(), 10))
js := string(response.Body())
js = strutil.ReplaceWithMap(js,
map[string]string{
"try{callback(": "var data=",
");}catch(e){};": ";",
})
//logger.SugaredLogger.Info(js)
vm := otto.New()
_, err := vm.Run(js)
if err != nil {
logger.SugaredLogger.Error(err)
}
vm.Run("var result = data.result;")
//vm.Run("var resultStr =JSON.stringify(data);")
vm.Run("var resultData = result.data;")
vm.Run("var feed = resultData.feed;")
vm.Run("var feedStr = JSON.stringify(feed);")
value, _ := vm.Get("feedStr")
//resultStr, _ := vm.Get("resultStr")
//logger.SugaredLogger.Info(resultStr)
feed := make(map[string]any)
err = json.Unmarshal([]byte(value.String()), &feed)
if err != nil {
logger.SugaredLogger.Errorf("json.Unmarshal error:%v", err.Error())
}
var telegraphs []models.Telegraph
if feed["list"] != nil {
for _, item := range feed["list"].([]any) {
telegraph := models.Telegraph{Source: "新浪财经"}
data := item.(map[string]any)
//logger.SugaredLogger.Infof("%s:%s", data["create_time"], data["rich_text"])
telegraph.Content = data["rich_text"].(string)
telegraph.Time = strings.Split(data["create_time"].(string), " ")[1]
tags := data["tag"].([]any)
telegraph.SubjectTags = lo.Map(tags, func(tagItem any, index int) string {
name := tagItem.(map[string]any)["name"].(string)
tag := &models.Tags{
Name: name,
Type: "sina_subject",
}
db.Dao.Model(tag).Where("name=? and type=?", name, "sina_subject").FirstOrCreate(&tag)
return name
})
if _, ok := lo.Find(telegraph.SubjectTags, func(item string) bool { return item == "焦点" }); ok {
telegraph.IsRed = true
}
logger.SugaredLogger.Infof("telegraph.SubjectTags:%v %s", telegraph.SubjectTags, telegraph.Content)
if telegraph.Content != "" {
cnt := int64(0)
db.Dao.Model(telegraph).Where("time=? and source=?", telegraph.Time, telegraph.Source).Count(&cnt)
if cnt == 0 {
db.Dao.Create(&telegraph)
telegraphs = append(telegraphs, telegraph)
for _, tag := range telegraph.SubjectTags {
tagInfo := &models.Tags{}
db.Dao.Model(models.Tags{}).Where("name=? and type=?", tag, "sina_subject").First(&tagInfo)
if tagInfo.ID > 0 {
db.Dao.Model(models.TelegraphTags{}).Where("telegraph_id=? and tag_id=?", telegraph.ID, tagInfo.ID).FirstOrCreate(&models.TelegraphTags{
TelegraphId: telegraph.ID,
TagId: tagInfo.ID,
})
}
}
}
}
}
return &telegraphs
}
return news
}
func (m MarketNewsApi) GlobalStockIndexes(crawlTimeOut uint) map[string]any {
response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R().
SetHeader("Referer", "https://stockapp.finance.qq.com/mstats").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get("https://proxy.finance.qq.com/ifzqgtimg/appstock/app/rank/indexRankDetail2")
js := string(response.Body())
res := make(map[string]any)
json.Unmarshal([]byte(js), &res)
return res["data"].(map[string]any)
}
func (m MarketNewsApi) GetIndustryRank(sort string, cnt int) map[string]any {
url := fmt.Sprintf("https://proxy.finance.qq.com/ifzqgtimg/appstock/app/mktHs/rank?l=%d&p=1&t=01/averatio&ordertype=&o=%s", cnt, sort)
response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R().
SetHeader("Referer", "https://stockapp.finance.qq.com/").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(url)
js := string(response.Body())
res := make(map[string]any)
json.Unmarshal([]byte(js), &res)
return res
}
func (m MarketNewsApi) GetIndustryMoneyRankSina(fenlei, sort string) []map[string]any {
url := fmt.Sprintf("https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_bkzj_bk?page=1&num=20&sort=%s&asc=0&fenlei=%s", sort, fenlei)
response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R().
SetHeader("Host", "vip.stock.finance.sina.com.cn").
SetHeader("Referer", "https://finance.sina.com.cn").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(url)
js := string(response.Body())
res := &[]map[string]any{}
err := json.Unmarshal([]byte(js), &res)
if err != nil {
logger.SugaredLogger.Error(err)
return *res
}
return *res
}
func (m MarketNewsApi) GetMoneyRankSina(sort string) []map[string]any {
if sort == "" {
sort = "netamount"
}
url := fmt.Sprintf("https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_bkzj_ssggzj?page=1&num=20&sort=%s&asc=0&bankuai=&shichang=", sort)
response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R().
SetHeader("Host", "vip.stock.finance.sina.com.cn").
SetHeader("Referer", "https://finance.sina.com.cn").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").
Get(url)
js := string(response.Body())
res := &[]map[string]any{}
err := json.Unmarshal([]byte(js), &res)
if err != nil {
logger.SugaredLogger.Error(err)
return *res
}
return *res
}
func (m MarketNewsApi) GetStockMoneyTrendByDay(stockCode string, days int) []map[string]any {
url := fmt.Sprintf("http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_qsfx_zjlrqs?page=1&num=%d&sort=opendate&asc=0&daima=%s", days, stockCode)
response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R().
SetHeader("Host", "vip.stock.finance.sina.com.cn").
SetHeader("Referer", "https://finance.sina.com.cn").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").Get(url)
js := string(response.Body())
res := &[]map[string]any{}
err := json.Unmarshal([]byte(js), &res)
if err != nil {
logger.SugaredLogger.Error(err)
return *res
}
return *res
}
func (m MarketNewsApi) TopStocksRankingList(date string) {
url := fmt.Sprintf("http://vip.stock.finance.sina.com.cn/q/go.php/vInvestConsult/kind/lhb/index.phtml?tradedate=%s", date)
response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R().
SetHeader("Host", "vip.stock.finance.sina.com.cn").
SetHeader("Referer", "https://finance.sina.com.cn").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").Get(url)
html, _ := (convertor.GbkToUtf8(response.Body()))
//logger.SugaredLogger.Infof("html:%s", html)
document, err := goquery.NewDocumentFromReader(bytes.NewReader(html))
if err != nil {
return
}
document.Find("table.list_table").Each(func(i int, s *goquery.Selection) {
title := strutil.Trim(s.Find("tr:first-child").First().Text())
logger.SugaredLogger.Infof("title:%s", title)
s.Find("tr:not(:first-child)").Each(func(i int, s *goquery.Selection) {
logger.SugaredLogger.Infof("s:%s", strutil.RemoveNonPrintable(s.Text()))
})
})
}
func (m MarketNewsApi) LongTiger(date string) *[]models.LongTigerRankData {
ranks := &[]models.LongTigerRankData{}
url := "https://datacenter-web.eastmoney.com/api/data/v1/get"
logger.SugaredLogger.Infof("url:%s", url)
params := make(map[string]string)
params["callback"] = "callback"
params["sortColumns"] = "TURNOVERRATE,TRADE_DATE,SECURITY_CODE"
params["sortTypes"] = "-1,-1,1"
params["pageSize"] = "500"
params["pageNumber"] = "1"
params["reportName"] = "RPT_DAILYBILLBOARD_DETAILSNEW"
params["columns"] = "SECURITY_CODE,SECUCODE,SECURITY_NAME_ABBR,TRADE_DATE,EXPLAIN,CLOSE_PRICE,CHANGE_RATE,BILLBOARD_NET_AMT,BILLBOARD_BUY_AMT,BILLBOARD_SELL_AMT,BILLBOARD_DEAL_AMT,ACCUM_AMOUNT,DEAL_NET_RATIO,DEAL_AMOUNT_RATIO,TURNOVERRATE,FREE_MARKET_CAP,EXPLANATION,D1_CLOSE_ADJCHRATE,D2_CLOSE_ADJCHRATE,D5_CLOSE_ADJCHRATE,D10_CLOSE_ADJCHRATE,SECURITY_TYPE_CODE"
params["source"] = "WEB"
params["client"] = "WEB"
params["filter"] = fmt.Sprintf("(TRADE_DATE<='%s')(TRADE_DATE>='%s')", date, date)
resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R().
SetHeader("Host", "datacenter-web.eastmoney.com").
SetHeader("Referer", "https://data.eastmoney.com/stock/tradedetail.html").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0").
SetQueryParams(params).
Get(url)
if err != nil {
return ranks
}
js := string(resp.Body())
logger.SugaredLogger.Infof("resp:%s", js)
js = strutil.ReplaceWithMap(js,
map[string]string{
"callback(": "var data=",
");": ";",
})
//logger.SugaredLogger.Info(js)
vm := otto.New()
_, err = vm.Run(js)
_, err = vm.Run("var data = JSON.stringify(data);")
value, err := vm.Get("data")
logger.SugaredLogger.Infof("resp-json:%s", value.String())
data := gjson.Get(value.String(), "result.data")
logger.SugaredLogger.Infof("resp:%v", data)
err = json.Unmarshal([]byte(data.String()), ranks)
if err != nil {
logger.SugaredLogger.Error(err)
return ranks
}
for _, rankData := range *ranks {
temp := &models.LongTigerRankData{}
db.Dao.Model(temp).Where(&models.LongTigerRankData{
TRADEDATE: rankData.TRADEDATE,
SECUCODE: rankData.SECUCODE,
}).First(temp)
if temp.SECURITYTYPECODE == "" {
db.Dao.Model(temp).Create(&rankData)
}
}
return ranks
}
func (m MarketNewsApi) StockResearchReport(stockCode string, days int) []any {
beginDate := time.Now().Add(-time.Duration(days) * 24 * time.Hour).Format("2006-01-02")
endDate := time.Now().Format("2006-01-02")
if strutil.ContainsAny(stockCode, []string{"."}) {
stockCode = strings.Split(stockCode, ".")[0]
beginDate = time.Now().Add(-time.Duration(days) * 365 * time.Hour).Format("2006-01-02")
}
logger.SugaredLogger.Infof("StockResearchReport-stockCode:%s", stockCode)
type Req struct {
BeginTime string `json:"beginTime"`
EndTime string `json:"endTime"`
IndustryCode string `json:"industryCode"`
RatingChange string `json:"ratingChange"`
Rating string `json:"rating"`
OrgCode interface{} `json:"orgCode"`
Code string `json:"code"`
Rcode string `json:"rcode"`
PageSize int `json:"pageSize"`
PageNo int `json:"pageNo"`
P int `json:"p"`
PageNum int `json:"pageNum"`
PageNumber int `json:"pageNumber"`
}
url := "https://reportapi.eastmoney.com/report/list2"
logger.SugaredLogger.Infof("beginDate:%s endDate:%s", beginDate, endDate)
resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R().
SetHeader("Host", "reportapi.eastmoney.com").
SetHeader("Origin", "https://data.eastmoney.com").
SetHeader("Referer", "https://data.eastmoney.com/report/stock.jshtml").
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0").
SetHeader("Content-Type", "application/json").
SetBody(&Req{
Code: stockCode,
IndustryCode: "*",
BeginTime: beginDate,
EndTime: endDate,
PageNo: 1,
PageSize: 50,
P: 1,
PageNum: 1,
PageNumber: 1,
}).Post(url)
respMap := map[string]any{}
if err != nil {
return []any{}
}
json.Unmarshal(resp.Body(), &respMap)
//logger.SugaredLogger.Infof("resp:%+v", respMap["data"])
return respMap["data"].([]any)
}