package data import ( "bytes" "encoding/json" "fmt" "github.com/PuerkitoBio/goquery" "github.com/coocood/freecache" "github.com/duke-git/lancet/v2/convertor" "github.com/duke-git/lancet/v2/strutil" "github.com/go-resty/resty/v2" "github.com/robertkrimen/otto" "github.com/samber/lo" "github.com/tidwall/gjson" "go-stock/backend/db" "go-stock/backend/logger" "go-stock/backend/models" "strconv" "strings" "time" ) // @Author spark // @Date 2025/4/23 14:54 // @Desc // ----------------------------------------------------------------------------------- type MarketNewsApi struct { } func NewMarketNewsApi() *MarketNewsApi { return &MarketNewsApi{} } func (m MarketNewsApi) GetNewTelegraph(crawlTimeOut int64) *[]models.Telegraph { url := "https://www.cls.cn/telegraph" response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R(). SetHeader("Referer", "https://www.cls.cn/"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get(fmt.Sprintf(url)) var telegraphs []models.Telegraph //logger.SugaredLogger.Info(string(response.Body())) document, _ := goquery.NewDocumentFromReader(strings.NewReader(string(response.Body()))) document.Find(".telegraph-content-box").Each(func(i int, selection *goquery.Selection) { //logger.SugaredLogger.Info(selection.Text()) telegraph := models.Telegraph{Source: "财联社电报"} spans := selection.Find("div.telegraph-content-box span") if spans.Length() == 2 { telegraph.Time = spans.First().Text() telegraph.Content = spans.Last().Text() if spans.Last().HasClass("c-de0422") { telegraph.IsRed = true } } labels := selection.Find("div a.label-item") labels.Each(func(i int, selection *goquery.Selection) { if selection.HasClass("link-label-item") { telegraph.Url = selection.AttrOr("href", "") } else { tag := &models.Tags{ Name: selection.Text(), Type: "subject", } db.Dao.Model(tag).Where("name=? and type=?", selection.Text(), "subject").FirstOrCreate(&tag) telegraph.SubjectTags = append(telegraph.SubjectTags, selection.Text()) } }) stocks := selection.Find("div.telegraph-stock-plate-box a") stocks.Each(func(i int, selection *goquery.Selection) { telegraph.StocksTags = append(telegraph.StocksTags, selection.Text()) }) //telegraph = append(telegraph, ReplaceSensitiveWords(selection.Text())) if telegraph.Content != "" { cnt := int64(0) db.Dao.Model(telegraph).Where("time=? and source=?", telegraph.Time, telegraph.Source).Count(&cnt) if cnt == 0 { db.Dao.Create(&telegraph) telegraphs = append(telegraphs, telegraph) for _, tag := range telegraph.SubjectTags { tagInfo := &models.Tags{} db.Dao.Model(models.Tags{}).Where("name=? and type=?", tag, "subject").First(&tagInfo) if tagInfo.ID > 0 { db.Dao.Model(models.TelegraphTags{}).Where("telegraph_id=? and tag_id=?", telegraph.ID, tagInfo.ID).FirstOrCreate(&models.TelegraphTags{ TelegraphId: telegraph.ID, TagId: tagInfo.ID, }) } } } } }) return &telegraphs } func (m MarketNewsApi) GetNewsList(source string, limit int) *[]*models.Telegraph { news := &[]*models.Telegraph{} if source != "" { db.Dao.Model(news).Preload("TelegraphTags").Where("source=?", source).Order("id desc").Limit(limit).Find(news) } else { db.Dao.Model(news).Preload("TelegraphTags").Order("id desc").Limit(limit).Find(news) } for _, item := range *news { tags := &[]models.Tags{} db.Dao.Model(&models.Tags{}).Where("id in ?", lo.Map(item.TelegraphTags, func(item models.TelegraphTags, index int) uint { return item.TagId })).Find(&tags) tagNames := lo.Map(*tags, func(item models.Tags, index int) string { return item.Name }) item.SubjectTags = tagNames logger.SugaredLogger.Infof("tagNames %v ,SubjectTags:%s", tagNames, item.SubjectTags) } return news } func (m MarketNewsApi) GetTelegraphList(source string) *[]*models.Telegraph { news := &[]*models.Telegraph{} if source != "" { db.Dao.Model(news).Preload("TelegraphTags").Where("source=?", source).Order("id desc").Limit(20).Find(news) } else { db.Dao.Model(news).Preload("TelegraphTags").Order("id desc").Limit(20).Find(news) } for _, item := range *news { tags := &[]models.Tags{} db.Dao.Model(&models.Tags{}).Where("id in ?", lo.Map(item.TelegraphTags, func(item models.TelegraphTags, index int) uint { return item.TagId })).Find(&tags) tagNames := lo.Map(*tags, func(item models.Tags, index int) string { return item.Name }) item.SubjectTags = tagNames logger.SugaredLogger.Infof("tagNames %v ,SubjectTags:%s", tagNames, item.SubjectTags) } return news } func (m MarketNewsApi) GetSinaNews(crawlTimeOut uint) *[]models.Telegraph { news := &[]models.Telegraph{} response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R(). SetHeader("Referer", "https://finance.sina.com.cn"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get("https://zhibo.sina.com.cn/api/zhibo/feed?callback=callback&page=1&page_size=20&zhibo_id=152&tag_id=0&dire=f&dpc=1&pagesize=20&id=4161089&type=0&_=" + strconv.FormatInt(time.Now().Unix(), 10)) js := string(response.Body()) js = strutil.ReplaceWithMap(js, map[string]string{ "try{callback(": "var data=", ");}catch(e){};": ";", }) //logger.SugaredLogger.Info(js) vm := otto.New() _, err := vm.Run(js) if err != nil { logger.SugaredLogger.Error(err) } vm.Run("var result = data.result;") //vm.Run("var resultStr =JSON.stringify(data);") vm.Run("var resultData = result.data;") vm.Run("var feed = resultData.feed;") vm.Run("var feedStr = JSON.stringify(feed);") value, _ := vm.Get("feedStr") //resultStr, _ := vm.Get("resultStr") //logger.SugaredLogger.Info(resultStr) feed := make(map[string]any) err = json.Unmarshal([]byte(value.String()), &feed) if err != nil { logger.SugaredLogger.Errorf("json.Unmarshal error:%v", err.Error()) } var telegraphs []models.Telegraph if feed["list"] != nil { for _, item := range feed["list"].([]any) { telegraph := models.Telegraph{Source: "新浪财经"} data := item.(map[string]any) //logger.SugaredLogger.Infof("%s:%s", data["create_time"], data["rich_text"]) telegraph.Content = data["rich_text"].(string) telegraph.Time = strings.Split(data["create_time"].(string), " ")[1] tags := data["tag"].([]any) telegraph.SubjectTags = lo.Map(tags, func(tagItem any, index int) string { name := tagItem.(map[string]any)["name"].(string) tag := &models.Tags{ Name: name, Type: "sina_subject", } db.Dao.Model(tag).Where("name=? and type=?", name, "sina_subject").FirstOrCreate(&tag) return name }) if _, ok := lo.Find(telegraph.SubjectTags, func(item string) bool { return item == "焦点" }); ok { telegraph.IsRed = true } logger.SugaredLogger.Infof("telegraph.SubjectTags:%v %s", telegraph.SubjectTags, telegraph.Content) if telegraph.Content != "" { cnt := int64(0) db.Dao.Model(telegraph).Where("time=? and source=?", telegraph.Time, telegraph.Source).Count(&cnt) if cnt == 0 { db.Dao.Create(&telegraph) telegraphs = append(telegraphs, telegraph) for _, tag := range telegraph.SubjectTags { tagInfo := &models.Tags{} db.Dao.Model(models.Tags{}).Where("name=? and type=?", tag, "sina_subject").First(&tagInfo) if tagInfo.ID > 0 { db.Dao.Model(models.TelegraphTags{}).Where("telegraph_id=? and tag_id=?", telegraph.ID, tagInfo.ID).FirstOrCreate(&models.TelegraphTags{ TelegraphId: telegraph.ID, TagId: tagInfo.ID, }) } } } } } return &telegraphs } return news } func (m MarketNewsApi) GlobalStockIndexes(crawlTimeOut uint) map[string]any { response, _ := resty.New().SetTimeout(time.Duration(crawlTimeOut)*time.Second).R(). SetHeader("Referer", "https://stockapp.finance.qq.com/mstats"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get("https://proxy.finance.qq.com/ifzqgtimg/appstock/app/rank/indexRankDetail2") js := string(response.Body()) res := make(map[string]any) json.Unmarshal([]byte(js), &res) return res["data"].(map[string]any) } func (m MarketNewsApi) GetIndustryRank(sort string, cnt int) map[string]any { url := fmt.Sprintf("https://proxy.finance.qq.com/ifzqgtimg/appstock/app/mktHs/rank?l=%d&p=1&t=01/averatio&ordertype=&o=%s", cnt, sort) response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R(). SetHeader("Referer", "https://stockapp.finance.qq.com/"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get(url) js := string(response.Body()) res := make(map[string]any) json.Unmarshal([]byte(js), &res) return res } func (m MarketNewsApi) GetIndustryMoneyRankSina(fenlei, sort string) []map[string]any { url := fmt.Sprintf("https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_bkzj_bk?page=1&num=20&sort=%s&asc=0&fenlei=%s", sort, fenlei) response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R(). SetHeader("Host", "vip.stock.finance.sina.com.cn"). SetHeader("Referer", "https://finance.sina.com.cn"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get(url) js := string(response.Body()) res := &[]map[string]any{} err := json.Unmarshal([]byte(js), &res) if err != nil { logger.SugaredLogger.Error(err) return *res } return *res } func (m MarketNewsApi) GetMoneyRankSina(sort string) []map[string]any { if sort == "" { sort = "netamount" } url := fmt.Sprintf("https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_bkzj_ssggzj?page=1&num=20&sort=%s&asc=0&bankuai=&shichang=", sort) response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R(). SetHeader("Host", "vip.stock.finance.sina.com.cn"). SetHeader("Referer", "https://finance.sina.com.cn"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60"). Get(url) js := string(response.Body()) res := &[]map[string]any{} err := json.Unmarshal([]byte(js), &res) if err != nil { logger.SugaredLogger.Error(err) return *res } return *res } func (m MarketNewsApi) GetStockMoneyTrendByDay(stockCode string, days int) []map[string]any { url := fmt.Sprintf("http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/MoneyFlow.ssl_qsfx_zjlrqs?page=1&num=%d&sort=opendate&asc=0&daima=%s", days, stockCode) response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R(). SetHeader("Host", "vip.stock.finance.sina.com.cn"). SetHeader("Referer", "https://finance.sina.com.cn"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").Get(url) js := string(response.Body()) res := &[]map[string]any{} err := json.Unmarshal([]byte(js), &res) if err != nil { logger.SugaredLogger.Error(err) return *res } return *res } func (m MarketNewsApi) TopStocksRankingList(date string) { url := fmt.Sprintf("http://vip.stock.finance.sina.com.cn/q/go.php/vInvestConsult/kind/lhb/index.phtml?tradedate=%s", date) response, _ := resty.New().SetTimeout(time.Duration(5)*time.Second).R(). SetHeader("Host", "vip.stock.finance.sina.com.cn"). SetHeader("Referer", "https://finance.sina.com.cn"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.60").Get(url) html, _ := convertor.GbkToUtf8(response.Body()) //logger.SugaredLogger.Infof("html:%s", html) document, err := goquery.NewDocumentFromReader(bytes.NewReader(html)) if err != nil { return } document.Find("table.list_table").Each(func(i int, s *goquery.Selection) { title := strutil.Trim(s.Find("tr:first-child").First().Text()) logger.SugaredLogger.Infof("title:%s", title) s.Find("tr:not(:first-child)").Each(func(i int, s *goquery.Selection) { logger.SugaredLogger.Infof("s:%s", strutil.RemoveNonPrintable(s.Text())) }) }) } func (m MarketNewsApi) LongTiger(date string) *[]models.LongTigerRankData { ranks := &[]models.LongTigerRankData{} url := "https://datacenter-web.eastmoney.com/api/data/v1/get" logger.SugaredLogger.Infof("url:%s", url) params := make(map[string]string) params["callback"] = "callback" params["sortColumns"] = "TURNOVERRATE,TRADE_DATE,SECURITY_CODE" params["sortTypes"] = "-1,-1,1" params["pageSize"] = "500" params["pageNumber"] = "1" params["reportName"] = "RPT_DAILYBILLBOARD_DETAILSNEW" params["columns"] = "SECURITY_CODE,SECUCODE,SECURITY_NAME_ABBR,TRADE_DATE,EXPLAIN,CLOSE_PRICE,CHANGE_RATE,BILLBOARD_NET_AMT,BILLBOARD_BUY_AMT,BILLBOARD_SELL_AMT,BILLBOARD_DEAL_AMT,ACCUM_AMOUNT,DEAL_NET_RATIO,DEAL_AMOUNT_RATIO,TURNOVERRATE,FREE_MARKET_CAP,EXPLANATION,D1_CLOSE_ADJCHRATE,D2_CLOSE_ADJCHRATE,D5_CLOSE_ADJCHRATE,D10_CLOSE_ADJCHRATE,SECURITY_TYPE_CODE" params["source"] = "WEB" params["client"] = "WEB" params["filter"] = fmt.Sprintf("(TRADE_DATE<='%s')(TRADE_DATE>='%s')", date, date) resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R(). SetHeader("Host", "datacenter-web.eastmoney.com"). SetHeader("Referer", "https://data.eastmoney.com/stock/tradedetail.html"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"). SetQueryParams(params). Get(url) if err != nil { return ranks } js := string(resp.Body()) logger.SugaredLogger.Infof("resp:%s", js) js = strutil.ReplaceWithMap(js, map[string]string{ "callback(": "var data=", ");": ";", }) //logger.SugaredLogger.Info(js) vm := otto.New() _, err = vm.Run(js) _, err = vm.Run("var data = JSON.stringify(data);") value, err := vm.Get("data") logger.SugaredLogger.Infof("resp-json:%s", value.String()) data := gjson.Get(value.String(), "result.data") logger.SugaredLogger.Infof("resp:%v", data) err = json.Unmarshal([]byte(data.String()), ranks) if err != nil { logger.SugaredLogger.Error(err) return ranks } for _, rankData := range *ranks { temp := &models.LongTigerRankData{} db.Dao.Model(temp).Where(&models.LongTigerRankData{ TRADEDATE: rankData.TRADEDATE, SECUCODE: rankData.SECUCODE, }).First(temp) if temp.SECURITYTYPECODE == "" { db.Dao.Model(temp).Create(&rankData) } } return ranks } func (m MarketNewsApi) IndustryResearchReport(industryCode string, days int) []any { beginDate := time.Now().Add(-time.Duration(days) * 24 * time.Hour).Format("2006-01-02") endDate := time.Now().Format("2006-01-02") if strutil.Trim(industryCode) != "" { beginDate = time.Now().Add(-time.Duration(days) * 365 * time.Hour).Format("2006-01-02") } logger.SugaredLogger.Infof("IndustryResearchReport-name:%s", industryCode) params := map[string]string{ "industry": "*", "industryCode": industryCode, "beginTime": beginDate, "endTime": endDate, "pageNo": "1", "pageSize": "50", "p": "1", "pageNum": "1", "pageNumber": "1", "qType": "1", } url := "https://reportapi.eastmoney.com/report/list" logger.SugaredLogger.Infof("beginDate:%s endDate:%s", beginDate, endDate) resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R(). SetHeader("Host", "reportapi.eastmoney.com"). SetHeader("Origin", "https://data.eastmoney.com"). SetHeader("Referer", "https://data.eastmoney.com/report/stock.jshtml"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"). SetHeader("Content-Type", "application/json"). SetQueryParams(params).Get(url) respMap := map[string]any{} if err != nil { return []any{} } json.Unmarshal(resp.Body(), &respMap) //logger.SugaredLogger.Infof("resp:%+v", respMap["data"]) return respMap["data"].([]any) } func (m MarketNewsApi) StockResearchReport(stockCode string, days int) []any { beginDate := time.Now().Add(-time.Duration(days) * 24 * time.Hour).Format("2006-01-02") endDate := time.Now().Format("2006-01-02") if strutil.ContainsAny(stockCode, []string{"."}) { stockCode = strings.Split(stockCode, ".")[0] beginDate = time.Now().Add(-time.Duration(days) * 365 * time.Hour).Format("2006-01-02") } logger.SugaredLogger.Infof("StockResearchReport-stockCode:%s", stockCode) type Req struct { BeginTime string `json:"beginTime"` EndTime string `json:"endTime"` IndustryCode string `json:"industryCode"` RatingChange string `json:"ratingChange"` Rating string `json:"rating"` OrgCode interface{} `json:"orgCode"` Code string `json:"code"` Rcode string `json:"rcode"` PageSize int `json:"pageSize"` PageNo int `json:"pageNo"` P int `json:"p"` PageNum int `json:"pageNum"` PageNumber int `json:"pageNumber"` } url := "https://reportapi.eastmoney.com/report/list2" logger.SugaredLogger.Infof("beginDate:%s endDate:%s", beginDate, endDate) resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R(). SetHeader("Host", "reportapi.eastmoney.com"). SetHeader("Origin", "https://data.eastmoney.com"). SetHeader("Referer", "https://data.eastmoney.com/report/stock.jshtml"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"). SetHeader("Content-Type", "application/json"). SetBody(&Req{ Code: stockCode, IndustryCode: "*", BeginTime: beginDate, EndTime: endDate, PageNo: 1, PageSize: 50, P: 1, PageNum: 1, PageNumber: 1, }).Post(url) respMap := map[string]any{} if err != nil { return []any{} } json.Unmarshal(resp.Body(), &respMap) //logger.SugaredLogger.Infof("resp:%+v", respMap["data"]) return respMap["data"].([]any) } func (m MarketNewsApi) StockNotice(stock_list string) []any { var stockCodes []string for _, stockCode := range strings.Split(stock_list, ",") { if strutil.ContainsAny(stockCode, []string{"."}) { stockCode = strings.Split(stockCode, ".")[0] stockCodes = append(stockCodes, stockCode) } } url := "https://np-anotice-stock.eastmoney.com/api/security/ann?page_size=50&page_index=1&ann_type=SHA%2CCYB%2CSZA%2CBJA%2CINV&client_source=web&f_node=0&stock_list=" + strings.Join(stockCodes, ",") resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R(). SetHeader("Host", "np-anotice-stock.eastmoney.com"). SetHeader("Referer", "https://data.eastmoney.com/notices/hsa/5.html"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"). Get(url) respMap := map[string]any{} if err != nil { return []any{} } json.Unmarshal(resp.Body(), &respMap) //logger.SugaredLogger.Infof("resp:%+v", respMap["data"]) return (respMap["data"].(map[string]any))["list"].([]any) } func (m MarketNewsApi) EMDictCode(code string, cache *freecache.Cache) []any { respMap := map[string]any{} d, _ := cache.Get([]byte(code)) if d != nil { json.Unmarshal(d, &respMap) return respMap["data"].([]any) } url := "https://reportapi.eastmoney.com/report/bk" params := map[string]string{ "bkCode": code, } resp, err := resty.New().SetTimeout(time.Duration(15)*time.Second).R(). SetHeader("Host", "reportapi.eastmoney.com"). SetHeader("Origin", "https://data.eastmoney.com"). SetHeader("Referer", "https://data.eastmoney.com/report/industry.jshtml"). SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"). SetHeader("Content-Type", "application/json"). SetQueryParams(params).Get(url) if err != nil { return []any{} } json.Unmarshal(resp.Body(), &respMap) //logger.SugaredLogger.Infof("resp:%+v", respMap["data"]) cache.Set([]byte(code), resp.Body(), 60*60*24) return respMap["data"].([]any) }