mirror of
https://github.com/ArvinLovegood/go-stock.git
synced 2025-07-19 00:00:09 +08:00
feat(stock):更新股票基础信息并优化相关功能
- 添加 CheckStockBaseInfo 方法,用于更新股票基础信息 - 修改 domReady 方法,移除初始化股票数据的逻辑 - 更新 StockBasic、StockInfoHK 和 StockInfoUS 模型,添加行业代码和名称字段 - 修改 getDCStockInfo 方法,支持获取更详细的股票信息 - 添加 DCToTsCode 函数,用于将东财代码转换为 TS 代码 - 优化行业报告信息获取功能
This commit is contained in:
parent
60e7d87918
commit
8db94da233
96
app.go
96
app.go
@ -301,21 +301,24 @@ func (a *App) CheckUpdate() {
|
||||
// domReady is called after front-end resources have been loaded
|
||||
func (a *App) domReady(ctx context.Context) {
|
||||
defer PanicHandler()
|
||||
defer func() {
|
||||
go runtime.EventsEmit(ctx, "loadingMsg", "done")
|
||||
}()
|
||||
|
||||
if stocksBin != nil && len(stocksBin) > 0 {
|
||||
go runtime.EventsEmit(a.ctx, "loadingMsg", "检查A股基础信息...")
|
||||
go initStockData(a.ctx)
|
||||
}
|
||||
|
||||
if stocksBinHK != nil && len(stocksBinHK) > 0 {
|
||||
go runtime.EventsEmit(a.ctx, "loadingMsg", "检查港股基础信息...")
|
||||
go initStockDataHK(a.ctx)
|
||||
}
|
||||
|
||||
if stocksBinUS != nil && len(stocksBinUS) > 0 {
|
||||
go runtime.EventsEmit(a.ctx, "loadingMsg", "检查美股基础信息...")
|
||||
go initStockDataUS(a.ctx)
|
||||
}
|
||||
//if stocksBin != nil && len(stocksBin) > 0 {
|
||||
// go runtime.EventsEmit(a.ctx, "loadingMsg", "检查A股基础信息...")
|
||||
// go initStockData(a.ctx)
|
||||
//}
|
||||
//
|
||||
//if stocksBinHK != nil && len(stocksBinHK) > 0 {
|
||||
// go runtime.EventsEmit(a.ctx, "loadingMsg", "检查港股基础信息...")
|
||||
// go initStockDataHK(a.ctx)
|
||||
//}
|
||||
//
|
||||
//if stocksBinUS != nil && len(stocksBinUS) > 0 {
|
||||
// go runtime.EventsEmit(a.ctx, "loadingMsg", "检查美股基础信息...")
|
||||
// go initStockDataUS(a.ctx)
|
||||
//}
|
||||
updateBasicInfo()
|
||||
|
||||
// Add your action here
|
||||
@ -424,6 +427,7 @@ func (a *App) domReady(ctx context.Context) {
|
||||
a.cron.AddFunc("30 05 8,12,20 * * *", func() {
|
||||
logger.SugaredLogger.Errorf("Checking for updates...")
|
||||
a.CheckUpdate()
|
||||
a.CheckStockBaseInfo()
|
||||
})
|
||||
|
||||
}()
|
||||
@ -460,7 +464,71 @@ func (a *App) domReady(ctx context.Context) {
|
||||
logger.SugaredLogger.Infof("domReady-cronEntrys:%+v", a.cronEntrys)
|
||||
|
||||
}
|
||||
func (a *App) CheckStockBaseInfo() {
|
||||
defer PanicHandler()
|
||||
|
||||
stockBasics := &[]data.StockBasic{}
|
||||
resty.New().R().
|
||||
SetHeader("user", "go-stock").
|
||||
SetResult(stockBasics).
|
||||
Get("https://go-stock.sparkmemory.top/stock_basic.json")
|
||||
|
||||
for _, stock := range *stockBasics {
|
||||
stockInfo := &data.StockBasic{
|
||||
TsCode: stock.TsCode,
|
||||
Name: stock.Name,
|
||||
Symbol: stock.Symbol,
|
||||
BKCode: stock.BKCode,
|
||||
BKName: stock.BKName,
|
||||
}
|
||||
db.Dao.Model(&data.StockBasic{}).Where("ts_code = ?", stock.TsCode).First(stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&data.StockBasic{}).Create(stockInfo)
|
||||
} else {
|
||||
db.Dao.Model(&data.StockBasic{}).Where("ts_code = ?", stock.TsCode).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
|
||||
stockHKBasics := &[]models.StockInfoHK{}
|
||||
resty.New().R().
|
||||
SetHeader("user", "go-stock").
|
||||
SetResult(stockHKBasics).
|
||||
Get("https://go-stock.sparkmemory.top/stock_base_info_hk.json")
|
||||
for _, stock := range *stockHKBasics {
|
||||
stockInfo := &models.StockInfoHK{
|
||||
Code: stock.Code,
|
||||
Name: stock.Name,
|
||||
BKName: stock.BKName,
|
||||
BKCode: stock.BKCode,
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoHK{}).Where("code = ?", stock.Code).First(stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&models.StockInfoHK{}).Create(stockInfo)
|
||||
} else {
|
||||
db.Dao.Model(&models.StockInfoHK{}).Where("code = ?", stock.Code).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
stockUSBasics := &[]models.StockInfoUS{}
|
||||
resty.New().R().
|
||||
SetHeader("user", "go-stock").
|
||||
SetResult(stockUSBasics).
|
||||
Get("https://go-stock.sparkmemory.top/stock_base_info_us.json")
|
||||
for _, stock := range *stockUSBasics {
|
||||
stockInfo := &models.StockInfoUS{
|
||||
Code: stock.Code,
|
||||
Name: stock.Name,
|
||||
BKName: stock.BKName,
|
||||
BKCode: stock.BKCode,
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoUS{}).Where("code = ?", stock.Code).First(stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&models.StockInfoUS{}).Create(stockInfo)
|
||||
} else {
|
||||
db.Dao.Model(&models.StockInfoUS{}).Where("code = ?", stock.Code).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
func (a *App) NewsPush(news *[]models.Telegraph) {
|
||||
for _, telegraph := range *news {
|
||||
//if telegraph.IsRed {
|
||||
|
21
app_test.go
21
app_test.go
@ -1,7 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"go-stock/backend/db"
|
||||
"go-stock/backend/logger"
|
||||
"go-stock/backend/models"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -23,3 +26,21 @@ func TestIsUSTradingTime(t *testing.T) {
|
||||
|
||||
t.Log(IsUSTradingTime(time.Now()))
|
||||
}
|
||||
|
||||
func TestCheckStockBaseInfo(t *testing.T) {
|
||||
db.Init("./data/stock.db")
|
||||
NewApp().CheckStockBaseInfo()
|
||||
}
|
||||
|
||||
func TestJson(t *testing.T) {
|
||||
db.Init("./data/stock.db")
|
||||
|
||||
jsonStr := "{\n\t\t\"id\" : 3334,\n\t\t\"created_at\" : \"2025-02-28 16:49:31.8342514+08:00\",\n\t\t\"updated_at\" : \"2025-02-28 16:49:31.8342514+08:00\",\n\t\t\"deleted_at\" : null,\n\t\t\"code\" : \"PUK.US\",\n\t\t\"name\" : \"英国保诚集团\",\n\t\t\"full_name\" : \"\",\n\t\t\"e_name\" : \"\",\n\t\t\"exchange\" : \"NASDAQ\",\n\t\t\"type\" : \"stock\",\n\t\t\"is_del\" : 0,\n\t\t\"bk_name\" : null,\n\t\t\"bk_code\" : null\n\t}"
|
||||
|
||||
v := &models.StockInfoUS{}
|
||||
json.Unmarshal([]byte(jsonStr), v)
|
||||
logger.SugaredLogger.Infof("v:%+v", v)
|
||||
|
||||
db.Dao.Model(v).Updates(v)
|
||||
|
||||
}
|
||||
|
@ -15,6 +15,7 @@ import (
|
||||
"go-stock/backend/db"
|
||||
"go-stock/backend/logger"
|
||||
"go-stock/backend/models"
|
||||
"go-stock/backend/util"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -832,3 +833,27 @@ func (m MarketNewsApi) GetPMI() *models.PMIResp {
|
||||
return res
|
||||
|
||||
}
|
||||
func (m MarketNewsApi) GetIndustryReportInfo(infoCode string) {
|
||||
url := "https://data.eastmoney.com/report/zw_industry.jshtml?infocode=" + infoCode
|
||||
resp, err := resty.New().SetTimeout(time.Duration(30)*time.Second).R().
|
||||
SetHeader("Host", "data.eastmoney.com").
|
||||
SetHeader("Origin", "https://data.eastmoney.com").
|
||||
SetHeader("Referer", "https://data.eastmoney.com/report/industry.jshtml").
|
||||
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0").
|
||||
Get(url)
|
||||
if err != nil {
|
||||
logger.SugaredLogger.Errorf("GetIndustryReportInfo err:%s", err.Error())
|
||||
return
|
||||
}
|
||||
body := resp.Body()
|
||||
//logger.SugaredLogger.Debugf("GetIndustryReportInfo:%s", body)
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body)))
|
||||
title, _ := doc.Find("div.c-title").Html()
|
||||
content, _ := doc.Find("div.ctx-content").Html()
|
||||
//logger.SugaredLogger.Infof("GetIndustryReportInfo:\n%s\n%s", title, content)
|
||||
markdown, err := util.HTMLToMarkdown(title + content)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
logger.SugaredLogger.Infof("GetIndustryReportInfo markdown:\n%s", markdown)
|
||||
}
|
||||
|
@ -201,3 +201,6 @@ func TestGetPMI(t *testing.T) {
|
||||
md := util.MarkdownTableWithTitle("采购经理人指数(PMI)", res.PMIResult.Data)
|
||||
logger.SugaredLogger.Debugf(md)
|
||||
}
|
||||
func TestGetIndustryReportInfo(t *testing.T) {
|
||||
NewMarketNewsApi().GetIndustryReportInfo("AP202507151709216483")
|
||||
}
|
||||
|
@ -153,6 +153,8 @@ type StockBasic struct {
|
||||
IsHs string `json:"is_hs"`
|
||||
ActName string `json:"act_name"`
|
||||
ActEntType string `json:"act_ent_type"`
|
||||
BKName string `json:"bk_name"`
|
||||
BKCode string `json:"bk_code"`
|
||||
}
|
||||
|
||||
type FollowedStock struct {
|
||||
@ -1454,7 +1456,7 @@ func getSinaStockInfo(receiver StockDataApi, page, pageSize int) *[]models.SinaS
|
||||
func (receiver StockDataApi) getDCStockInfo(market string, page, pageSize int) {
|
||||
//m:105,m:106,m:107 //美股
|
||||
//m:128+t:3,m:128+t:4,m:128+t:1,m:128+t:2 //港股
|
||||
fs := ""
|
||||
fs := "m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048"
|
||||
switch market {
|
||||
case "hk":
|
||||
fs = "m:128+t:3,m:128+t:4,m:128+t:1,m:128+t:2"
|
||||
@ -1462,62 +1464,108 @@ func (receiver StockDataApi) getDCStockInfo(market string, page, pageSize int) {
|
||||
fs = "m:105,m:106,m:107"
|
||||
}
|
||||
|
||||
url := "https://push2.eastmoney.com/api/qt/clist/get?cb=jQuery371047843066631541353_1745889398012&fs=%s&fields=f12,f13,f14,f19,f1,f2,f4,f3,f152,f17,f18,f15,f16,f5,f6&fid=f3&pn=%d&pz=%d&po=1&dect=1"
|
||||
url := "https://push2.eastmoney.com/api/qt/clist/get?np=1&fltt=1&invt=2&cb=data&fs=%s&fields=f12,f13,f14,f1,f2,f4,f3,f152,f5,f6,f7,f15,f18,f16,f17,f10,f8,f9,f23,f100,f265&fid=f3&pn=%d&pz=%d&po=1&dect=1&wbp2u=|0|0|0|web&_=%d"
|
||||
sprintfUrl := fmt.Sprintf(url, fs, page, pageSize, time.Now().UnixMilli())
|
||||
logger.SugaredLogger.Infof("url:%s", sprintfUrl)
|
||||
resp, err := receiver.client.SetTimeout(time.Duration(receiver.config.CrawlTimeOut)*time.Second).R().
|
||||
SetHeader("Host", "push2.eastmoney.com").
|
||||
SetHeader("Referer", "https://quote.eastmoney.com/center/gridlist.html").
|
||||
SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0").
|
||||
Get(fmt.Sprintf(url, fs, page, pageSize))
|
||||
Get(sprintfUrl)
|
||||
if err != nil {
|
||||
logger.SugaredLogger.Errorf("err:%s", err.Error())
|
||||
return
|
||||
}
|
||||
body := string(resp.Body())
|
||||
body = strutil.ReplaceWithMap(body, map[string]string{
|
||||
"jQuery371047843066631541353_1745889398012(": "",
|
||||
");": "",
|
||||
})
|
||||
js := "var d=" + body
|
||||
logger.SugaredLogger.Infof("resp:%s", body)
|
||||
vm := otto.New()
|
||||
_, err = vm.Run(js)
|
||||
_, err = vm.Run("var data = JSON.stringify(d);")
|
||||
value, err := vm.Get("data")
|
||||
data := make(map[string]any)
|
||||
err = json.Unmarshal([]byte(value.String()), &data)
|
||||
vm.Run("function data(res){return res};")
|
||||
val, err := vm.Run(body)
|
||||
if err != nil {
|
||||
logger.SugaredLogger.Errorf("vm.Run error:%v", err.Error())
|
||||
}
|
||||
value, _ := val.Object().Value().Export()
|
||||
marshal, err := json.Marshal(value)
|
||||
data := make(map[string]any)
|
||||
err = json.Unmarshal(marshal, &data)
|
||||
if err != nil {
|
||||
logger.SugaredLogger.Errorf("json:%s", value.String())
|
||||
logger.SugaredLogger.Errorf("json.Unmarshal error:%v", err.Error())
|
||||
}
|
||||
logger.SugaredLogger.Infof("resp:%s", data)
|
||||
if data["data"] != nil {
|
||||
datas := data["data"].(map[string]any)
|
||||
total := datas["total"].(float64)
|
||||
diff := datas["diff"].(map[string]any)
|
||||
diff := datas["diff"].([]any)
|
||||
logger.SugaredLogger.Infof("total:%d", int(total))
|
||||
for k, item := range diff {
|
||||
stock := item.(map[string]any)
|
||||
logger.SugaredLogger.Infof("k:%s,%s:%s", k, stock["f14"], stock["f12"])
|
||||
logger.SugaredLogger.Infof("k:%d,%s:%s:%s %s:%s", k, stock["f14"], stock["f12"], DCToTsCode(stock["f12"].(string)), stock["f100"], stock["f265"])
|
||||
|
||||
if market == "" {
|
||||
stockInfo := &StockBasic{
|
||||
Symbol: stock["f12"].(string),
|
||||
TsCode: DCToTsCode(stock["f12"].(string)),
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&StockBasic{}).Where("symbol = ?", stockInfo.Symbol).First(stockInfo)
|
||||
logger.SugaredLogger.Infof("stockInfo:%+v", stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&StockBasic{}).Create(stockInfo)
|
||||
} else {
|
||||
stockInfo = &StockBasic{
|
||||
Symbol: stock["f12"].(string),
|
||||
TsCode: DCToTsCode(stock["f12"].(string)),
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&StockBasic{}).Where("symbol = ?", stockInfo.Symbol).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if market == "hk" {
|
||||
stockInfo := &models.StockInfoHK{
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".HK",
|
||||
Name: stock["f14"].(string),
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".HK",
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoHK{}).Where("code = ?", stockInfo.Code).First(stockInfo)
|
||||
logger.SugaredLogger.Infof("stockInfo:%+v", stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&models.StockInfoHK{}).Create(stockInfo)
|
||||
} else {
|
||||
stockInfo = &models.StockInfoHK{
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".HK",
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoHK{}).Where("code = ?", stockInfo.Code).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
|
||||
if market == "us" {
|
||||
stockInfo := &models.StockInfoUS{
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".US",
|
||||
Name: stock["f14"].(string),
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".US",
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoUS{}).Where("code = ?", stockInfo.Code).First(stockInfo)
|
||||
logger.SugaredLogger.Infof("stockInfo:%+v", stockInfo)
|
||||
if stockInfo.ID == 0 {
|
||||
db.Dao.Model(&models.StockInfoUS{}).Create(stockInfo)
|
||||
} else {
|
||||
stockInfo = &models.StockInfoUS{
|
||||
Code: strutil.PadStart(stock["f12"].(string), 5, "0") + ".US",
|
||||
Name: stock["f14"].(string),
|
||||
BKName: stock["f100"].(string),
|
||||
BKCode: stock["f265"].(string),
|
||||
}
|
||||
db.Dao.Model(&models.StockInfoUS{}).Where("code = ?", stockInfo.Code).Updates(stockInfo)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1526,6 +1574,25 @@ func (receiver StockDataApi) getDCStockInfo(market string, page, pageSize int) {
|
||||
}
|
||||
}
|
||||
|
||||
func DCToTsCode(dcCode string) string {
|
||||
//北京证券交易所 8(83、87、88 等) 创新型中小企业(专精特新为主)
|
||||
//上海证券交易所 6(60、688 等) 大盘蓝筹、科创板(高新技术)
|
||||
//深圳证券交易所 0、3(000、002、30 等) 中小盘、创业板(成长型创新企业)
|
||||
switch dcCode[0:1] {
|
||||
case "8":
|
||||
return dcCode + ".BJ"
|
||||
case "9":
|
||||
return dcCode + ".BJ"
|
||||
case "6":
|
||||
return dcCode + ".SH"
|
||||
case "0":
|
||||
return dcCode + ".SZ"
|
||||
case "3":
|
||||
return dcCode + ".SZ"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (receiver StockDataApi) GetHKStockInfo(pageSize int) {
|
||||
url := "https://stock.gtimg.cn/data/hk_rank.php?board=main_all&metric=price&pageSize=%d&reqPage=1&order=desc&var_name=list_data"
|
||||
resp, err := receiver.client.SetTimeout(time.Duration(receiver.config.CrawlTimeOut)*time.Second).R().
|
||||
|
@ -110,7 +110,8 @@ func TestGetHKStockInfo(t *testing.T) {
|
||||
//NewStockDataApi().GetSinaHKStockInfo()
|
||||
//m:105,m:106,m:107 //美股
|
||||
//m:128+t:3,m:128+t:4,m:128+t:1,m:128+t:2 //港股
|
||||
for i := 1; i <= 592; i++ {
|
||||
//287 224 605
|
||||
for i := 1; i <= 605; i++ {
|
||||
NewStockDataApi().getDCStockInfo("us", i, 20)
|
||||
time.Sleep(time.Duration(random.RandInt(1, 3)) * time.Second)
|
||||
}
|
||||
|
@ -172,6 +172,8 @@ type StockInfoHK struct {
|
||||
FullName string `json:"fullName"`
|
||||
EName string `json:"eName"`
|
||||
IsDel soft_delete.DeletedAt `gorm:"softDelete:flag"`
|
||||
BKName string `json:"bk_name"`
|
||||
BKCode string `json:"bk_code"`
|
||||
}
|
||||
|
||||
func (receiver StockInfoHK) TableName() string {
|
||||
@ -187,6 +189,8 @@ type StockInfoUS struct {
|
||||
Exchange string `json:"exchange"`
|
||||
Type string `json:"type"`
|
||||
IsDel soft_delete.DeletedAt `gorm:"softDelete:flag"`
|
||||
BKName string `json:"bk_name"`
|
||||
BKCode string `json:"bk_code"`
|
||||
}
|
||||
|
||||
func (receiver StockInfoUS) TableName() string {
|
||||
|
221
backend/util/html_to_markdown.go
Normal file
221
backend/util/html_to_markdown.go
Normal file
@ -0,0 +1,221 @@
|
||||
package util
|
||||
|
||||
// @Author spark
|
||||
// @Date 2025/7/15 14:08
|
||||
// @Desc
|
||||
//-----------------------------------------------------------------------------------
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"golang.org/x/net/html"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HTMLNode 表示HTML文档中的一个节点
|
||||
type HTMLNode struct {
|
||||
Type html.NodeType
|
||||
Data string
|
||||
Attr []html.Attribute
|
||||
Children []*HTMLNode
|
||||
}
|
||||
|
||||
// HTMLToMarkdown 将HTML转换为Markdown
|
||||
func HTMLToMarkdown(htmlContent string) (string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlContent))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
root := parseHTMLNode(doc)
|
||||
var buf bytes.Buffer
|
||||
convertNode(&buf, root, 0)
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// parseHTMLNode 递归解析HTML节点
|
||||
func parseHTMLNode(n *html.Node) *HTMLNode {
|
||||
node := &HTMLNode{
|
||||
Type: n.Type,
|
||||
Data: n.Data,
|
||||
Attr: n.Attr,
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
node.Children = append(node.Children, parseHTMLNode(c))
|
||||
}
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
// convertNode 递归转换节点为Markdown
|
||||
func convertNode(buf *bytes.Buffer, node *HTMLNode, depth int) {
|
||||
switch node.Type {
|
||||
case html.ElementNode:
|
||||
convertElementNode(buf, node, depth)
|
||||
case html.TextNode:
|
||||
// 处理文本节点,去除多余的空白
|
||||
text := strings.TrimSpace(node.Data)
|
||||
if text != "" {
|
||||
buf.WriteString(text)
|
||||
}
|
||||
}
|
||||
|
||||
// 递归处理子节点
|
||||
for _, child := range node.Children {
|
||||
convertNode(buf, child, depth+1)
|
||||
}
|
||||
|
||||
// 处理需要在结束标签后添加内容的元素
|
||||
switch node.Data {
|
||||
case "p", "h1", "h2", "h3", "h4", "h5", "h6", "li":
|
||||
buf.WriteString("\n\n")
|
||||
case "blockquote":
|
||||
buf.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
// convertElementNode 转换元素节点为Markdown
|
||||
func convertElementNode(buf *bytes.Buffer, node *HTMLNode, depth int) {
|
||||
switch node.Data {
|
||||
case "h1":
|
||||
buf.WriteString("# ")
|
||||
case "h2":
|
||||
buf.WriteString("## ")
|
||||
case "h3":
|
||||
buf.WriteString("### ")
|
||||
case "h4":
|
||||
buf.WriteString("#### ")
|
||||
case "h5":
|
||||
buf.WriteString("##### ")
|
||||
case "h6":
|
||||
buf.WriteString("###### ")
|
||||
case "p":
|
||||
// 段落标签不需要特殊标记,直接处理内容
|
||||
case "strong", "b":
|
||||
buf.WriteString("**")
|
||||
case "em", "i":
|
||||
buf.WriteString("*")
|
||||
case "u":
|
||||
buf.WriteString("<u>")
|
||||
case "s", "del":
|
||||
buf.WriteString("~~")
|
||||
case "a":
|
||||
//href := getAttrValue(node.Attr, "href")
|
||||
buf.WriteString("[")
|
||||
case "img":
|
||||
src := getAttrValue(node.Attr, "src")
|
||||
alt := getAttrValue(node.Attr, "alt")
|
||||
buf.WriteString(fmt.Sprintf("", alt, src))
|
||||
case "ul":
|
||||
// 无序列表不需要特殊标记,子项会处理
|
||||
case "ol":
|
||||
// 有序列表不需要特殊标记,子项会处理
|
||||
case "li":
|
||||
if isParentListType(node, "ul") {
|
||||
buf.WriteString("- ")
|
||||
} else {
|
||||
// 计算当前列表项的序号
|
||||
index := 1
|
||||
if parent := findParentList(node); parent != nil {
|
||||
for i, sibling := range parent.Children {
|
||||
if sibling == node {
|
||||
index = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
buf.WriteString(fmt.Sprintf("%d. ", index))
|
||||
}
|
||||
case "blockquote":
|
||||
buf.WriteString("> ")
|
||||
case "code":
|
||||
if isParentPre(node) {
|
||||
// 父节点是pre,使用代码块
|
||||
buf.WriteString("\n```\n")
|
||||
} else {
|
||||
// 行内代码
|
||||
buf.WriteString("`")
|
||||
}
|
||||
case "pre":
|
||||
// 前置代码块由子节点code处理
|
||||
case "br":
|
||||
buf.WriteString("\n")
|
||||
case "hr":
|
||||
buf.WriteString("\n---\n")
|
||||
}
|
||||
|
||||
// 处理闭合标签
|
||||
if needsClosingTag(node.Data) {
|
||||
defer func() {
|
||||
switch node.Data {
|
||||
case "strong", "b":
|
||||
buf.WriteString("**")
|
||||
case "em", "i":
|
||||
buf.WriteString("*")
|
||||
case "u":
|
||||
buf.WriteString("</u>")
|
||||
case "s", "del":
|
||||
buf.WriteString("~~")
|
||||
case "a":
|
||||
href := getAttrValue(node.Attr, "href")
|
||||
buf.WriteString(fmt.Sprintf("](%s)", href))
|
||||
case "code":
|
||||
if isParentPre(node) {
|
||||
buf.WriteString("\n```\n")
|
||||
} else {
|
||||
buf.WriteString("`")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// getAttrValue 获取属性值
|
||||
func getAttrValue(attrs []html.Attribute, key string) string {
|
||||
for _, attr := range attrs {
|
||||
if attr.Key == key {
|
||||
return attr.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isParentListType 检查父节点是否为指定类型的列表
|
||||
func isParentListType(node *HTMLNode, listType string) bool {
|
||||
parent := findParentList(node)
|
||||
return parent != nil && parent.Data == listType
|
||||
}
|
||||
|
||||
// findParentList 查找父列表节点
|
||||
func findParentList(node *HTMLNode) *HTMLNode {
|
||||
// 简化实现,实际应该递归查找父节点
|
||||
if node.Type == html.ElementNode && (node.Data == "ul" || node.Data == "ol") {
|
||||
return node
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isParentPre 检查父节点是否为pre
|
||||
func isParentPre(node *HTMLNode) bool {
|
||||
if len(node.Children) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, child := range node.Children {
|
||||
if child.Type == html.ElementNode && child.Data == "pre" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// needsClosingTag 判断元素是否需要闭合标签
|
||||
func needsClosingTag(tag string) bool {
|
||||
switch tag {
|
||||
case "img", "br", "hr", "input", "meta", "link":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
6
backend/util/html_to_markdown_test.go
Normal file
6
backend/util/html_to_markdown_test.go
Normal file
@ -0,0 +1,6 @@
|
||||
package util
|
||||
|
||||
// @Author spark
|
||||
// @Date 2025/7/15 14:08
|
||||
// @Desc
|
||||
//-----------------------------------------------------------------------------------
|
@ -426,6 +426,8 @@ export namespace data {
|
||||
is_hs: string;
|
||||
act_name: string;
|
||||
act_ent_type: string;
|
||||
bk_name: string;
|
||||
bk_code: string;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new StockBasic(source);
|
||||
@ -454,6 +456,8 @@ export namespace data {
|
||||
this.is_hs = source["is_hs"];
|
||||
this.act_name = source["act_name"];
|
||||
this.act_ent_type = source["act_ent_type"];
|
||||
this.bk_name = source["bk_name"];
|
||||
this.bk_code = source["bk_code"];
|
||||
}
|
||||
|
||||
convertValues(a: any, classs: any, asMap: boolean = false): any {
|
||||
|
2
go.mod
2
go.mod
@ -21,6 +21,7 @@ require (
|
||||
github.com/tidwall/gjson v1.14.2
|
||||
github.com/wailsapp/wails/v2 v2.10.1
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/net v0.38.0
|
||||
golang.org/x/sys v0.31.0
|
||||
golang.org/x/text v0.23.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
@ -82,7 +83,6 @@ require (
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/crypto v0.36.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
gopkg.in/sourcemap.v1 v1.0.5 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.22.5 // indirect
|
||||
|
Loading…
x
Reference in New Issue
Block a user