package service import ( "bytes" "context" "encoding/json" "fmt" "os/exec" "regexp" "strconv" "strings" "time" "go-common/app/admin/main/apm/conf" "go-common/app/admin/main/apm/model/pprof" "go-common/library/log" xtime "go-common/library/time" ) var ( port = "2333" cpuURL = "%s/x/admin/apm/pprof/svg?name=%s&uri=%s&hostname=%s" msg = "点击查看详情(注:生成时间有延迟,如未显示,稍后重试)" ) // kind . const ( CPUPerformace = 1 // CPU性能图 CPUFrame = 2 // CPU火焰图 HeapPerformance = 3 // 内存性能图 HeapFrame = 4 // 内存火焰图 ) // Pprof ... func (s *Service) Pprof(url, uri, svgName, hostName string, time int64, sType int8) (err error) { var ( out bytes.Buffer errOut bytes.Buffer ) goPath := "go" if len(conf.Conf.Pprof.GoPath) > 0 { goPath = conf.Conf.Pprof.GoPath } f, err := exec.LookPath(goPath) if err != nil { log.Error("pprof go error(%v) goPath=(%v)", err, goPath) fmt.Printf("pprof=(%v)", err) return } cmd := exec.Command(f, "tool", "pprof", "--seconds="+strconv.FormatInt(time, 10), "--svg", "--output="+conf.Conf.Pprof.Dir+"/"+svgName+"_"+hostName+"_"+uri+".svg", url+"/debug/pprof/"+uri) cmd.Stdout = &out cmd.Stderr = &errOut // 执行命令 if sType == 1 { //串行 if err = cmd.Run(); err != nil { log.Error("pprof Run stdout=(%s) stderr=(%s) error(%v)", out.String(), errOut.String(), err) } } else { //阻塞 if err = cmd.Start(); err != nil { log.Error("pprof Start stdout=(%s) stderr=(%s) error(%v)", out.String(), errOut.String(), err) } if err = cmd.Wait(); err != nil { log.Error("s.Pprof cmd.Wait() error(%v)", err) } } return } // Torch ... func (s *Service) Torch(c context.Context, url, uri, svgName, hostName string, time int64, sType int8) (err error) { goPath := "go-torch" // if len(conf.Conf.Pprof.GoPath) > 0 { // goPath = conf.Conf.Pprof.GoPath // } f, err := exec.LookPath(goPath) if err != nil { log.Error("go-torch error(%v) goPath=(%v)", err, goPath) fmt.Printf("go-torch=(%v)", err) return } cmd := exec.Command(f, "--url="+url, "--suffix=/debug/pprof/"+uri, "--seconds="+strconv.FormatInt(time, 10), "-f="+conf.Conf.Pprof.Dir+"/"+svgName+"_"+hostName+"_"+uri+"_flame.svg") var ( out bytes.Buffer cmdErr bytes.Buffer ) cmd.Stdout = &out cmd.Stderr = &cmdErr // 执行命令 if sType == 1 { //串行 if err = cmd.Run(); err != nil { log.Error("go-torch Run stdout=(%s) stderr=(%s) error(%v)", out.String(), cmdErr.String(), err) } } else { //阻塞 if err = cmd.Start(); err != nil { log.Error("go-torch Start stdout=(%s) stderr=(%s) error(%v)", out.String(), cmdErr.String(), err) } if err = cmd.Wait(); err != nil { log.Error("cmd.Wait() error(%v)", err) } } return } // ActiveWarning active func (s *Service) ActiveWarning(c context.Context, text string) (err error) { var ( ins *pprof.Ins title = "【%s】性能告警抓取通知" warn = &pprof.Warning{} pws = make([]*pprof.Warn, 0) times = time.Now().Unix() curTime = xtime.Time(times) reg = regexp.MustCompile(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`) ) if err = json.Unmarshal([]byte(text), warn); err != nil { log.Error("s.ActiveWarning json.Unmarshal data error(%v)", err) return } if warn.Tags.App == "" { log.Info("s.ActiveWarning skipped(AppName is empty)") return } title = fmt.Sprintf(title, warn.Tags.App) if ins, err = s.dao.Instances(c, warn.Tags.App); err != nil { log.Error("s.ActiveWaring get instances error(%v)", err) return } for _, instance := range ins.Instances { var ( ip string addrs = instance.Addrs strs = strings.Split(instance.Hostname, "-") hostName = fmt.Sprintf("%s-%d", strs[len(strs)-1], times) pprofWarn = &pprof.Warn{} ) if len(addrs) < 1 { log.Info("s.ActiveWarning not found adds") continue } ip = reg.FindString(addrs[0]) host := fmt.Sprintf("http://%s:%s", ip, port) pprofWarn.IP = ip pprofWarn.AppID = warn.Tags.App pprofWarn.SvgName = hostName pprofWarn.Ctime = curTime pprofWarn.Mtime = curTime if err = s.Torch(c, host, "profile", warn.Tags.App, hostName, 30, 2); err == nil { pprofWarn.Kind = CPUPerformace pws = append(pws, packing(pprofWarn)) } if err = s.Pprof(host, "profile", warn.Tags.App, hostName, 30, 2); err == nil { pprofWarn.Kind = CPUFrame pws = append(pws, packing(pprofWarn)) } if err = s.Torch(c, host, "heap", warn.Tags.App, hostName, 30, 2); err == nil { pprofWarn.Kind = HeapPerformance pws = append(pws, packing(pprofWarn)) } if err = s.Pprof(host, "heap", warn.Tags.App, hostName, 30, 2); err == nil { pprofWarn.Kind = HeapFrame pws = append(pws, packing(pprofWarn)) } } if len(pws) == 0 { return } if err = s.AddPprofWarn(c, pws); err != nil { return } return s.SendWeChat(c, title, fmt.Sprintf(msg, s.c.Host.SVENCo), warn.Tags.App, strings.Join(s.c.WeChat.Users, ",")) } // packing . func packing(pw *pprof.Warn) (pprofWarn *pprof.Warn) { pprofWarn = &pprof.Warn{ AppID: pw.AppID, SvgName: pw.SvgName, IP: pw.IP, Kind: pw.Kind, Mtime: pw.Mtime, Ctime: pw.Ctime, } return } // AddPprofWarn . func (s *Service) AddPprofWarn(c context.Context, pws []*pprof.Warn) (err error) { var ( sql = "INSERT INTO `pprof_warn`(`app_id`, `svg_name`, `ip`, `kind`, `ctime`, `mtime`) VALUES %s" key = make([]string, 0) value = make([]interface{}, 0) ) for _, pw := range pws { key = append(key, "(?,?,?,?,?,?)") value = append(value, pw.AppID, pw.SvgName, pw.IP, pw.Kind, pw.Ctime, pw.Mtime) } if err = s.DB.Exec(fmt.Sprintf(sql, strings.Join(key, ",")), value...).Error; err != nil { log.Error("s.AddPprofWarn error(%v)", err) } return } // PprofWarn . func (s *Service) PprofWarn(c context.Context, req *pprof.Params) (pws []*pprof.Warn, err error) { var ( query = s.DB.Where("1=1") ) pws = make([]*pprof.Warn, 0) if req.AppID == "" && req.IP == "" && req.SvgName == "" && req.Kind == 0 && req.StartTime == 0 && req.EndTime == 0 { return } if req.AppID != "" { query = query.Where("app_id=?", req.AppID) } if req.SvgName != "" { query = query.Where("svg_name=?", req.SvgName) } if req.IP != "" { query = query.Where("ip=?", req.IP) } if req.Kind != 0 { query = query.Where("kind=?", req.Kind) } if req.StartTime != 0 && req.EndTime != 0 { query = query.Where("mtime between ? and ?", req.StartTime, req.EndTime) } if err = query.Order("mtime desc").Find(&pws).Error; err != nil { log.Error("s.PprofWarn query error(%v)", err) } s.setSvgURL(pws) return } // setSvgURL . func (s *Service) setSvgURL(pws []*pprof.Warn) { for _, pw := range pws { switch { case pw.Kind == CPUPerformace: pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "profile", pw.SvgName) case pw.Kind == CPUFrame: pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "profile_flame", pw.SvgName) case pw.Kind == HeapPerformance: pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "heap", pw.SvgName) case pw.Kind == HeapFrame: pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "heap_flame", pw.SvgName) default: } } }