package pages import ( "bufio" "bytes" "gitler.moe/suwako/gitlin/utils" "context" htmlfmt "github.com/alecthomas/chroma/v2/formatters/html" "github.com/alecthomas/chroma/v2/lexers" "github.com/alecthomas/chroma/v2/styles" "github.com/carlmjohnson/requests" "github.com/gocolly/colly" "github.com/gofiber/fiber/v2" "html/template" "log" "net/http" "os" "path/filepath" "strings" ) type Gist struct { Fullname string Refname string Description string Revision string Parent string Stars string Forks string } type GistFiles struct { Name string File template.HTML Css template.CSS } func HandleGist(c *fiber.Ctx) error { var gistArray []Gist var gistFilesArray []GistFiles resp, statusErr := http.Get("https://gists.github.com/" + c.Params("user") + "/" + c.Params("gistID")) if statusErr != nil { log.Println(statusErr) } if resp.StatusCode == 404 { // I need a better way to do this return c.Status(404).Render("error", fiber.Map{ "title": "Error", "ver": utils.Ver, "ves": utils.Ves, "error": "Gist repository " + c.Params("user") + "/" + c.Params("gistID") + " not found", }) } // Scraping Scrape := Gist{} UserAgent, ok := os.LookupEnv("GITLIN_USER_AGENT") if !ok { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" } sc := colly.NewCollector(colly.AllowedDomains("gist.github.com"), colly.UserAgent(UserAgent)) sc.OnHTML("div.gisthead", func(e *colly.HTMLElement) { Scrape.Refname = c.Params("user") + "/" + c.Params("gistID") Scrape.Fullname = c.Params("user") + "/" + e.ChildText("a[href='/"+c.Params("user")+"/"+c.Params("gistID")+"' i]") dl := e.ChildAttr("a[href$='.zip']", "href") Scrape.Revision = dl[strings.LastIndex(dl, "/")+1 : len(dl)-len(".zip")] Scrape.Stars = e.ChildText("li a[href*='/" + c.Params("user") + "/" + c.Params("gistID") + "/stargazers' i]") Scrape.Parent = e.ChildText("span.note a") Scrape.Forks = e.ChildText("li span.social-count") // I need a better way to do this if Scrape.Forks == "" { Scrape.Forks = e.ChildText("li a[href*='/" + c.Params("user") + "/" + c.Params("gistID") + "/forks' i]") } }) sc.OnHTML("div[itemprop='about']", func(e *colly.HTMLElement) { Scrape.Description = e.Text }) sc.OnHTML("div.repository-content.gist-content", func(e *colly.HTMLElement) { e.ForEach("div.file-header", func(i int, el *colly.HTMLElement) { var file string url := "https://gist.github.com" + el.ChildAttr("div.file-actions a.Button", "href") err := requests. URL(url). ToString(&file). Fetch(context.Background()) if err != nil { log.Println(err) } lexer := lexers.Match(el.ChildText("strong.user-select-contain.gist-blob-name.css-truncate-target")) if lexer == nil { lexer = lexers.Fallback } formatter := htmlfmt.New( htmlfmt.WithClasses(true), htmlfmt.ClassPrefix(strings.TrimPrefix(filepath.Ext(el.ChildText("strong.user-select-contain.gist-blob-name.css-truncate-target")), ".")), htmlfmt.WithLineNumbers(true), htmlfmt.PreventSurroundingPre(false), ) buf := bytes.Buffer{} writer := bufio.NewWriter(&buf) style := styles.Get("native") tokens, err := lexer.Tokenise(nil, file) if err != nil { log.Println(err) } formatter.Format(writer, style, tokens) cssbuf := bytes.Buffer{} cssw := bufio.NewWriter(&cssbuf) formatter.WriteCSS(cssw, style) cssw.Flush() writer.Flush() fileFmt := template.HTML(buf.String()) cssFmt := template.CSS(cssbuf.String()) gistFilesArray = append(gistFilesArray, GistFiles{ Name: el.ChildText("strong.user-select-contain.gist-blob-name.css-truncate-target"), File: fileFmt, Css: cssFmt, }) }) }) sc.Visit("https://gist.github.com/" + c.Params("user") + "/" + c.Params("gistID") + "/") // Add scrape-based info to gistArray gistArray = append(gistArray, Scrape) return c.Render("gist", fiber.Map{ "title": "Repository " + c.Params("user") + "/" + c.Params("gistID"), "ver": utils.Ver, "ves": utils.Ves, "gist": gistArray, "files": gistFilesArray, }) }