implement file retrive with scraping

pages/repo.go

@@ -48,7 +48,6 @@ func HandleRepo(c *fiber.Ctx) error {
 			"error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found",
 		})
 	}
-	// API
 
 	// Scraping
 	Scrape := Repo{}
@@ -81,24 +80,28 @@ func HandleRepo(c *fiber.Ctx) error {
 	sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) {
 		Scrape.DefaultBranch = e.ChildText("span.css-truncate-target")
 	})
+	sc.OnHTML("div.js-details-container div.Details-content--hidden-not-important", func(e *colly.HTMLElement) {
+		e.ForEach("div.js-navigation-item", func(i int, el *colly.HTMLElement) {
+			var FileType string
+			if el.ChildAttr("div.flex-shrink-0 svg", "aria-label") == "Directory" {
+				FileType = "dir"
+			} else {
+				FileType = "file"
+			}
+			repoFilesArray = append(repoFilesArray, RepoFiles{
+				Name:          el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
+				Path:          el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
+				Type:          FileType,
+				Fullname:      Scrape.Fullname,
+				DefaultBranch: Scrape.DefaultBranch,
+			})
+		})
+	})
 
 	sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/")
 	// Add scrape-based info to repoArray
 	repoArray = append(repoArray, Scrape)
 
-	// API
-	repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents")
-	bruh := repoFiles.Get("#.@pretty").Array()
-	for _, item := range bruh {
-		repoFilesArray = append(repoFilesArray, RepoFiles{
-			Name:          item.Get("path").String(),
-			Path:          item.Get("path").String(),
-			Type:          item.Get("type").String(),
-			Fullname:      Scrape.Fullname,
-			DefaultBranch: Scrape.DefaultBranch,
-		})
-	}
-
 	// README
 	var readmee string
 	err := requests.