implement file retrive with scraping
このコミットが含まれているのは:
コミット
86f5f72f7c
|
@ -48,7 +48,6 @@ func HandleRepo(c *fiber.Ctx) error {
|
|||
"error": "Repository " + c.Params("user") + "/" + c.Params("repo") + " not found",
|
||||
})
|
||||
}
|
||||
// API
|
||||
|
||||
// Scraping
|
||||
Scrape := Repo{}
|
||||
|
@ -81,24 +80,28 @@ func HandleRepo(c *fiber.Ctx) error {
|
|||
sc.OnHTML("summary[title*='Switch branches or tags']", func(e *colly.HTMLElement) {
|
||||
Scrape.DefaultBranch = e.ChildText("span.css-truncate-target")
|
||||
})
|
||||
sc.OnHTML("div.js-details-container div.Details-content--hidden-not-important", func(e *colly.HTMLElement) {
|
||||
e.ForEach("div.js-navigation-item", func(i int, el *colly.HTMLElement) {
|
||||
var FileType string
|
||||
if el.ChildAttr("div.flex-shrink-0 svg", "aria-label") == "Directory" {
|
||||
FileType = "dir"
|
||||
} else {
|
||||
FileType = "file"
|
||||
}
|
||||
repoFilesArray = append(repoFilesArray, RepoFiles{
|
||||
Name: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
|
||||
Path: el.ChildText("div.flex-auto span.d-block a.js-navigation-open"),
|
||||
Type: FileType,
|
||||
Fullname: Scrape.Fullname,
|
||||
DefaultBranch: Scrape.DefaultBranch,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
sc.Visit("https://github.com/" + c.Params("user") + "/" + c.Params("repo") + "/")
|
||||
// Add scrape-based info to repoArray
|
||||
repoArray = append(repoArray, Scrape)
|
||||
|
||||
// API
|
||||
repoFiles := utils.GetRequest("https://api.github.com/repos/" + c.Params("user") + "/" + c.Params("repo") + "/contents")
|
||||
bruh := repoFiles.Get("#.@pretty").Array()
|
||||
for _, item := range bruh {
|
||||
repoFilesArray = append(repoFilesArray, RepoFiles{
|
||||
Name: item.Get("path").String(),
|
||||
Path: item.Get("path").String(),
|
||||
Type: item.Get("type").String(),
|
||||
Fullname: Scrape.Fullname,
|
||||
DefaultBranch: Scrape.DefaultBranch,
|
||||
})
|
||||
}
|
||||
|
||||
// README
|
||||
var readmee string
|
||||
err := requests.
|
||||
|
|
読み込み中…
新しいイシューから参照