scrape.goも従う
このコミットが含まれているのは:
コミット
6fa86399d9
|
@ -0,0 +1,68 @@
|
||||||
|
package src
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/gocolly/colly"
|
||||||
|
)
|
||||||
|
|
||||||
|
func getRouteDetail(e *colly.HTMLElement) Route {
|
||||||
|
r := Route{}
|
||||||
|
e.ForEach("div.routeSummary div ul.priority li span", func (j int, el *colly.HTMLElement) {
|
||||||
|
if el.Attr("class") == "icnPriTime" {
|
||||||
|
r.Badges = append(r.Badges, 1)
|
||||||
|
}
|
||||||
|
if el.Attr("class") == "icnPriFare" {
|
||||||
|
r.Badges = append(r.Badges, 2)
|
||||||
|
}
|
||||||
|
if el.Attr("class") == "icnPriTrans" {
|
||||||
|
r.Badges = append(r.Badges, 3)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
base := e.ChildText("ul.summary li.time span")
|
||||||
|
time := strings.ReplaceAll(base, e.ChildText("ul.summary li.time span.small"), "")
|
||||||
|
time2 := strings.Split(time, "着")
|
||||||
|
r.Time = time2[0] + "着"
|
||||||
|
durabase := e.ChildText("ul.summary li.time")
|
||||||
|
durasi := strings.Index(durabase, "着") + len("着")
|
||||||
|
duraei := strings.Index(durabase[durasi:], "分") + len("分") + durasi
|
||||||
|
|
||||||
|
r.Duration = durabase[durasi:duraei]
|
||||||
|
r.TransitCunt = strings.ReplaceAll(e.ChildText("ul.summary li.transfer"), "乗換:", "")
|
||||||
|
r.Fare = strings.ReplaceAll(e.ChildText("ul.summary li.fare"), "[priic]IC優先:", "")
|
||||||
|
Stations := Station{}
|
||||||
|
Fares := Fare{}
|
||||||
|
Stops := Stop{}
|
||||||
|
e.ForEach("div.routeDetail div.station", func (j int, el *colly.HTMLElement) {
|
||||||
|
Stations.Time = el.ChildText("ul.time li")
|
||||||
|
if el.ChildText("p.icon span") == "[dep]" { Stations.Time += "発" }
|
||||||
|
if el.ChildText("p.icon span") == "[arr]" { Stations.Time += "着" }
|
||||||
|
Stations.Name = el.ChildText("dl dt a")
|
||||||
|
e.ForEach("div.routeDetail div.fareSection div.access", func (jf int, elf *colly.HTMLElement) {
|
||||||
|
Fares.Stops = nil
|
||||||
|
if jf == j {
|
||||||
|
Fares.Train = strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(elf.ChildText("li.transport div"), "[train]", "【電車】"), "[bus]", "【バス】"), "[air]", "【空路】"), "当駅始発", "【当駅始発】") + " "
|
||||||
|
|
||||||
|
Fares.Platform = elf.ChildText("li.platform")
|
||||||
|
Fares.Color = strings.ReplaceAll(elf.ChildAttr("span", "style"), "border-color:#", "")
|
||||||
|
elf.ForEach("li.stop ul", func (js int, els *colly.HTMLElement) {
|
||||||
|
Stops.Time = els.ChildText("li dl dt")
|
||||||
|
Stops.Name = strings.ReplaceAll(els.ChildText("li dl dd"), "○", "")
|
||||||
|
Fares.Stops = append(Fares.Stops, Stops)
|
||||||
|
})
|
||||||
|
Stations.Fares = append(Stations.Fares, Fares)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
e.ForEach("div.routeDetail div.walk ul.info", func (jw int, elw *colly.HTMLElement) {
|
||||||
|
if jw == j {
|
||||||
|
Fares.Train = strings.ReplaceAll(elw.ChildText("li.transport"), "[line][walk]", "")
|
||||||
|
Fares.Platform = ""
|
||||||
|
Fares.Color = "a8a8a8"
|
||||||
|
Stations.Fares = append(Stations.Fares, Fares)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
r.Stations = append(r.Stations, Stations)
|
||||||
|
})
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
|
@ -6,13 +6,15 @@ import (
|
||||||
"time"
|
"time"
|
||||||
"net"
|
"net"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/gocolly/colly"
|
"github.com/gocolly/colly"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Scrape (gurl string) []Route {
|
func Scrape (gurl string) []Route {
|
||||||
ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
|
ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) " +
|
||||||
|
"Chrome/110.0.0.0 Safari/537.36"
|
||||||
|
|
||||||
sc := colly.NewCollector(
|
sc := colly.NewCollector(
|
||||||
colly.AllowURLRevisit(),
|
colly.AllowURLRevisit(),
|
||||||
colly.Async(true),
|
colly.Async(true),
|
||||||
|
@ -34,7 +36,10 @@ func Scrape (gurl string) []Route {
|
||||||
|
|
||||||
sc.OnRequest(func(r *colly.Request) {
|
sc.OnRequest(func(r *colly.Request) {
|
||||||
r.Headers.Set("User-Agent", ua)
|
r.Headers.Set("User-Agent", ua)
|
||||||
r.Headers.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
r.Headers.Set(
|
||||||
|
"Accept",
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
)
|
||||||
r.Headers.Set("Accept-Language", "en-US,en;q=0.5")
|
r.Headers.Set("Accept-Language", "en-US,en;q=0.5")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -47,63 +52,7 @@ func Scrape (gurl string) []Route {
|
||||||
for i := 1; i <= 3; i++ {
|
for i := 1; i <= 3; i++ {
|
||||||
route := fmt.Sprintf("div#route%02d", i)
|
route := fmt.Sprintf("div#route%02d", i)
|
||||||
sc.OnHTML("div.elmRouteDetail " + route, func (e *colly.HTMLElement) {
|
sc.OnHTML("div.elmRouteDetail " + route, func (e *colly.HTMLElement) {
|
||||||
Routes := Route{}
|
Routes := getRouteDetail(e)
|
||||||
e.ForEach("div.routeSummary div ul.priority li span", func (j int, el *colly.HTMLElement) {
|
|
||||||
if el.Attr("class") == "icnPriTime" {
|
|
||||||
Routes.Badges = append(Routes.Badges, 1)
|
|
||||||
}
|
|
||||||
if el.Attr("class") == "icnPriFare" {
|
|
||||||
Routes.Badges = append(Routes.Badges, 2)
|
|
||||||
}
|
|
||||||
if el.Attr("class") == "icnPriTrans" {
|
|
||||||
Routes.Badges = append(Routes.Badges, 3)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
base := e.ChildText("ul.summary li.time span")
|
|
||||||
time := strings.ReplaceAll(base, e.ChildText("ul.summary li.time span.small"), "")
|
|
||||||
time2 := strings.Split(time, "着")
|
|
||||||
Routes.Time = time2[0] + "着"
|
|
||||||
durabase := e.ChildText("ul.summary li.time")
|
|
||||||
durasi := strings.Index(durabase, "着") + len("着")
|
|
||||||
duraei := strings.Index(durabase[durasi:], "分") + len("分") + durasi
|
|
||||||
|
|
||||||
Routes.Duration = durabase[durasi:duraei]
|
|
||||||
Routes.TransitCunt = strings.ReplaceAll(e.ChildText("ul.summary li.transfer"), "乗換:", "")
|
|
||||||
Routes.Fare = strings.ReplaceAll(e.ChildText("ul.summary li.fare"), "[priic]IC優先:", "")
|
|
||||||
Stations := Station{}
|
|
||||||
Fares := Fare{}
|
|
||||||
Stops := Stop{}
|
|
||||||
e.ForEach("div.routeDetail div.station", func (j int, el *colly.HTMLElement) {
|
|
||||||
Stations.Time = el.ChildText("ul.time li")
|
|
||||||
if el.ChildText("p.icon span") == "[dep]" { Stations.Time += "発" }
|
|
||||||
if el.ChildText("p.icon span") == "[arr]" { Stations.Time += "着" }
|
|
||||||
Stations.Name = el.ChildText("dl dt a")
|
|
||||||
e.ForEach("div.routeDetail div.fareSection div.access", func (jf int, elf *colly.HTMLElement) {
|
|
||||||
Fares.Stops = nil
|
|
||||||
if jf == j {
|
|
||||||
Fares.Train = strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(elf.ChildText("li.transport div"), "[train]", "【電車】"), "[bus]", "【バス】"), "[air]", "【空路】"), "当駅始発", "【当駅始発】") + " "
|
|
||||||
|
|
||||||
Fares.Platform = elf.ChildText("li.platform")
|
|
||||||
Fares.Color = strings.ReplaceAll(elf.ChildAttr("span", "style"), "border-color:#", "")
|
|
||||||
elf.ForEach("li.stop ul", func (js int, els *colly.HTMLElement) {
|
|
||||||
Stops.Time = els.ChildText("li dl dt")
|
|
||||||
Stops.Name = strings.ReplaceAll(els.ChildText("li dl dd"), "○", "")
|
|
||||||
Fares.Stops = append(Fares.Stops, Stops)
|
|
||||||
})
|
|
||||||
Stations.Fares = append(Stations.Fares, Fares)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
e.ForEach("div.routeDetail div.walk ul.info", func (jw int, elw *colly.HTMLElement) {
|
|
||||||
if jw == j {
|
|
||||||
Fares.Train = strings.ReplaceAll(elw.ChildText("li.transport"), "[line][walk]", "")
|
|
||||||
Fares.Platform = ""
|
|
||||||
Fares.Color = "a8a8a8"
|
|
||||||
Stations.Fares = append(Stations.Fares, Fares)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
Routes.Stations = append(Routes.Stations, Stations)
|
|
||||||
})
|
|
||||||
|
|
||||||
routeArr = append(routeArr, Routes)
|
routeArr = append(routeArr, Routes)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
読み込み中…
新しいイシューから参照