other_mirrors/ wi
1
0
Fork 0
mirror of https://code.dumpstack.io/tools/wi.git synced 2024-11-14 06:19:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implements redirect support

This commit is contained in:
Mikhail Klementyev 2016-07-29 21:05:05 +03:00
parent 4e6b6796f3
commit a3ee60dcc3
1 changed files with 28 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
commands/commands.go
View File

@ -17,6 +17,7 @@ import (
"io/ioutil" "io/ioutil"
"log" "log"
"net/http" "net/http"
"net/url"
"github.com/jollheef/wi/storage" "github.com/jollheef/wi/storage"
@ -25,15 +26,15 @@ import (
"golang.org/x/net/html/charset" "golang.org/x/net/html/charset"
) )
func parseLink(db *sql.DB, oldPage, value string, req *http.Request) (htmlPage string, err error) { func parseLink(db *sql.DB, oldPage, value string, lastUrl *url.URL) (htmlPage string, err error) {
url, err := req.URL.Parse(value) linkUrl, err := lastUrl.Parse(value)
if err != nil { if err != nil {
return return
} }
linkNo, err := storage.GetLinkID(db, url.String()) linkNo, err := storage.GetLinkID(db, linkUrl.String())
if err != nil { if err != nil {
linkNo, err = storage.AddLink(db, url.String()) linkNo, err = storage.AddLink(db, linkUrl.String())
if err != nil { if err != nil {
return return
} }
@ -49,7 +50,7 @@ func parseLink(db *sql.DB, oldPage, value string, req *http.Request) (htmlPage s
return return
} }
func parseLinks(db *sql.DB, body []byte, req *http.Request) (htmlPage string, err error) { func parseLinks(db *sql.DB, body []byte, lastUrl *url.URL) (htmlPage string, err error) {
htmlPage = string(body) htmlPage = string(body)
z := html.NewTokenizer(bytes.NewReader(body)) z := html.NewTokenizer(bytes.NewReader(body))
@ -64,7 +65,7 @@ func parseLinks(db *sql.DB, body []byte, req *http.Request) (htmlPage string, er
key, value, moreAttr := z.TagAttr() key, value, moreAttr := z.TagAttr()
if string(key) == "href" { if string(key) == "href" {
htmlPage, err = parseLink(db, htmlPage, string(value), req) htmlPage, err = parseLink(db, htmlPage, string(value), lastUrl)
if err != nil { if err != nil {
return return
} }
@ -79,15 +80,22 @@ func parseLinks(db *sql.DB, body []byte, req *http.Request) (htmlPage string, er
return return
} }
func Get(db *sql.DB, url string) { func Get(db *sql.DB, linkUrl string) {
client := &http.Client{} client := &http.Client{}
if !strings.Contains(url, "://") { var lastUrl *url.URL
url = "https://" + url
client.CheckRedirect = func(r *http.Request, via []*http.Request) (err error) {
lastUrl = r.URL
return
}
if !strings.Contains(linkUrl, "://") {
linkUrl = "https://" + linkUrl
} }
// TODO Full url encoding // TODO Full url encoding
req, err := http.NewRequest("GET", strings.Replace(url, " ", "%20", -1), nil) req, err := http.NewRequest("GET", strings.Replace(linkUrl, " ", "%20", -1), nil)
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
@ -99,7 +107,11 @@ func Get(db *sql.DB, url string) {
log.Fatalln(err) log.Fatalln(err)
} }
storage.AddHistoryURL(db, url) if lastUrl == nil {
lastUrl = req.URL
}
storage.AddHistoryURL(db, linkUrl)
defer resp.Body.Close() defer resp.Body.Close()
@ -113,7 +125,7 @@ func Get(db *sql.DB, url string) {
log.Fatalln("IO error:", err) log.Fatalln("IO error:", err)
} }
htmlPage, err := parseLinks(db, body, req) htmlPage, err := parseLinks(db, body, lastUrl)
if err != nil { if err != nil {
log.Fatalln("Parse links error:", err) log.Fatalln("Parse links error:", err)
} }
@ -129,20 +141,20 @@ func Get(db *sql.DB, url string) {
func Link(db *sql.DB, linkID int64, fromHistory bool) { func Link(db *sql.DB, linkID int64, fromHistory bool) {
var url string var linkUrl string
var err error var err error
if fromHistory { if fromHistory {
url, err = storage.GetHistoryUrl(db, linkID) linkUrl, err = storage.GetHistoryUrl(db, linkID)
} else { } else {
url, err = storage.GetLink(db, linkID) linkUrl, err = storage.GetLink(db, linkID)
} }
if err != nil { if err != nil {
log.Fatalln("Get link/history url error:", err) log.Fatalln("Get link/history url error:", err)
} }
Get(db, url) Get(db, linkUrl)
} }
func History(db *sql.DB, argAmount, defaultAmount int64, all bool) { func History(db *sql.DB, argAmount, defaultAmount int64, all bool) {