wi/commands/commands.go

295 lines
5.5 KiB
Go
Raw Normal View History

2016-07-25 18:35:09 +03:00
/**
* @file commands.go
* @author Mikhail Klementyev jollheef<AT>riseup.net
* @license GNU GPLv3
* @date July, 2016
2016-07-28 22:43:53 +03:00
* @brief Command line options ('wi (get|link|...)')
2016-07-25 18:35:09 +03:00
*/
package commands
import (
"database/sql"
"strings"
"fmt"
"log"
"net/http"
2016-07-29 21:05:05 +03:00
"net/url"
2016-07-25 18:35:09 +03:00
"github.com/jollheef/wi/storage"
2016-11-20 12:22:00 +02:00
"github.com/PuerkitoBio/goquery"
2016-07-25 18:35:09 +03:00
"github.com/jaytaylor/html2text"
2016-11-20 16:57:27 +02:00
cookiejar "github.com/juju/persistent-cookiejar"
2016-07-25 18:35:09 +03:00
"golang.org/x/net/html/charset"
)
2016-11-20 12:22:00 +02:00
func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
doc.Find("a").Each(func(i int, s *goquery.Selection) {
url, exists := s.Attr("href")
if !exists {
return
}
linkUrl, err := pageUrl.Parse(url)
2016-07-25 18:35:09 +03:00
if err != nil {
return
}
2016-11-20 12:22:00 +02:00
linkNo, err := storage.GetLinkID(db, linkUrl.String())
if err != nil {
linkNo, err = storage.AddLink(db, linkUrl.String())
if err != nil {
log.Fatalln("Add link:", err)
}
}
2016-07-26 00:04:32 +03:00
2016-11-20 12:22:00 +02:00
s.SetAttr("href", fmt.Sprintf("%d", linkNo))
})
2016-07-25 18:35:09 +03:00
return
}
2016-11-20 12:22:00 +02:00
func fixForms(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
doc.Find("form").Each(func(i int, s *goquery.Selection) {
var fields []storage.Field
s.Find("input").Map(
func(i int, s *goquery.Selection) (str string) {
f := storage.Field{}
var exists bool
f.Name, exists = s.Attr("name")
if !exists {
return
}
f.Value, _ = s.Attr("value")
hidden, _ := s.Attr("type")
if hidden == "hidden" {
f.Hidden = true
}
fields = append(fields, f)
return
})
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
action, _ := s.Attr("action")
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
actionUrl, err := pageUrl.Parse(action)
if err != nil {
return
}
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
method, _ := s.Attr("method")
2016-07-25 18:35:09 +03:00
2016-11-20 12:22:00 +02:00
formNo, err := storage.GetFormID(db, fields, actionUrl.String(), method)
if err != nil {
formNo, err = storage.AddForm(db, fields, actionUrl.String(), method)
if err != nil {
log.Fatalln(err)
2016-07-25 18:35:09 +03:00
}
}
2016-11-20 12:22:00 +02:00
s.AppendHtml(fmt.Sprintf("(%d %s)", formNo, strings.ToUpper(method)))
})
2016-07-25 18:35:09 +03:00
return
}
2016-11-20 17:32:09 +02:00
func handleResponse(db *sql.DB, resp *http.Response, lastUrl *url.URL) {
utf8, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
if err != nil {
log.Fatalln("Encoding error:", err)
}
doc, err := goquery.NewDocumentFromReader(utf8)
if err != nil {
log.Fatalln("Create document error:", err)
}
err = fixLinks(db, doc, lastUrl)
if err != nil {
log.Fatalln("Fix links error:", err)
}
err = fixForms(db, doc, lastUrl)
if err != nil {
log.Fatalln("Fix forms error", err)
}
htmlPage, err := doc.Html()
if err != nil {
log.Fatalln("Convert to html error:", err)
}
text, err := html2text.FromString(htmlPage)
if err != nil {
log.Fatalln("Html to text error:", err)
}
text += ""
fmt.Println(text)
}
2016-11-20 16:57:27 +02:00
func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
client := &http.Client{Jar: jar}
2016-07-25 18:35:09 +03:00
2016-07-29 21:05:05 +03:00
var lastUrl *url.URL
client.CheckRedirect = func(r *http.Request, via []*http.Request) (err error) {
lastUrl = r.URL
return
}
if !strings.Contains(linkUrl, "://") {
linkUrl = "https://" + linkUrl
2016-07-25 18:35:09 +03:00
}
2016-11-20 12:22:00 +02:00
u, err := url.Parse(linkUrl)
if err != nil {
log.Fatalln(err)
}
req, err := http.NewRequest("GET", u.String(), nil)
2016-07-25 18:35:09 +03:00
if err != nil {
log.Fatalln(err)
}
req.Header.Set("User-Agent", "Wi 0.0")
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
2016-07-29 21:05:05 +03:00
if lastUrl == nil {
lastUrl = req.URL
}
storage.AddHistoryURL(db, linkUrl)
2016-07-25 18:35:09 +03:00
defer resp.Body.Close()
2016-11-20 17:32:09 +02:00
handleResponse(db, resp, lastUrl)
2016-07-25 18:35:09 +03:00
}
2016-11-20 16:57:27 +02:00
func Form(db *sql.DB, jar *cookiejar.Jar, formID int64, formArgs []string) {
2016-11-20 12:22:00 +02:00
fields, formUrl, post, err := storage.GetForm(db, formID)
if err != nil {
log.Fatalln("Get form:", err)
}
if len(formArgs) == 0 {
if post {
fmt.Print("POST ")
}
fmt.Println(formUrl)
fmt.Print("Values: ")
for i, f := range fields {
if i != 0 {
fmt.Print("\n\t")
}
fmt.Printf(`%s="%s"`, f.Name, f.Value)
}
fmt.Println()
return
}
urlData := url.Values{}
for _, f := range fields {
urlData.Set(f.Name, f.Value)
}
for _, fa := range formArgs {
2016-11-20 15:17:06 +02:00
nameAndValue := strings.Split(fa, "=")
2016-11-20 12:22:00 +02:00
if len(nameAndValue) != 2 {
continue
}
name := nameAndValue[0]
value := nameAndValue[1]
urlData.Set(name, value)
}
2016-11-20 16:57:27 +02:00
client := &http.Client{Jar: jar}
2016-11-20 12:22:00 +02:00
var lastUrl *url.URL
client.CheckRedirect = func(r *http.Request, via []*http.Request) (err error) {
lastUrl = r.URL
return
}
resp, err := client.PostForm(formUrl, urlData)
if err != nil {
fmt.Println(err)
}
if lastUrl == nil {
lastUrl, _ = resp.Location()
}
log.Println(resp.Status)
var status int64
fmt.Sscanf(resp.Status, "%d", &status)
if status >= 300 && status < 400 {
2016-11-20 16:57:27 +02:00
Get(db, jar, lastUrl.String())
2016-11-20 17:32:09 +02:00
return
2016-11-20 12:22:00 +02:00
}
2016-11-20 17:32:09 +02:00
defer resp.Body.Close()
handleResponse(db, resp, lastUrl)
2016-08-01 03:11:48 +03:00
}
2016-11-20 16:57:27 +02:00
func Link(db *sql.DB, jar *cookiejar.Jar, linkID int64, fromHistory bool) {
2016-07-25 18:41:12 +03:00
2016-07-29 21:05:05 +03:00
var linkUrl string
2016-07-25 18:41:12 +03:00
var err error
if fromHistory {
2016-07-29 21:05:05 +03:00
linkUrl, err = storage.GetHistoryUrl(db, linkID)
2016-07-25 18:41:12 +03:00
} else {
2016-07-29 21:05:05 +03:00
linkUrl, err = storage.GetLink(db, linkID)
2016-07-25 18:41:12 +03:00
}
2016-07-25 18:35:09 +03:00
if err != nil {
2016-07-25 23:24:25 +03:00
log.Fatalln("Get link/history url error:", err)
2016-07-25 18:35:09 +03:00
}
2016-11-20 16:57:27 +02:00
Get(db, jar, linkUrl)
2016-07-25 18:35:09 +03:00
}
func History(db *sql.DB, argAmount, defaultAmount int64, all bool) {
history, err := storage.GetHistory(db)
if err != nil {
2016-07-25 23:24:25 +03:00
log.Fatalln("Get history error:", err)
2016-07-25 18:35:09 +03:00
}
var amount int64
if all {
amount = int64(len(history))
} else if argAmount == 0 {
if int64(len(history)) < defaultAmount {
amount = int64(len(history))
} else {
amount = defaultAmount
}
} else {
if amount > int64(len(history)) {
amount = int64(len(history))
} else {
amount = argAmount
}
}
for _, h := range history[int64(len(history))-amount:] {
fmt.Println(h.ID, h.URL)
}
}