Implements embedded TOR support

This commit is contained in:
Mikhail Klementev 2019-06-20 16:20:51 +00:00
parent e78fc03417
commit d8865df7a9
No known key found for this signature in database
GPG Key ID: BE44DA8C062D87DC
2 changed files with 53 additions and 1 deletions

View File

@ -25,6 +25,11 @@ import (
"golang.org/x/net/html/charset" "golang.org/x/net/html/charset"
) )
var (
Transport *http.Transport = nil
UserAgent string
)
func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) { func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
doc.Find("a").Each(func(i int, s *goquery.Selection) { doc.Find("a").Each(func(i int, s *goquery.Selection) {
@ -133,6 +138,9 @@ func handleResponse(db *sql.DB, resp *http.Response, lastUrl *url.URL) {
func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) { func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
client := &http.Client{Jar: jar} client := &http.Client{Jar: jar}
if Transport != nil {
client.Transport = Transport
}
var lastUrl *url.URL var lastUrl *url.URL
@ -155,7 +163,7 @@ func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
log.Fatalln(err) log.Fatalln(err)
} }
req.Header.Set("User-Agent", "Wi 0.0") req.Header.Set("User-Agent", UserAgent)
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
@ -214,6 +222,9 @@ func Form(db *sql.DB, jar *cookiejar.Jar, formID int64, formArgs []string) {
} }
client := &http.Client{Jar: jar} client := &http.Client{Jar: jar}
if Transport != nil {
client.Transport = Transport
}
var lastUrl *url.URL var lastUrl *url.URL

41
main.go
View File

@ -9,6 +9,8 @@
package main package main
import ( import (
"context"
"net/http"
"os" "os"
"strings" "strings"
@ -17,8 +19,20 @@ import (
cookiejar "github.com/juju/persistent-cookiejar" cookiejar "github.com/juju/persistent-cookiejar"
kingpin "gopkg.in/alecthomas/kingpin.v2" kingpin "gopkg.in/alecthomas/kingpin.v2"
"github.com/cretz/bine/process"
"github.com/cretz/bine/tor"
"github.com/ipsn/go-libtor"
) )
var creator = libtor.Creator
type LibTorWrapper struct{}
func (LibTorWrapper) New(ctx context.Context, args ...string) (process.Process, error) {
return creator.New(ctx, args...)
}
type searchList []string type searchList []string
func (l *searchList) Set(value string) (err error) { func (l *searchList) Set(value string) (err error) {
@ -41,6 +55,9 @@ func SearchList(settings kingpin.Settings) (target *[]string) {
} }
var ( var (
useTor = kingpin.Flag("tor", "Use embedded tor").Default("false").Bool()
ua = kingpin.Flag("ua", "User-Agent").Default("Wi 0.1").String()
get = kingpin.Command("get", "Get url") get = kingpin.Command("get", "Get url")
getUrl = get.Arg("url", "Url").Required().String() getUrl = get.Arg("url", "Url").Required().String()
@ -93,6 +110,30 @@ func main() {
defer jar.Save() defer jar.Save()
kingpin.Parse()
var t *tor.Tor
if *useTor {
t, err = tor.Start(nil, &tor.StartConf{
ProcessCreator: LibTorWrapper{},
DataDir: wiDir + "/tor",
})
if err != nil {
panic(err)
}
defer t.Close()
dialer, err := t.Dialer(nil, nil)
if err != nil {
panic(err)
}
commands.Transport = &http.Transport{DialContext: dialer.DialContext}
}
commands.UserAgent = *ua
switch kingpin.Parse() { switch kingpin.Parse() {
case "get": case "get":
commands.Get(db, jar, *getUrl) commands.Get(db, jar, *getUrl)