mirror of
https://code.dumpstack.io/tools/wi.git
synced 2024-11-14 14:29:19 +02:00
Implements embedded TOR support
This commit is contained in:
parent
e78fc03417
commit
d8865df7a9
|
@ -25,6 +25,11 @@ import (
|
||||||
"golang.org/x/net/html/charset"
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
Transport *http.Transport = nil
|
||||||
|
UserAgent string
|
||||||
|
)
|
||||||
|
|
||||||
func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
|
func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
|
||||||
|
|
||||||
doc.Find("a").Each(func(i int, s *goquery.Selection) {
|
doc.Find("a").Each(func(i int, s *goquery.Selection) {
|
||||||
|
@ -133,6 +138,9 @@ func handleResponse(db *sql.DB, resp *http.Response, lastUrl *url.URL) {
|
||||||
|
|
||||||
func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
|
func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
|
||||||
client := &http.Client{Jar: jar}
|
client := &http.Client{Jar: jar}
|
||||||
|
if Transport != nil {
|
||||||
|
client.Transport = Transport
|
||||||
|
}
|
||||||
|
|
||||||
var lastUrl *url.URL
|
var lastUrl *url.URL
|
||||||
|
|
||||||
|
@ -155,7 +163,7 @@ func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
|
||||||
log.Fatalln(err)
|
log.Fatalln(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("User-Agent", "Wi 0.0")
|
req.Header.Set("User-Agent", UserAgent)
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -214,6 +222,9 @@ func Form(db *sql.DB, jar *cookiejar.Jar, formID int64, formArgs []string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
client := &http.Client{Jar: jar}
|
client := &http.Client{Jar: jar}
|
||||||
|
if Transport != nil {
|
||||||
|
client.Transport = Transport
|
||||||
|
}
|
||||||
|
|
||||||
var lastUrl *url.URL
|
var lastUrl *url.URL
|
||||||
|
|
||||||
|
|
41
main.go
41
main.go
|
@ -9,6 +9,8 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
@ -17,8 +19,20 @@ import (
|
||||||
|
|
||||||
cookiejar "github.com/juju/persistent-cookiejar"
|
cookiejar "github.com/juju/persistent-cookiejar"
|
||||||
kingpin "gopkg.in/alecthomas/kingpin.v2"
|
kingpin "gopkg.in/alecthomas/kingpin.v2"
|
||||||
|
|
||||||
|
"github.com/cretz/bine/process"
|
||||||
|
"github.com/cretz/bine/tor"
|
||||||
|
"github.com/ipsn/go-libtor"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var creator = libtor.Creator
|
||||||
|
|
||||||
|
type LibTorWrapper struct{}
|
||||||
|
|
||||||
|
func (LibTorWrapper) New(ctx context.Context, args ...string) (process.Process, error) {
|
||||||
|
return creator.New(ctx, args...)
|
||||||
|
}
|
||||||
|
|
||||||
type searchList []string
|
type searchList []string
|
||||||
|
|
||||||
func (l *searchList) Set(value string) (err error) {
|
func (l *searchList) Set(value string) (err error) {
|
||||||
|
@ -41,6 +55,9 @@ func SearchList(settings kingpin.Settings) (target *[]string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
useTor = kingpin.Flag("tor", "Use embedded tor").Default("false").Bool()
|
||||||
|
ua = kingpin.Flag("ua", "User-Agent").Default("Wi 0.1").String()
|
||||||
|
|
||||||
get = kingpin.Command("get", "Get url")
|
get = kingpin.Command("get", "Get url")
|
||||||
getUrl = get.Arg("url", "Url").Required().String()
|
getUrl = get.Arg("url", "Url").Required().String()
|
||||||
|
|
||||||
|
@ -93,6 +110,30 @@ func main() {
|
||||||
|
|
||||||
defer jar.Save()
|
defer jar.Save()
|
||||||
|
|
||||||
|
kingpin.Parse()
|
||||||
|
|
||||||
|
var t *tor.Tor
|
||||||
|
if *useTor {
|
||||||
|
t, err = tor.Start(nil, &tor.StartConf{
|
||||||
|
ProcessCreator: LibTorWrapper{},
|
||||||
|
DataDir: wiDir + "/tor",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer t.Close()
|
||||||
|
|
||||||
|
dialer, err := t.Dialer(nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
commands.Transport = &http.Transport{DialContext: dialer.DialContext}
|
||||||
|
}
|
||||||
|
|
||||||
|
commands.UserAgent = *ua
|
||||||
|
|
||||||
switch kingpin.Parse() {
|
switch kingpin.Parse() {
|
||||||
case "get":
|
case "get":
|
||||||
commands.Get(db, jar, *getUrl)
|
commands.Get(db, jar, *getUrl)
|
||||||
|
|
Loading…
Reference in New Issue