Commit 22b282f1 authored by dmknght's avatar dmknght

Use async httpclient

parent d8705e53
Pipeline #1348 canceled with stages
import httpClient
import htmlparser
import xmltree
import asyncdispatch
from strutils import intToStr, repeat, split, startsWith, endsWith, replace, contains
from uri import encodeURL, decodeURL
from os import paramCount, paramStr, getCurrentDir, existsDir, createDir, getHomeDir
......@@ -87,10 +88,10 @@ proc downloadFiles(folder: string, agent: string) =
]#
var
client = newHttpClient()
client = newAsyncHttpClient()
downloaded = 0
client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"}) else: newHttpHeaders({"User-Agent": agent})
client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}) else: newHttpHeaders({"User-Agent": agent})
for eachResult in allDorkResults:
let
......@@ -99,7 +100,7 @@ proc downloadFiles(folder: string, agent: string) =
try:
echo "[+] Downloading " & eachResult.realURL
client.downloadFile(eachResult.realURL, finalPath)
discard client.downloadFile(eachResult.realURL, finalPath)
echo "[*] Saved at " & finalPath
downloaded += 1
except:
......@@ -108,9 +109,10 @@ proc downloadFiles(folder: string, agent: string) =
echo "[-] Download from cache is not supported. Please try it manually: " & eachResult.cacheURL
echo "[*] Downloaded " & intToStr(downloaded) & " files"
client.close()
proc getResult(resp: Response, ext: string) =
proc getResult(resp: AsyncResponse, ext: string) =
#[
Check respose and create filter for the URL
Expect result:
......@@ -122,7 +124,7 @@ proc getResult(resp: Response, ext: string) =
if resp.status != "200 OK":
stderr.write("[x] Get status code " & resp.status & "\n")
else:
let allResults = parseHTML(resp.body)
let allResults = parseHTML(waitfor(resp.body))
for eachResult in allResults.findAll("div"):
# Filter all result thing
if eachResult.attr("class") == "r":
......@@ -166,7 +168,7 @@ proc basicSearch(domain, extension, query, agent: string) =
Q= Our queries
]#
var
client = newHttpClient()
client = newAsyncHttpClient()
payload = "site:" & domain & " ext:" & extension
if query != "":
......@@ -174,9 +176,10 @@ proc basicSearch(domain, extension, query, agent: string) =
payload = encodeUrl(payload)
client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"}) else: newHttpHeaders({"User-Agent": agent})
let resp = client.get("https://google.com/search?num=500&q=" & payload)
getResult(resp, extension)
client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}) else: newHttpHeaders({"User-Agent": agent})
let resp = client.get("https://www.google.com/search?num=500&q=" & payload)
client.close()
getResult(waitfor(resp), extension)
proc handleOptions(options: Options) =
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment