Use async httpclient

src/goofileN.nim

 import httpClient
 import htmlparser
 import xmltree
+import asyncdispatch
 from strutils import intToStr, repeat, split, startsWith, endsWith, replace, contains
 from uri import encodeURL, decodeURL
 from os import paramCount, paramStr, getCurrentDir, existsDir, createDir, getHomeDir
@@ -87,10 +88,10 @@ proc downloadFiles(folder: string, agent: string) =
   ]#
 
   var
-    client = newHttpClient()
+    client = newAsyncHttpClient()
     downloaded = 0
 
-  client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"}) else: newHttpHeaders({"User-Agent": agent})
+  client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}) else: newHttpHeaders({"User-Agent": agent})
   
   for eachResult in allDorkResults:
     let
@@ -99,7 +100,7 @@ proc downloadFiles(folder: string, agent: string) =
 
     try:
       echo "[+] Downloading " & eachResult.realURL
-      client.downloadFile(eachResult.realURL, finalPath)
+      discard client.downloadFile(eachResult.realURL, finalPath)
       echo "[*] Saved at " & finalPath
       downloaded += 1
     except:
@@ -108,9 +109,10 @@ proc downloadFiles(folder: string, agent: string) =
         echo "[-] Download from cache is not supported. Please try it manually: " & eachResult.cacheURL
     
   echo "[*] Downloaded " & intToStr(downloaded) & " files"
+  client.close()
 
 
-proc getResult(resp: Response, ext: string) =
+proc getResult(resp: AsyncResponse, ext: string) =
   #[
     Check respose and create filter for the URL
     Expect result:
@@ -122,7 +124,7 @@ proc getResult(resp: Response, ext: string) =
   if resp.status != "200 OK":
     stderr.write("[x] Get status code " & resp.status & "\n")
   else:
-    let allResults = parseHTML(resp.body)
+    let allResults = parseHTML(waitfor(resp.body))
     for eachResult in allResults.findAll("div"):
       # Filter all result thing
       if eachResult.attr("class") == "r":
@@ -166,7 +168,7 @@ proc basicSearch(domain, extension, query, agent: string) =
     Q= Our queries
   ]#
   var
-    client = newHttpClient()
+    client = newAsyncHttpClient()
     payload = "site:" & domain & " ext:" & extension
   
   if query != "":
@@ -174,9 +176,10 @@ proc basicSearch(domain, extension, query, agent: string) =
   
   payload = encodeUrl(payload)
 
-  client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"}) else: newHttpHeaders({"User-Agent": agent})
-  let resp = client.get("https://google.com/search?num=500&q=" & payload)
-  getResult(resp, extension)
+  client.headers = if agent == "": newHttpHeaders({"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}) else: newHttpHeaders({"User-Agent": agent})
+  let resp = client.get("https://www.google.com/search?num=500&q=" & payload)
+  client.close()
+  getResult(waitfor(resp), extension)
 
 
 proc handleOptions(options: Options) =