Commit 0f1e5bca authored by Lorenzo Faletra's avatar Lorenzo Faletra

Import Upstream version 3.7.0

parent fb64ee38
Pipeline #265 failed with stages
in 0 seconds
......@@ -16,6 +16,8 @@ The PRIMARY AUTHORS are:
* Micaela Ranea Sánchez
* Sebastian Kulesz
* Eric Horvat
* Jorge Luis González Iznaga
* Javier Montilva
Project contributors
......
* Add vulnerability preview to status report
* Update Fierce Plugin. Import can be done from GTK console.
* Update Goohost plugin and now Faraday imports Goohost .txt report.
* Update plugin for support WPScan v-3.4.5
* Update Qualysguard plugin to its 8.17.1.0.2 version
* Update custom fields with Searcher
* Update Recon-ng Plugin so that it accepts XML reports
* Add postres version to status-change command
* Couchdb configuration section will not be added anymore
* Add unit test for config/default.xml
......@@ -9,6 +9,20 @@ New features in the latest update
=====================================
3.7 [Apr 3rd, 2019]:
---
* New feature vulnerability preview to view vulnerability data.
* Update Fierce Plugin. Import can be done from GTK console.
* Update Goohost plugin and now Faraday imports Goohost .txt report.
* Update plugin for support WPScan v-3.4.5
* Update Qualysguard plugin to its 8.17.1.0.2 version
* Update custom fields with Searcher
* Update Recon-ng Plugin so that it accepts XML reports
* Add postres version to status-change command
* Couchdb configuration section will not be added anymore
* Add unit test for config/default.xml
3.6 [Feb 21th, 2019]:
---
* Fix CSRF (Cross-Site Request Forgery) vulnerability in vulnerability attachments API.
......
......@@ -9,6 +9,20 @@ New features in the latest update
=====================================
3.7 [Apr 3rd, 2019]:
---
* New feature vulnerability preview to view vulnerability data.
* Update Fierce Plugin. Import can be done from GTK console.
* Update Goohost plugin and now Faraday imports Goohost .txt report.
* Update plugin for support WPScan v-3.4.5
* Update Qualysguard plugin to its 8.17.1.0.2 version
* Update custom fields with Searcher
* Update Recon-ng Plugin so that it accepts XML reports
* Add postres version to status-change command
* Couchdb configuration section will not be added anymore
* Add unit test for config/default.xml
3.6 [Feb 21th, 2019]:
---
* Fix CSRF (Cross-Site Request Forgery) vulnerability in vulnerability attachments API.
......
......@@ -2,7 +2,7 @@
<faraday>
<appname>Faraday - Penetration Test IDE</appname>
<version>3.6.0</version>
<version>3.7.0</version>
<debug_status>0</debug_status>
<font>-Misc-Fixed-medium-r-normal-*-12-100-100-100-c-70-iso8859-1</font>
<home_path>~/</home_path>
......
"""add markdown column to exectuive reports
Revision ID: 5272b3f5a820
Revises: 2ca03a8feef5
Create Date: 2019-03-27 19:26:28.354078+00:00
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '5272b3f5a820'
down_revision = '2ca03a8feef5'
branch_labels = None
depends_on = None
def upgrade():
op.add_column('executive_report', sa.Column('markdown', sa.Boolean(), nullable=False, server_default='False'))
def downgrade():
op.drop_column('executive_report', 'markdown')
......@@ -12,6 +12,7 @@ import socket
import re
import os
import sys
import random
current_path = os.path.abspath(os.getcwd())
......@@ -41,48 +42,61 @@ class FierceParser(object):
"""
def __init__(self, output):
self.target = None
self.items = []
r = re.search(
"DNS Servers for ([\w\.-]+):\r\n([^$]+)Trying zone transfer first...",
regex = re.search(
"DNS Servers for ([\w\.-]+):\n([^$]+)Trying zone transfer first...",
output)
if r is not None:
self.target = r.group(1)
mstr = re.sub("\t", "", r.group(2))
self.dns = mstr.split()
if regex is not None:
self.target = regex.group(1)
mstr = re.sub("\t", "", regex.group(2))
self.dns = filter(None, mstr.splitlines())
r = re.search(
"Now performing [\d]+ test\(s\)...\r\n([^$]+)\x0D\nSubnets found ",
regex = re.search(
"Now performing [\d]+ test\(s\)...\n([^$]+)\nSubnets found ",
output)
if r is not None:
list = r.group(1).split("\r\n")
for i in list:
if regex is not None:
hosts_list = regex.group(1).splitlines()
for i in hosts_list:
if i != "":
mstr = i.split("\t")
item = {'host': mstr[1], 'type': "A", 'ip': mstr[0]}
self.items.append(item)
host = mstr[1]
record = "A"
ip = mstr[0]
self.add_host_info_to_items(ip, host, record)
self.isZoneVuln = False
output= output.replace('\\$', '')
r = re.search(
output = output.replace('\\$', '')
regex = re.search(
"Whoah, it worked - misconfigured DNS server found:([^$]+)\There isn't much point continuing, you have everything.", output)
if r is not None:
if regex is not None:
self.isZoneVuln = True
list = r.group(1).split("\n")
for i in list:
dns_list = regex.group(1).splitlines()
for i in dns_list:
if i != "":
mstr = i.split()
if (mstr and mstr[0] != "" and len(mstr) > 3 and mstr[3] in valid_records):
item = {'host': mstr[0],
'type': mstr[3], 'ip': mstr[4]}
self.items.append(item)
host = mstr[0]
record = mstr[3]
ip = mstr[4]
self.add_host_info_to_items(ip, host, record)
def add_host_info_to_items(self, ip_address, hostname, record):
data = {}
exists = False
for item in self.items:
if ip_address in item['ip']:
item['hosts'].append(hostname)
exists = True
if not exists:
data['ip'] = ip_address
data['hosts'] = [hostname]
data['record'] = record
self.items.append(data)
class FiercePlugin(core.PluginBase):
......@@ -103,6 +117,8 @@ class FiercePlugin(core.PluginBase):
r'^(sudo fierce|fierce|sudo fierce\.pl|fierce\.pl|perl fierce\.pl|\.\/fierce\.pl).*?')
global current_path
self.xml_arg_re = re.compile(r"^.*(>\s*[^\s]+).*$")
def canParseCommandString(self, current_input):
if self._command_regex.match(current_input.strip()):
return True
......@@ -111,7 +127,7 @@ class FiercePlugin(core.PluginBase):
def resolveCNAME(self, item, items):
for i in items:
if (i['host'] == item['ip']):
if (item['ip'] in i['hosts']):
item['ip'] = i['ip']
return item
try:
......@@ -122,7 +138,7 @@ class FiercePlugin(core.PluginBase):
def resolveNS(self, item, items):
try:
item['host'] = item['ip']
item['hosts'][0] = item['ip']
item['ip'] = socket.gethostbyname(item['ip'])
except:
pass
......@@ -135,9 +151,9 @@ class FiercePlugin(core.PluginBase):
item['isResolver'] = False
item['isZoneVuln'] = False
if (item['type'] == "CNAME"):
if (item['record'] == "CNAME"):
self.resolveCNAME(item, parser.items)
if (item['type'] == "NS"):
if (item['record'] == "NS"):
self.resolveNS(item, parser.items)
item['isResolver'] = True
item['isZoneVuln'] = parser.isZoneVuln
......@@ -149,20 +165,15 @@ class FiercePlugin(core.PluginBase):
item['ip'] = ''
for item in parser.items:
if item['ip'] == "127.0.0.1" or item['ip'] == '':
continue
h_id = self.createAndAddHost(item['ip'])
i_id = self.createAndAddInterface(
h_id,
h_id = self.createAndAddHost(
item['ip'],
ipv4_address=item['ip'],
hostname_resolution=[item['host']])
hostnames=item['hosts'])
if item['isResolver']:
s_id = self.createAndAddServiceToInterface(
s_id = self.createAndAddServiceToHost(
h_id,
i_id,
"domain",
"tcp",
ports=['53'])
......@@ -176,7 +187,22 @@ class FiercePlugin(core.PluginBase):
ref=["CVE-1999-0532"])
def processCommandString(self, username, current_path, command_string):
return None
self._output_file_path = os.path.join(
self.data_path,
"%s_%s_output-%s.txt" % (
self.get_ws(),
self.id,
random.uniform(1, 10))
)
arg_match = self.xml_arg_re.match(command_string)
if arg_match is None:
return "%s > %s" % (command_string, self._output_file_path)
else:
return re.sub(arg_match.group(1),
r"> %s" % self._output_file_path,
command_string)
def createPlugin():
......
......@@ -34,30 +34,24 @@ class GoohostParser(object):
TODO: Test goohost output version. Handle what happens if the parser doesn't support it.
TODO: Test cases.
@param goohost_filepath A proper simple report generated by goohost
@param goohost_scantype You could select scan type ip, mail or host
"""
def __init__(self, goohost_filepath, goohost_scantype):
self.filepath = goohost_filepath
self.scantype = goohost_scantype
with open(self.filepath, "r") as f:
line = f.readline()
self.items = []
while line:
if self.scantype == 'ip':
minfo = line.split()
item = {'host': minfo[0], 'ip': minfo[1]}
elif self.scantype == 'host':
line = line.strip()
item = {'host': line, 'ip': self.resolve(line)}
else:
item = {'data': line}
self.items.append(item)
line = f.readline()
def __init__(self, output, goohost_scantype):
self.items = []
lines = filter(None, output.split('\n'))
for line in lines:
if goohost_scantype == 'ip':
data = line.split()
item = {'host': data[0], 'ip': data[1]}
self.add_host_info_to_items(item['ip'], item['host'])
elif goohost_scantype == 'host':
data = line.strip()
item = {'host': data, 'ip': self.resolve(data)}
self.add_host_info_to_items(item['ip'], item['host'])
else:
item = {'data': line}
def resolve(self, host):
try:
......@@ -66,6 +60,19 @@ class GoohostParser(object):
pass
return host
def add_host_info_to_items(self, ip_address, hostname):
data = {}
exists = False
for item in self.items:
if ip_address in item['ip']:
item['hosts'].append(hostname)
exists = True
if not exists:
data['ip'] = ip_address
data['hosts'] = [hostname]
self.items.append(data)
class GoohostPlugin(core.PluginBase):
"""
......@@ -83,44 +90,45 @@ class GoohostPlugin(core.PluginBase):
self._current_path = None
self._command_regex = re.compile(
r'^(sudo goohost\.sh|goohost\.sh|sh goohost\.sh|\.\/goohost\.sh).*?')
self.scantype = "host"
self.host = None
global current_path
self.output_path = None
self._command_string = None
def parseOutputString(self, output, debug=False):
"""
This method will discard the output the shell sends, it will read it from
the xml where it expects it to be present.
This method will check if the import was made through the console or by importing a Goohost report.
Import from Console:The method will take the path of the report generated by Goohost from the output the shell sends and will read
the information from the txt where it expects it to be present.
Import from Report: The method receives the output of the txt report as parameter.
self.scantype defines the method used to generate the Goohost report
NOTE: if 'debug' is true then it is being run from a test case and the
output being sent is valid.
"""
if self.output_path is None:
mypath = re.search("Results saved in file (\S+)", output)
if mypath is not None:
self.output_path = self._current_path + "/" + mypath.group(1)
else:
return False
if self._command_string:
# Import from console
self.scantype = self.define_scantype_by_command(self._command_string)
report_output = output
output = self.read_output_file(report_output)
else:
# Import from report
self.scantype = self.define_scantype_by_output(output)
if debug:
parser = GoohostParser(output, self.scantype)
else:
if not os.path.exists(self.output_path):
return False
parser = GoohostParser(self.output_path, self.scantype)
parser = GoohostParser(output, self.scantype)
if self.scantype == 'host' or self.scantype == 'ip':
for item in parser.items:
h_id = self.createAndAddHost(item['ip'])
i_id = self.createAndAddInterface(
h_id,
h_id = self.createAndAddHost(
item['ip'],
ipv4_address=item['ip'],
hostname_resolution=item['host'])
hostnames=item['hosts'])
del parser
......@@ -129,16 +137,47 @@ class GoohostPlugin(core.PluginBase):
Set output path for parser...
"""
self._current_path = current_path
self._command_string = command_string
def define_scantype_by_command(self, command):
method_regex = re.compile(r'-m (mail|host|ip)')
method = method_regex.search(command)
if method:
return method.group(1)
return 'host'
def define_scantype_by_output(self, output):
lines = output.split('\n')
line = lines[0].split(' ')
if len(line) == 1:
return 'host'
elif len(line) == 2:
return 'ip'
def read_output_file(self, report_path):
mypath = re.search("Results saved in file (\S+)", report_path)
if not mypath:
return False
else:
self.output_path = self._current_path + "/" + mypath.group(1)
if not os.path.exists(self.output_path):
return False
with open(self.output_path, 'r') as report:
output = report.read()
def setHost(self):
pass
return output
def createPlugin():
return GoohostPlugin()
if __name__ == '__main__':
parser = GoohostParser(sys.argv[1])
with open('/home/javier/Plugins/goohost/report-10071-google.com.txt','r') as report:
output = report.read()
parser = GoohostPlugin()
parser.parseOutputString(output)
for item in parser.items:
if item.status == 'up':
print item
......@@ -11,6 +11,7 @@ from plugins import core
import re
import os
import sys
import logging
try:
......@@ -23,6 +24,8 @@ except ImportError:
ETREE_VERSION = [int(i) for i in ETREE_VERSION.split('.')]
logger = logging.getLogger(__name__)
current_path = os.path.abspath(os.getcwd())
__author__ = 'Francisco Amato'
......@@ -69,7 +72,6 @@ class QualysguardXmlParser():
"""
def __init__(self, xml_output):
tree, type_report = self.parse_xml(xml_output)
if not tree or type_report is None:
......@@ -105,7 +107,7 @@ class QualysguardXmlParser():
type_report = None
except SyntaxError, err:
self.devlog('SyntaxError: %s. %s' % (err, xml_output))
logger.error('SyntaxError: %s.' % (err))
return None, None
return tree, type_report
......@@ -254,7 +256,7 @@ class ItemScanReport():
self.node = item_node
self.ip = item_node.get('value')
self.os = self.get_text_from_subnode('OS')
self.hostname = self.get_hostname(item_node)
self.vulns = self.getResults(item_node)
def getResults(self, tree):
......@@ -267,6 +269,12 @@ class ItemScanReport():
for self.issues in tree.findall('INFOS/CAT'):
for v in self.issues.findall('INFO'):
yield ResultsScanReport(v, self.issues)
for self.issues in tree.findall('SERVICES/CAT'):
for v in self.issues.findall('SERVICE'):
yield ResultsScanReport(v, self.issues)
for self.issues in tree.findall('PRACTICES/CAT'):
for v in self.issues.findall('PRACTICE'):
yield ResultsScanReport(v, self.issues)
def get_text_from_subnode(self, subnode_xpath_expr):
"""
......@@ -280,6 +288,14 @@ class ItemScanReport():
return None
def get_hostname(self, node):
hostname = node.get('name')
if hostname == 'No registered hostname':
return ""
return hostname
class ResultsScanReport():
"""
......@@ -294,10 +310,10 @@ class ResultsScanReport():
self.severity = self.node.get('severity')
self.title = self.get_text_from_subnode('TITLE')
self.cvss = self.get_text_from_subnode('CVSS_BASE')
self.pci = self.get_text_from_subnode('PCI_FLAG')
self.diagnosis = self.get_text_from_subnode('DIAGNOSIS')
self.solution = self.get_text_from_subnode('SOLUTION')
self.result = self.get_text_from_subnode('RESULT')
self.consequence = self.get_text_from_subnode('CONSEQUENCE')
self.desc = cleaner_results(self.diagnosis)
if self.result:
......@@ -305,6 +321,11 @@ class ResultsScanReport():
else:
self.desc += ''
if self.consequence:
self.desc += '\nConsequence: ' + cleaner_results(self.consequence)
else:
self.desc += ''
self.ref = []
for r in issue_node.findall('CVE_ID_LIST/CVE_ID'):
self.node = r
......@@ -313,6 +334,9 @@ class ResultsScanReport():
self.node = r
self.ref.append('bid-' + self.get_text_from_subnode('ID'))
if self.cvss:
self.ref.append('CVSS BASE: ' + self.cvss)
def get_text_from_subnode(self, subnode_xpath_expr):
"""
Finds a subnode in the host node and the retrieves a value from it.
......@@ -337,7 +361,7 @@ class QualysguardPlugin(core.PluginBase):
self.id = 'Qualysguard'
self.name = 'Qualysguard XML Output Plugin'
self.plugin_version = '0.0.2'
self.version = 'Qualysguard 2016 March '
self.version = 'Qualysguard 8.17.1.0.2'
self.framework_version = '1.0.0'
self.options = None
self._current_output = None
......@@ -354,19 +378,12 @@ class QualysguardPlugin(core.PluginBase):
parser = QualysguardXmlParser(output)
for item in parser.items:
h_id = self.createAndAddHost(
item.ip,
item.os)
i_id = self.createAndAddInterface(
h_id,
item.ip,
ipv4_address=item.ip,
hostname_resolution=item.ip)
item.os,
hostnames=[item.hostname])
for v in item.vulns:
if v.port is None:
self.createAndAddVulnToHost(
h_id,
......@@ -379,9 +396,8 @@ class QualysguardPlugin(core.PluginBase):
else:
web = False
s_id = self.createAndAddServiceToInterface(
s_id = self.createAndAddServiceToHost(
h_id,
i_id,
v.port,
v.protocol,
ports=[str(v.port)],
......@@ -403,19 +419,6 @@ class QualysguardPlugin(core.PluginBase):
desc=v.desc,
resolution=v.solution if v.solution else '')
n_id = self.createAndAddNoteToService(
h_id,
s_id,
'website',
'')
self.createAndAddNoteToNote(
h_id,
s_id,
n_id,
item.ip,
'')
else:
self.createAndAddVulnToService(
h_id,
......
......@@ -7,7 +7,12 @@ See the file 'doc/LICENSE' for the license information
'''
import re
import json
import socket
import logging
try:
from lxml import etree as ET
except ImportError:
import xml.etree.ElementTree as ET
from plugins.core import PluginBase
......@@ -23,6 +28,102 @@ __status__ = 'Development'
logger = logging.getLogger(__name__)
class ReconngParser(object):
def __init__(self, output):
self._format = self.report_format(output)
self.hosts = []
self.vulns = []
if self._format == 'xml':
self.parsable_tree = self.get_parseable_xml_output(output)
self.parse_xml_report(self.parsable_tree)
elif self._format == 'json':
self.parse_json_report(output)
def report_format(self, output):
xml_format_regex = re.compile(r'^<(.*?)>')
json_format_regex = re.compile(r'(^{)')
if xml_format_regex.match(output):
output_format = 'xml'
elif json_format_regex.match(output):
output_format = 'json'
else:
return False
return output_format
def get_parseable_xml_output(self, xml_output):
try:
tree = ET.fromstring(xml_output)
return tree
except IndexError:
print "Syntax error"
return None
def parse_xml_report(self, tree):
hosts_items = tree.xpath('//hosts/item')
self.hosts_from_report(hosts_items)
vulnerabilities_items = tree.xpath('//vulnerabilities/item')
self.vulns_from_report(vulnerabilities_items)
def parse_json_report(self, output):
reconng_data = json.loads(output)
hosts_items = reconng_data.get('hosts', '')
self.hosts_from_report(hosts_items)
vulns_items = reconng_data.get('vulnerabilities','')
self.vulns_from_report(vulns_items)
def hosts_from_report(self, hosts_items):
for host in hosts_items:
host_info = self.get_info_from_host_element(host)
self.hosts.append(host_info)
def vulns_from_report(self, vulns_items):
for vuln in vulns_items:
vuln_info = self.get_info_from_vuln_element(vuln)
self.vulns.append(vuln_info)
def get_info_from_host_element(self, element):
info = {}
if self._format == 'xml':
info['host'] = element.find('host').text
info['ip'] = element.find('ip_address').text
elif self._format == 'json':
info['host'] = element['host']
info['ip'] = element['ip_address']
return info
def get_info_from_vuln_element(self, element):
info = {}
if self._format == 'xml':
info['host'] = element.find('host').text
info['reference'] = element.find(