You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
368 lines
14 KiB
Python
368 lines
14 KiB
Python
#!/usr/local/bin/python3.sh
|
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
|
|
|
import argparse
|
|
from argparse import Namespace
|
|
import os
|
|
import sys
|
|
import re
|
|
from io import StringIO
|
|
import logging
|
|
import warnings
|
|
global LOG
|
|
|
|
from toxygen_wrapper.tests.support_onions import (woGetStemController,
|
|
vwait_for_controller,)
|
|
|
|
try:
|
|
from ruamel.yaml import YAML
|
|
yaml = YAML(typ='rt')
|
|
yaml.indent(mapping=2, sequence=2)
|
|
safe_load = yaml.load
|
|
except:
|
|
yaml = None
|
|
if yaml is None:
|
|
try:
|
|
import yaml
|
|
safe_load = yaml.safe_load
|
|
except:
|
|
yaml = None
|
|
|
|
try:
|
|
# if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
|
# os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
|
# https://pypi.org/project/coloredlogs/
|
|
import coloredlogs
|
|
except ImportError:
|
|
coloredlogs = False
|
|
|
|
def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict:
|
|
# cleanups
|
|
for elt in lINTS:
|
|
if elt in a:
|
|
a[elt] = int(a[elt])
|
|
for elt in lBOOLS:
|
|
if elt not in a: continue
|
|
if a[elt] in ['y', 'yes', 'true', 'True']:
|
|
a[elt] = True
|
|
else:
|
|
a[elt] = False
|
|
for elt in lEMAILS:
|
|
if elt not in a: continue
|
|
a[elt] = sCleanEmail(a[elt], lAT_REPS, lDOT_REPS, lNO_EMAIL)
|
|
if 'url' in a.keys():
|
|
a['url'] = a['url'].rstrip('/')
|
|
if a['url'].startswith('http://'):
|
|
domain = a['url'].replace('http://', '')
|
|
elif a['url'].startswith('https://'):
|
|
domain = a['url'].replace('https://', '')
|
|
else:
|
|
domain = a['url']
|
|
a['url'] = 'https://' + domain
|
|
a.update({'fps': []})
|
|
return a
|
|
|
|
def sCleanEmail(s, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> str:
|
|
s = s.lower()
|
|
for elt in lAT_REPS:
|
|
if not elt.startswith(' '):
|
|
s = s.replace(' ' + elt + ' ', '@')
|
|
s = s.replace(elt, '@')
|
|
for elt in lDOT_REPS:
|
|
if not elt.startswith(' '):
|
|
s = s.replace(' ' + elt + ' ', '.')
|
|
s = s.replace(elt, '.')
|
|
s = s.replace('(dash)', '-')
|
|
s = s.replace('hyphen ', '-')
|
|
for elt in lNO_EMAIL:
|
|
s = s.replace(elt, '?')
|
|
return s
|
|
|
|
lMORONS = ['hoster:Quintex Alliance Consulting ']
|
|
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
|
|
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
|
|
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
|
|
'sandbox', 'offlinemasterkey']
|
|
lEMAILS = ['abuse', 'email']
|
|
|
|
ETC_DIR = '/usr/local/etc/tor/yaml'
|
|
|
|
def oStemController(oargs, sEXCLUDE_EXIT_GROUP):
|
|
if os.path.exists(oargs.proxy_ctl):
|
|
controller = oGetStemController(log_level=oargs.log_level,
|
|
sock_or_pair=oargs.proxy_ctl,
|
|
password=oargs.torctl_pass)
|
|
else:
|
|
port =int(oargs.proxy_ctl)
|
|
controller = oGetStemController(log_level=oargs.log_level,
|
|
sock_or_pair=port,
|
|
password=oargs.torctl_pass)
|
|
|
|
vwait_for_controller(controller, oargs.wait_boot)
|
|
|
|
elt = controller.get_conf('UseMicrodescriptors')
|
|
if elt != '0':
|
|
LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
|
|
controller.set_conf('UseMicrodescriptors', 0)
|
|
# does it work dynamically?
|
|
return 2
|
|
|
|
elt = controller.get_conf(sEXCLUDE_EXIT_GROUP)
|
|
if elt and elt != '{??}':
|
|
LOG.warn(f"{sEXCLUDE_EXIT_GROUP} is in use already")
|
|
|
|
return controller
|
|
|
|
def aParseContactYaml(contact, fp) -> dict:
|
|
"""
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
"""
|
|
l = [line for line in contact.strip().replace('"', '').split(' ')
|
|
if ':' in line]
|
|
LOG.debug(f"{fp} {len(l)} fields")
|
|
s = f'"{fp}":\n'
|
|
s += '\n'.join([f" {line}\"".replace(':', ': \"', 1)
|
|
for line in l])
|
|
oFd = StringIO(s)
|
|
a = safe_load(oFd)
|
|
return a
|
|
|
|
def aParseContact(contact, fp, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict:
|
|
"""
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
"""
|
|
a = {}
|
|
if not contact:
|
|
LOG.warn(f"BC null contact for {fp}")
|
|
LOG.debug(f"{fp} {contact}")
|
|
return {}
|
|
|
|
contact = contact.split(r'\n')[0]
|
|
for elt in lMORONS:
|
|
contact = contact.replace(elt, '')
|
|
m = oCONTACT_RE.match(contact)
|
|
# 450 matches!
|
|
if m and m.groups and len(m.groups(0)) > 2 and m.span()[1] > 0:
|
|
i = len(m.groups(0)[0]) + len(m.groups(0)[1])
|
|
contact = contact[i:]
|
|
|
|
# shlex?
|
|
lelts = contact.split(' ')
|
|
if not lelts:
|
|
LOG.warn(f"BC empty contact for {fp}")
|
|
LOG.debug(f"{fp} {contact}")
|
|
return {}
|
|
|
|
for elt in lelts:
|
|
if ':' not in elt:
|
|
# hoster:Quintex Alliance Consulting
|
|
LOG.warn(f"BC no : in {elt} for {contact} in {fp}")
|
|
# return {}
|
|
# try going with what we have
|
|
break
|
|
(key , val,) = elt.split(':', 1)
|
|
if key == '':
|
|
continue
|
|
key = key.rstrip(':')
|
|
a[key] = val
|
|
a = aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL)
|
|
return a
|
|
|
|
def vwrite_good_contacts(oargs, aGOOD_CONTACTS_DB) -> None:
|
|
good_contacts_tmp = oargs.good_contacts + '.tmp'
|
|
with open(good_contacts_tmp, 'wt') as oFYaml:
|
|
yaml.dump(aGOOD_CONTACTS_DB, oFYaml)
|
|
oFYaml.close()
|
|
if os.path.exists(oargs.good_contacts):
|
|
bak = oargs.good_contacts +'.bak'
|
|
os.rename(oargs.good_contacts, bak)
|
|
os.rename(good_contacts_tmp, oargs.good_contacts)
|
|
LOG.info(f"Wrote {len(list(aGOOD_CONTACTS_DB.keys()))} good contact details to {oargs.good_contacts}")
|
|
|
|
def vwrite_bad_contacts(oargs, aBAD_CONTACTS_DB) -> None:
|
|
bad_contacts_tmp = oargs.bad_contacts + '.tmp'
|
|
with open(bad_contacts_tmp, 'wt') as oFYaml:
|
|
yaml.dump(aBAD_CONTACTS_DB, oFYaml)
|
|
oFYaml.close()
|
|
if os.path.exists(oargs.bad_contacts):
|
|
bak = oargs.bad_contacts +'.bak'
|
|
os.rename(oargs.bad_contacts, bak)
|
|
os.rename(bad_contacts_tmp, oargs.bad_contacts)
|
|
LOG.info(f"Wrote {len(list(aBAD_CONTACTS_DB.keys()))} bad contact details to {oargs.bad_contacts}")
|
|
|
|
def vwrite_badnodes(oargs, aBAD_NODES, slen, stag) -> None:
|
|
if not aBAD_NODES: return
|
|
tmp = oargs.bad_nodes +'.tmp'
|
|
bak = oargs.bad_nodes +'.bak'
|
|
with open(tmp, 'wt') as oFYaml:
|
|
yaml.dump(aBAD_NODES, oFYaml)
|
|
LOG.info(f"Wrote to {oargs.bad_nodes}")
|
|
oFYaml.close()
|
|
if os.path.exists(oargs.bad_nodes):
|
|
os.rename(oargs.bad_nodes, bak)
|
|
os.rename(tmp, oargs.bad_nodes)
|
|
|
|
def vwrite_goodnodes(oargs, aGOOD_NODES, ilen) -> None:
|
|
tmp = oargs.good_nodes +'.tmp'
|
|
bak = oargs.good_nodes +'.bak'
|
|
with open(tmp, 'wt') as oFYaml:
|
|
yaml.dump(aGOOD_NODES, oFYaml)
|
|
LOG.info(f"Wrote good relays to {oargs.good_nodes}")
|
|
oFYaml.close()
|
|
if os.path.exists(oargs.good_nodes):
|
|
os.rename(oargs.good_nodes, bak)
|
|
os.rename(tmp, oargs.good_nodes)
|
|
|
|
def vwritefinale(oargs, lNOT_IN_RELAYS_DB) -> None:
|
|
|
|
if len(lNOT_IN_RELAYS_DB):
|
|
LOG.warn(f"{len(lNOT_IN_RELAYS_DB)} relays from stem were not in onionoo.torproject.org")
|
|
|
|
LOG.info(f"For info on a FP, use: https://nusenu.github.io/OrNetStats/w/relay/<FP>.html")
|
|
LOG.info(f"For info on relays, try: https://onionoo.torproject.org/details")
|
|
# https://onionoo.torproject.org/details
|
|
|
|
def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
|
|
global LOG
|
|
LOG = theLOG
|
|
add = True
|
|
# stem fucks up logging
|
|
# from stem.util import log
|
|
logging.getLogger('stem').setLevel(30)
|
|
|
|
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
|
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
|
|
logging._defaultFormatter.default_msec_format = ''
|
|
|
|
if logfile:
|
|
add = logfile.startswith('+')
|
|
sub = logfile.startswith('-')
|
|
if add or sub:
|
|
logfile = logfile[1:]
|
|
kwargs['filename'] = logfile
|
|
|
|
if coloredlogs:
|
|
# https://pypi.org/project/coloredlogs/
|
|
coloredlogs.install(
|
|
level=log_level,
|
|
logger=LOG,
|
|
stream=stream,
|
|
fmt='%(levelname)s %(message)s',
|
|
isatty=True, # required!
|
|
milliseconds=False,
|
|
)
|
|
if logfile:
|
|
oHandler = logging.FileHandler(logfile)
|
|
LOG.addHandler(oHandler)
|
|
LOG.info(f"Setting coloured log_level to {log_level} {stream}")
|
|
else:
|
|
kwargs = dict(level=log_level,
|
|
force=True,
|
|
format='%(levelname)s %(message)s')
|
|
|
|
logging.basicConfig(**kwargs)
|
|
if add and logfile:
|
|
oHandler = logging.StreamHandler(stream)
|
|
LOG.addHandler(oHandler)
|
|
LOG.info(f"Setting log_level to {log_level}")
|
|
|
|
def oMainArgparser(_=None, __prolog__='') -> Namespace:
|
|
|
|
try:
|
|
from OpenSSL import SSL
|
|
lCAfs = SSL._CERTIFICATE_FILE_LOCATIONS
|
|
except:
|
|
lCAfs = []
|
|
|
|
CAfs = []
|
|
for elt in lCAfs:
|
|
if os.path.exists(elt):
|
|
CAfs.append(elt)
|
|
if not CAfs:
|
|
CAfs = ['']
|
|
|
|
parser = argparse.ArgumentParser(add_help=True,
|
|
epilog=__prolog__)
|
|
|
|
# important settings
|
|
parser.add_argument('--bad_on', type=str, default='Empty,NoEmail,NotGood',
|
|
help="comma sep list of conditions - Empty,NoEmail,NotGood")
|
|
parser.add_argument('--points_timeout', type=int, default=0,
|
|
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
|
|
parser.add_argument('--saved_only', default=False,
|
|
action='store_true',
|
|
help="Just use the info in the last *.yaml files without querying the Tor controller")
|
|
parser.add_argument('--hs_dir', type=str,
|
|
default='/var/lib/tor',
|
|
help="Parse the files name hostname below this dir to find Hidden Services to whitelist")
|
|
parser.add_argument('--notice_log', type=str,
|
|
default='',
|
|
help="Parse the notice log for relays and services")
|
|
parser.add_argument('--strict_nodes', type=str, default=0,
|
|
choices=['0', '1'],
|
|
help="Set StrictNodes: 1 is less anonymous but more secure, although some onion sites may be unreachable")
|
|
|
|
# proxy
|
|
parser.add_argument('--proxy_host', '--proxy-host', type=str,
|
|
default='127.0.0.1',
|
|
help='proxy host')
|
|
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
|
|
help='proxy socks port')
|
|
parser.add_argument('--proxy_ctl', '--proxy-ctl',
|
|
default='/run/tor/control' if os.path.exists('/run/tor/control') else '9051',
|
|
type=str,
|
|
help='control socket - or port')
|
|
parser.add_argument('--torctl_pass',
|
|
default=os.environ.get('TOR_CONTROLLER_PASSWORD', ''),
|
|
type=str,
|
|
help='password for the tor controller')
|
|
|
|
parser.add_argument('--torrc',
|
|
default='/etc/tor/torrc-defaults',
|
|
type=str,
|
|
help='torrc to check for suggestions')
|
|
|
|
# output
|
|
parser.add_argument('--torrc_output', type=str,
|
|
default=os.path.join(ETC_DIR, 'torrc.new'),
|
|
help="Write the torrc configuration to a file")
|
|
parser.add_argument('--good_nodes', type=str,
|
|
default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
|
|
help="Yaml file of good info that should not be excluded")
|
|
parser.add_argument('--bad_nodes', type=str,
|
|
default=os.path.join(ETC_DIR, 'badnodes.yaml'),
|
|
help="Yaml file of bad nodes that should also be excluded")
|
|
parser.add_argument('--bad_contacts', type=str,
|
|
default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
|
|
help="Yaml file of bad contacts that bad FPs are using")
|
|
parser.add_argument('--relays_output', type=str,
|
|
default=os.path.join(ETC_DIR, 'relays.json'),
|
|
help="Write the download relays in json to a file")
|
|
parser.add_argument('--wellknown_output', type=str,
|
|
default=os.path.join(ETC_DIR, 'https'),
|
|
help="Write the well-known files to a directory")
|
|
parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
|
|
help="Write the proof data of the included nodes to a YAML file")
|
|
|
|
# timeouts
|
|
parser.add_argument('--timeout', default=60, type=int,
|
|
help='proxy download connect timeout')
|
|
parser.add_argument('--wait_boot', type=int, default=120,
|
|
help="Seconds to wait for Tor to booststrap")
|
|
parser.add_argument('--https_cafile', type=str,
|
|
help="Certificate Authority file (in PEM)",
|
|
default=CAfs[0])
|
|
|
|
parser.add_argument('--log_level', type=int, default=20,
|
|
help="10=debug 20=info 30=warn 40=error")
|
|
parser.add_argument('--bad_sections', type=str,
|
|
default='BadExit',
|
|
help="sections of the badnodes.yaml to use, in addition to BadExit, comma separated")
|
|
parser.add_argument('--white_onions', type=str,
|
|
default='',
|
|
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
|
|
|
return parser
|