|
|
|
@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions
|
|
|
|
|
More may be added later.
|
|
|
|
|
|
|
|
|
|
Because you don't want to exclude the introduction points to any onion
|
|
|
|
|
you want to connect to, ```--white_services``` should whitelist the
|
|
|
|
|
you want to connect to, ```--white_onions``` should whitelist the
|
|
|
|
|
introduction points to a comma sep list of onions, but is
|
|
|
|
|
currently broken in stem 1.8.0: see:
|
|
|
|
|
* https://github.com/torproject/stem/issues/96
|
|
|
|
@ -65,7 +65,7 @@ not just exclude Exit.
|
|
|
|
|
If the Contact info is good we add the list of fingerprints to add
|
|
|
|
|
to ExitNodes, a whitelist of relays to use as exits.
|
|
|
|
|
|
|
|
|
|
```--proof_output``` will write the contact info as a ciiss dictionary
|
|
|
|
|
```--good_contacts``` will write the contact info as a ciiss dictionary
|
|
|
|
|
to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
|
|
|
|
|
is downloaded and the fingerprints are added on a 'fps' field we create
|
|
|
|
|
of that fingerprint's entry of the YAML dictionary. This file is read at the
|
|
|
|
@ -88,7 +88,7 @@ import time
|
|
|
|
|
import argparse
|
|
|
|
|
import string
|
|
|
|
|
from io import StringIO
|
|
|
|
|
import ipaddr
|
|
|
|
|
import ipaddress
|
|
|
|
|
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
|
|
|
|
|
|
|
|
|
|
from urllib3.util.ssl_match_hostname import CertificateError
|
|
|
|
@ -98,15 +98,24 @@ from stem.control import Controller
|
|
|
|
|
from stem.connection import IncorrectPassword
|
|
|
|
|
from stem.util.tor_tools import is_valid_fingerprint
|
|
|
|
|
try:
|
|
|
|
|
import yaml
|
|
|
|
|
from ruamel.yaml import YAML
|
|
|
|
|
yaml = YAML(typ='rt')
|
|
|
|
|
yaml.indent(mapping=2, sequence=2)
|
|
|
|
|
safe_load = yaml.load
|
|
|
|
|
except:
|
|
|
|
|
yaml = None
|
|
|
|
|
if yaml is None:
|
|
|
|
|
try:
|
|
|
|
|
import yaml
|
|
|
|
|
safe_load = yaml.safe_load
|
|
|
|
|
except:
|
|
|
|
|
yaml = None
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
|
|
|
|
|
except:
|
|
|
|
|
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
global LOG
|
|
|
|
|
import logging
|
|
|
|
|
import warnings
|
|
|
|
@ -158,7 +167,7 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
if not yaml: return l
|
|
|
|
|
if os.path.exists(sFile):
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
oBAD_NODES = yaml.safe_load(oFd)
|
|
|
|
|
oBAD_NODES = safe_load(oFd)
|
|
|
|
|
|
|
|
|
|
# BROKEN
|
|
|
|
|
# root = 'ExcludeNodes'
|
|
|
|
@ -170,7 +179,7 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
|
|
|
|
|
root = 'ExcludeDomains'
|
|
|
|
|
if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
|
|
|
|
|
lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS))
|
|
|
|
|
lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS))
|
|
|
|
|
else:
|
|
|
|
|
lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
|
|
|
|
|
return l
|
|
|
|
@ -184,7 +193,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
|
|
|
|
if not yaml: return l
|
|
|
|
|
if os.path.exists(sFile):
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
o = yaml.safe_load(oFd)
|
|
|
|
|
o = safe_load(oFd)
|
|
|
|
|
oGOOD_NODES = o
|
|
|
|
|
if 'GuardNodes' in o[oGOOD_ROOT].keys():
|
|
|
|
|
l = o[oGOOD_ROOT]['GuardNodes']
|
|
|
|
@ -271,7 +280,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
LOG.warn(f"{domain} is bad from {a['url']}")
|
|
|
|
|
LOG.debug(f"{fp} is bad from {a}")
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ip = zResolveDomain(domain)
|
|
|
|
|
if ip == '':
|
|
|
|
|
aFP_EMAIL[fp] = a['email']
|
|
|
|
@ -373,7 +382,7 @@ def aParseContact(contact, fp):
|
|
|
|
|
s += '\n'.join([f" {line}\"".replace(':',': \"', 1)
|
|
|
|
|
for line in l])
|
|
|
|
|
oFd = StringIO(s)
|
|
|
|
|
a = yaml.safe_load(oFd)
|
|
|
|
|
a = safe_load(oFd)
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def oMainArgparser(_=None):
|
|
|
|
@ -414,10 +423,10 @@ def oMainArgparser(_=None):
|
|
|
|
|
help='proxy download connect timeout')
|
|
|
|
|
|
|
|
|
|
parser.add_argument('--good_nodes', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'),
|
|
|
|
|
default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
|
|
|
|
|
help="Yaml file of good info that should not be excluded")
|
|
|
|
|
parser.add_argument('--bad_nodes', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'),
|
|
|
|
|
default=os.path.join(ETC_DIR, 'badnodes.yaml'),
|
|
|
|
|
help="Yaml file of bad nodes that should also be excluded")
|
|
|
|
|
parser.add_argument('--contact', type=str, default='Empty,NoEmail',
|
|
|
|
|
help="comma sep list of conditions - Empty,NoEmail")
|
|
|
|
@ -437,13 +446,13 @@ def oMainArgparser(_=None):
|
|
|
|
|
parser.add_argument('--bad_sections', type=str,
|
|
|
|
|
default='MyBadExit',
|
|
|
|
|
help="sections of the badnodes.yaml to use, comma separated, '' BROKEN")
|
|
|
|
|
parser.add_argument('--white_services', type=str,
|
|
|
|
|
parser.add_argument('--white_onions', type=str,
|
|
|
|
|
default='',
|
|
|
|
|
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
|
|
|
|
parser.add_argument('--torrc_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'torrc.new'),
|
|
|
|
|
help="Write the torrc configuration to a file")
|
|
|
|
|
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'),
|
|
|
|
|
parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
|
|
|
|
|
help="Write the proof data of the included nodes to a YAML file")
|
|
|
|
|
return parser
|
|
|
|
|
|
|
|
|
@ -452,7 +461,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen):
|
|
|
|
|
tmp = oArgs.bad_nodes +'.tmp'
|
|
|
|
|
bak = oArgs.bad_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
|
|
|
|
|
yaml.dump(oBAD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.bad_nodes):
|
|
|
|
@ -464,7 +473,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
|
|
|
|
|
tmp = oArgs.good_nodes +'.tmp'
|
|
|
|
|
bak = oArgs.good_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
|
|
|
|
|
yaml.dump(oGOOD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.good_nodes):
|
|
|
|
@ -489,12 +498,11 @@ def iMain(lArgs):
|
|
|
|
|
icheck_torrc(sFile, oArgs)
|
|
|
|
|
|
|
|
|
|
twhitelist_set = set()
|
|
|
|
|
sFile = oArgs.proof_output
|
|
|
|
|
sFile = oArgs.good_contacts
|
|
|
|
|
if sFile and os.path.exists(sFile):
|
|
|
|
|
try:
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
aTRUST_DB = yaml.safe_load(oFd)
|
|
|
|
|
assert type(aTRUST_DB) == dict
|
|
|
|
|
aTRUST_DB = safe_load(oFd)
|
|
|
|
|
LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
|
|
|
|
|
# reverse lookup of fps to contacts
|
|
|
|
|
# but...
|
|
|
|
@ -520,8 +528,8 @@ def iMain(lArgs):
|
|
|
|
|
|
|
|
|
|
vwait_for_controller(controller, oArgs.wait_boot)
|
|
|
|
|
|
|
|
|
|
if oArgs.proof_output:
|
|
|
|
|
proof_output_tmp = oArgs.proof_output + '.tmp'
|
|
|
|
|
if oArgs.good_contacts:
|
|
|
|
|
good_contacts_tmp = oArgs.good_contacts + '.tmp'
|
|
|
|
|
|
|
|
|
|
elt = controller.get_conf('UseMicrodescriptors')
|
|
|
|
|
if elt != '0' :
|
|
|
|
@ -541,10 +549,17 @@ def iMain(lArgs):
|
|
|
|
|
t = set()
|
|
|
|
|
if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys():
|
|
|
|
|
t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints'])
|
|
|
|
|
# not working = maybe when stem is updated
|
|
|
|
|
w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
|
|
|
|
|
if oArgs.white_services:
|
|
|
|
|
w.update(oArgs.white_services.split(','))
|
|
|
|
|
w = set()
|
|
|
|
|
if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys():
|
|
|
|
|
# 'Onions' can I use the IntroductionPoints for Services too?
|
|
|
|
|
# w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
|
|
|
|
|
pass
|
|
|
|
|
if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys():
|
|
|
|
|
# Provides the descriptor for a hidden service. The **address** is the
|
|
|
|
|
# '.onion' address of the hidden service
|
|
|
|
|
w = set(oGOOD_NODES[oGOOD_ROOT]['Onions'])
|
|
|
|
|
if oArgs.white_onions:
|
|
|
|
|
w.update(oArgs.white_onions.split(','))
|
|
|
|
|
if oArgs.points_timeout > 0:
|
|
|
|
|
LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
|
|
|
|
|
t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))
|
|
|
|
@ -697,10 +712,10 @@ def iMain(lArgs):
|
|
|
|
|
aTRUST_DB[relay.fingerprint] = b
|
|
|
|
|
for elt in b['fps']:
|
|
|
|
|
aTRUST_DB_INDEX[elt] = b
|
|
|
|
|
if oArgs.proof_output and oArgs.log_level <= 20:
|
|
|
|
|
if oArgs.good_contacts and oArgs.log_level <= 20:
|
|
|
|
|
# as we go along then clobber
|
|
|
|
|
with open(proof_output_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
|
|
|
|
|
with open(good_contacts_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aTRUST_DB, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
|
|
|
|
|
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
|
|
|
|
@ -709,16 +724,6 @@ def iMain(lArgs):
|
|
|
|
|
texclude_set = texclude_set.difference(tdns_urls)
|
|
|
|
|
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
|
|
|
|
|
|
|
|
|
|
if oArgs.proof_output and aTRUST_DB:
|
|
|
|
|
with open(proof_output_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.proof_output):
|
|
|
|
|
bak = oArgs.proof_output +'.bak'
|
|
|
|
|
os.rename(oArgs.proof_output, bak)
|
|
|
|
|
os.rename(proof_output_tmp, oArgs.proof_output)
|
|
|
|
|
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
|
|
|
|
|
|
|
|
|
|
if oArgs.torrc_output and texclude_set:
|
|
|
|
|
with open(oArgs.torrc_output, 'wt') as oFTorrc:
|
|
|
|
|
oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
|
|
|
|
@ -730,8 +735,18 @@ def iMain(lArgs):
|
|
|
|
|
if oArgs.bad_contacts and aBadContacts:
|
|
|
|
|
# for later analysis
|
|
|
|
|
with open(oArgs.bad_contacts, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBadContacts, indent=2, stream=oFYaml)
|
|
|
|
|
yaml.dump(aBadContacts, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
|
|
|
|
|
if oArgs.good_contacts != '' and aTRUST_DB:
|
|
|
|
|
with open(good_contacts_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aTRUST_DB, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oArgs.good_contacts):
|
|
|
|
|
bak = oArgs.good_contacts +'.bak'
|
|
|
|
|
os.rename(oArgs.good_contacts, bak)
|
|
|
|
|
os.rename(good_contacts_tmp, oArgs.good_contacts)
|
|
|
|
|
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}")
|
|
|
|
|
|
|
|
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
|
|
|
|
|
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
|
|
|
|
@ -740,6 +755,7 @@ def iMain(lArgs):
|
|
|
|
|
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
|
|
|
|
|
# GuardNodes are readonl
|
|
|
|
|
vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
|
|
|
|
|
|
|
|
|
|
retval = 0
|
|
|
|
|
try:
|
|
|
|
|
logging.getLogger('stem').setLevel(30)
|
|
|
|
@ -776,9 +792,6 @@ def iMain(lArgs):
|
|
|
|
|
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
|
|
|
|
retval += 1
|
|
|
|
|
|
|
|
|
|
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
|
|
|
|
|
return retval
|
|
|
|
|
|
|
|
|
|
except InvalidRequest as e:
|
|
|
|
|
# Unacceptable option value: Invalid router list.
|
|
|
|
|
LOG.error(str(e))
|
|
|
|
@ -802,6 +815,9 @@ def iMain(lArgs):
|
|
|
|
|
controller.close()
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(str(e))
|
|
|
|
|
|
|
|
|
|
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
|
|
|
|
|
return retval
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
try:
|
|
|
|
|