|
|
|
@ -83,6 +83,7 @@ import sys
|
|
|
|
|
|
|
|
|
|
import os
|
|
|
|
|
import re
|
|
|
|
|
import socket
|
|
|
|
|
import time
|
|
|
|
|
import argparse
|
|
|
|
|
from io import StringIO
|
|
|
|
@ -120,6 +121,7 @@ LOG = logging.getLogger()
|
|
|
|
|
|
|
|
|
|
ETC_DIR = '/etc/tor/yaml'
|
|
|
|
|
aTRUST_DB = {}
|
|
|
|
|
aTRUST_DB_INDEX = {}
|
|
|
|
|
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
|
|
|
|
|
# You can call this while bootstrapping
|
|
|
|
|
sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
|
|
|
|
@ -131,14 +133,30 @@ sINCLUDE_GUARD_KEY = 'EntryNodes'
|
|
|
|
|
lKNOWN_NODNS = [
|
|
|
|
|
'0x0.is',
|
|
|
|
|
'a9.wtf',
|
|
|
|
|
'artikel5ev.de',
|
|
|
|
|
'arvanode.net',
|
|
|
|
|
'dodo.pm',
|
|
|
|
|
'dra-family.github.io',
|
|
|
|
|
'eraldonion.org',
|
|
|
|
|
'galtland.network',
|
|
|
|
|
'interfesse.net',
|
|
|
|
|
'kryptonit.org',
|
|
|
|
|
'lonet.sh',
|
|
|
|
|
'moneneis.de',
|
|
|
|
|
'nx42.de',
|
|
|
|
|
'ormycloud.org',
|
|
|
|
|
'plied-privacy.net',
|
|
|
|
|
'rification-for-nusenu.net',
|
|
|
|
|
'sv.ch',
|
|
|
|
|
'thingtohide.nl',
|
|
|
|
|
'tikel10.org',
|
|
|
|
|
'tor-exit-2.aa78i2efsewr0neeknk.xyz',
|
|
|
|
|
'tor-exit-3.aa78i2efsewr0neeknk.xyz',
|
|
|
|
|
'torix-relays.org',
|
|
|
|
|
'tse.com',
|
|
|
|
|
'tuxli.org',
|
|
|
|
|
'w.digidow.eu',
|
|
|
|
|
'www.quintex.com',
|
|
|
|
|
]
|
|
|
|
|
def oMakeController(sSock='', port=9051):
|
|
|
|
|
import getpass
|
|
|
|
@ -157,7 +175,6 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
section=sEXCLUDE_EXIT_KEY,
|
|
|
|
|
lWanted=['BadExit']):
|
|
|
|
|
global oBAD_NODES
|
|
|
|
|
root = 'ExcludeNodes'
|
|
|
|
|
l = []
|
|
|
|
|
if not yaml: return l
|
|
|
|
|
if os.path.exists(sFile):
|
|
|
|
@ -166,54 +183,19 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
oBAD_NODES = o
|
|
|
|
|
|
|
|
|
|
# BROKEN
|
|
|
|
|
# root = 'ExcludeNodes'
|
|
|
|
|
# for elt in o[oBAD_ROOT][root][section].keys():
|
|
|
|
|
# if lWanted and elt not in lWanted: continue
|
|
|
|
|
# # l += o[oBAD_ROOT][root][section][elt]
|
|
|
|
|
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
root = 'ExcludeDomains'
|
|
|
|
|
if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]:
|
|
|
|
|
o[oBAD_ROOT][root] = lKNOWN_NODNS
|
|
|
|
|
else:
|
|
|
|
|
lKNOWN_NODNS = o[oBAD_ROOT][root]
|
|
|
|
|
return l
|
|
|
|
|
|
|
|
|
|
def icheck_torrc(sFile, oArgs):
|
|
|
|
|
l = open(sFile, 'rt').readlines()
|
|
|
|
|
a = {}
|
|
|
|
|
for elt in l:
|
|
|
|
|
elt = elt.strip()
|
|
|
|
|
if not elt or not ' ' in elt: continue
|
|
|
|
|
k,v = elt.split(' ', 1)
|
|
|
|
|
a[k] = v
|
|
|
|
|
keys = a
|
|
|
|
|
|
|
|
|
|
if 'HashedControlPassword' not in keys:
|
|
|
|
|
LOG.info('Add HashedControlPassword for security')
|
|
|
|
|
print('run: tor --hashcontrolpassword <TopSecretWord>')
|
|
|
|
|
if 'ExcludeExitNodes' in keys:
|
|
|
|
|
elt = 'BadNodes.ExcludeExitNodes.BadExit'
|
|
|
|
|
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
|
|
|
|
|
print(f"move to the {elt} section as a list")
|
|
|
|
|
if 'GuardNodes' in keys:
|
|
|
|
|
elt = 'GoodNodes.GuardNodes'
|
|
|
|
|
LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}")
|
|
|
|
|
print(f"move to the {elt} section as a list")
|
|
|
|
|
if 'ExcludeNodes' in keys:
|
|
|
|
|
elt = 'BadNodes.ExcludeNodes.BadExit'
|
|
|
|
|
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
|
|
|
|
|
print(f"move to the {elt} section as a list")
|
|
|
|
|
if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'):
|
|
|
|
|
LOG.info('Add ControlSocket /run/tor/control for us')
|
|
|
|
|
print('ControlSocket /run/tor/control GroupWritable RelaxDirModeCheck')
|
|
|
|
|
if 'UseMicrodescriptors' not in keys or keys['UseMicrodescriptors'] != '1':
|
|
|
|
|
LOG.info('Add UseMicrodescriptors 0 for us')
|
|
|
|
|
print('UseMicrodescriptors 0')
|
|
|
|
|
if 'AutomapHostsSuffixes' not in keys:
|
|
|
|
|
LOG.info('Add AutomapHostsSuffixes for onions')
|
|
|
|
|
print('AutomapHostsSuffixes .exit,.onion')
|
|
|
|
|
if 'AutoMapHostsOnResolve' not in keys:
|
|
|
|
|
LOG.info('Add AutoMapHostsOnResolve for onions')
|
|
|
|
|
print('AutoMapHostsOnResolve 1')
|
|
|
|
|
if 'VirtualAddrNetworkIPv4' not in keys:
|
|
|
|
|
LOG.info('Add VirtualAddrNetworkIPv4 for onions')
|
|
|
|
|
print('VirtualAddrNetworkIPv4 172.16.0.0/12')
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
oGOOD_NODES = {}
|
|
|
|
|
oGOOD_ROOT = 'GoodNodes'
|
|
|
|
|
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
|
|
|
@ -248,13 +230,14 @@ def lIntroductionPoints(controller, lOnions):
|
|
|
|
|
l += [introduction_point.address]
|
|
|
|
|
return l
|
|
|
|
|
|
|
|
|
|
lBAD_URLS = []
|
|
|
|
|
tBAD_URLS = set()
|
|
|
|
|
lATS = ['abuse', 'email']
|
|
|
|
|
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
|
|
|
|
|
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
|
|
|
|
|
'sandbox', 'offlinemasterkey']
|
|
|
|
|
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
|
|
|
|
|
global lBAD_URLS
|
|
|
|
|
global tBAD_URLS
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
# cleanups for yaml
|
|
|
|
|
for elt in lINTS:
|
|
|
|
|
if elt in a:
|
|
|
|
@ -271,12 +254,17 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
|
|
|
|
|
a.update({'fps': []})
|
|
|
|
|
keys = list(a.keys())
|
|
|
|
|
|
|
|
|
|
# test the url for fps and add it to the array
|
|
|
|
|
if 'proof' not in keys:
|
|
|
|
|
LOG.warn(f"{fp} 'proof' not in {keys}")
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
|
|
|
|
|
aCachedContact = aTRUST_DB_INDEX[fp]
|
|
|
|
|
if aCachedContact['email'] = a['email']:
|
|
|
|
|
return aCachedContact
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if 'url' not in keys:
|
|
|
|
|
if 'uri' not in keys:
|
|
|
|
|
a['url'] = ''
|
|
|
|
@ -292,12 +280,21 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
a['url'] = 'https:' +a['url']
|
|
|
|
|
|
|
|
|
|
# domain should be a unique key for contacts
|
|
|
|
|
domain = a['url'][8:]
|
|
|
|
|
domain = a['url'][8:].strip('/')
|
|
|
|
|
try:
|
|
|
|
|
ip = sTorResolve(domain)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
lpair = getaddrinfo(domain, 443)
|
|
|
|
|
if lpait is None:
|
|
|
|
|
try:
|
|
|
|
|
lpair = getaddrinfo(domain, 443)
|
|
|
|
|
except (socket.gaierror, ) as e:
|
|
|
|
|
LOG.debug("{e}")
|
|
|
|
|
lpair = None
|
|
|
|
|
lKNOWN_NODNS.append(domain)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn("{e}")
|
|
|
|
|
lpair = None
|
|
|
|
|
lKNOWN_NODNS.append(domain)
|
|
|
|
|
if lpair is None:
|
|
|
|
|
LOG.warn(f"TorResolv and getaddrinfo failed for {domain}")
|
|
|
|
|
return a
|
|
|
|
|
ip = lpair[0]
|
|
|
|
@ -324,24 +321,24 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|
|
|
|
LOG.exception(f"AttributeError downloading from {domain} {e}")
|
|
|
|
|
except CertificateError as e:
|
|
|
|
|
LOG.warn(f"CertificateError downloading from {domain} {e}")
|
|
|
|
|
lBAD_URLS += [a['url']]
|
|
|
|
|
tBAD_URLS.add(a['url'])
|
|
|
|
|
except TrustorError as e:
|
|
|
|
|
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
|
|
|
|
|
lBAD_URLS += [a['url']]
|
|
|
|
|
tBAD_URLS.add(a['url'])
|
|
|
|
|
except (BaseException ) as e:
|
|
|
|
|
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
|
|
|
|
|
else:
|
|
|
|
|
if o.status_code >= 300:
|
|
|
|
|
LOG.warn(f"Error downloading from {domain} {o.status_code} {o.reason}")
|
|
|
|
|
# any reason retry?
|
|
|
|
|
lBAD_URLS += [a['url']]
|
|
|
|
|
tBAD_URLS.add(a['url'])
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
l = o.text.upper().strip().split('\n')
|
|
|
|
|
if not l:
|
|
|
|
|
# already squacked in lD
|
|
|
|
|
LOG.warn(f"Downloading from {domain} empty for {fp}")
|
|
|
|
|
lBAD_URLS += [a['url']]
|
|
|
|
|
tBAD_URLS.add(a['url'])
|
|
|
|
|
else:
|
|
|
|
|
a['fps'] = [elt for elt in l if elt and len(elt) == 40
|
|
|
|
|
and not elt.startswith('#')]
|
|
|
|
@ -521,13 +518,23 @@ def iMain(lArgs):
|
|
|
|
|
icheck_torrc(sFile, oArgs)
|
|
|
|
|
|
|
|
|
|
global aTRUST_DB
|
|
|
|
|
global aTRUST_DB_INDEX
|
|
|
|
|
|
|
|
|
|
sFile = oArgs.proof_output
|
|
|
|
|
if sFile and os.path.exists(sFile):
|
|
|
|
|
try:
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
aTRUST_DB = yaml.safe_load(oFd)
|
|
|
|
|
except:
|
|
|
|
|
aTRUST_DB = {}
|
|
|
|
|
# reverse lookup of fps to contacts
|
|
|
|
|
# but...
|
|
|
|
|
for k,v in aTRUST_DB:
|
|
|
|
|
aTRUST_DB_INDEX[k] = v
|
|
|
|
|
if 'fps' in aTRUST_DB[k].keys():
|
|
|
|
|
for fp in aTRUST_DB[k]['fps']:
|
|
|
|
|
aTRUST_DB_INDEX[fp] = v
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"Error reading YAML TrustDB {sFile} {e}")
|
|
|
|
|
|
|
|
|
|
if os.path.exists(oArgs.proxy_ctl):
|
|
|
|
|
controller = oMakeController(sSock=oArgs.proxy_ctl)
|
|
|
|
@ -615,7 +622,12 @@ def iMain(lArgs):
|
|
|
|
|
i = c.find(' ')
|
|
|
|
|
if i >=0:
|
|
|
|
|
c = c[:i]
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping 'dns-rsa' {c} {sofar}")
|
|
|
|
|
if c in lKNOWN_NODNS:
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}")
|
|
|
|
|
iDnsContact += 1
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
@ -626,17 +638,18 @@ def iMain(lArgs):
|
|
|
|
|
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
if 'url' in a and a['url'] and a['url'] in lBAD_URLS:
|
|
|
|
|
# The fp is using a contact with a URL we know is bad
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lBAD_URLS {a['url']} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
domain = a['url'][8:]
|
|
|
|
|
if domain in lKNOWN_NODNS:
|
|
|
|
|
# The fp is using a contact with a URL we know is nonexistent
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
if 'url' in a and a['url']:
|
|
|
|
|
if a['url'] in tBAD_URLS:
|
|
|
|
|
# The fp is using a contact with a URL we know is bad
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
domain = a['url'].replace('https://', '').replace('http://', '')
|
|
|
|
|
if domain in lKNOWN_NODNS:
|
|
|
|
|
# The fp is using a contact with a URL we know is bogus
|
|
|
|
|
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
|
|
|
|
|
exit_excludelist.append(relay.fingerprint)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
b = aVerifyContact(list(a.values())[0],
|
|
|
|
@ -661,9 +674,13 @@ def iMain(lArgs):
|
|
|
|
|
iFakeContact += 1
|
|
|
|
|
aBadContacts[relay.fingerprint] = b
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# great contact had good fps and we are in them
|
|
|
|
|
lProofGoodFps += b['fps']
|
|
|
|
|
if relay.fingerprint in aProofUri.keys():
|
|
|
|
|
# a cached entry
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
LOG.info(f"{relay.fingerprint} verified {b['url']} {sofar}")
|
|
|
|
|
# add our contact info to the trustdb
|
|
|
|
|
aProofUri[relay.fingerprint] = b
|
|
|
|
@ -706,7 +723,10 @@ def iMain(lArgs):
|
|
|
|
|
|
|
|
|
|
global oBAD_NODES
|
|
|
|
|
oBAD_NODES['BadNodes']['ExcludeNodes']['BadExit'] = exit_excludelist
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
o[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
|
|
|
|
|
vwrite_badnodes(oArgs)
|
|
|
|
|
|
|
|
|
|
global oGOOD_NODES
|
|
|
|
|
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = lProofGoodFps
|
|
|
|
|
vwrite_goodnodes(oArgs)
|
|
|
|
@ -739,7 +759,7 @@ def iMain(lArgs):
|
|
|
|
|
except stem.SocketClosed as e:
|
|
|
|
|
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
|
|
|
|
retval += 1
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return retval
|
|
|
|
|
|
|
|
|
|
except InvalidRequest as e:
|
|
|
|
|