Improvements
This commit is contained in:
emdee
parent
d11d95aafe
commit
6f3207f02e
4 changed files with 241 additions and 104 deletions
|
|
@ -83,6 +83,7 @@ import sys
|
|||
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
import argparse
|
||||
from io import StringIO
|
||||
|
|
@ -120,6 +121,7 @@ LOG = logging.getLogger()
|
|||
|
||||
ETC_DIR = '/etc/tor/yaml'
|
||||
aTRUST_DB = {}
|
||||
aTRUST_DB_INDEX = {}
|
||||
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
|
||||
# You can call this while bootstrapping
|
||||
sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
|
||||
|
|
@ -131,14 +133,30 @@ sINCLUDE_GUARD_KEY = 'EntryNodes'
|
|||
lKNOWN_NODNS = [
|
||||
'0x0.is',
|
||||
'a9.wtf',
|
||||
'artikel5ev.de',
|
||||
'arvanode.net',
|
||||
'dodo.pm',
|
||||
'dra-family.github.io',
|
||||
'eraldonion.org',
|
||||
'galtland.network',
|
||||
'interfesse.net',
|
||||
'kryptonit.org',
|
||||
'lonet.sh',
|
||||
'moneneis.de',
|
||||
'nx42.de',
|
||||
'ormycloud.org',
|
||||
'plied-privacy.net',
|
||||
'rification-for-nusenu.net',
|
||||
'sv.ch',
|
||||
'thingtohide.nl',
|
||||
'tikel10.org',
|
||||
'tor-exit-2.aa78i2efsewr0neeknk.xyz',
|
||||
'tor-exit-3.aa78i2efsewr0neeknk.xyz',
|
||||
'torix-relays.org',
|
||||
'tse.com',
|
||||
'tuxli.org',
|
||||
'w.digidow.eu',
|
||||
'www.quintex.com',
|
||||
]
|
||||
def oMakeController(sSock='', port=9051):
|
||||
import getpass
|
||||
|
|
@ -157,7 +175,6 @@ def lYamlBadNodes(sFile,
|
|||
section=sEXCLUDE_EXIT_KEY,
|
||||
lWanted=['BadExit']):
|
||||
global oBAD_NODES
|
||||
root = 'ExcludeNodes'
|
||||
l = []
|
||||
if not yaml: return l
|
||||
if os.path.exists(sFile):
|
||||
|
|
@ -166,54 +183,19 @@ def lYamlBadNodes(sFile,
|
|||
oBAD_NODES = o
|
||||
|
||||
# BROKEN
|
||||
# root = 'ExcludeNodes'
|
||||
# for elt in o[oBAD_ROOT][root][section].keys():
|
||||
# if lWanted and elt not in lWanted: continue
|
||||
# # l += o[oBAD_ROOT][root][section][elt]
|
||||
|
||||
global lKNOWN_NODNS
|
||||
root = 'ExcludeDomains'
|
||||
if root not in o[oBAD_ROOT] or not o[oBAD_ROOT][root]:
|
||||
o[oBAD_ROOT][root] = lKNOWN_NODNS
|
||||
else:
|
||||
lKNOWN_NODNS = o[oBAD_ROOT][root]
|
||||
return l
|
||||
|
||||
def icheck_torrc(sFile, oArgs):
|
||||
l = open(sFile, 'rt').readlines()
|
||||
a = {}
|
||||
for elt in l:
|
||||
elt = elt.strip()
|
||||
if not elt or not ' ' in elt: continue
|
||||
k,v = elt.split(' ', 1)
|
||||
a[k] = v
|
||||
keys = a
|
||||
|
||||
if 'HashedControlPassword' not in keys:
|
||||
LOG.info('Add HashedControlPassword for security')
|
||||
print('run: tor --hashcontrolpassword <TopSecretWord>')
|
||||
if 'ExcludeExitNodes' in keys:
|
||||
elt = 'BadNodes.ExcludeExitNodes.BadExit'
|
||||
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
|
||||
print(f"move to the {elt} section as a list")
|
||||
if 'GuardNodes' in keys:
|
||||
elt = 'GoodNodes.GuardNodes'
|
||||
LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}")
|
||||
print(f"move to the {elt} section as a list")
|
||||
if 'ExcludeNodes' in keys:
|
||||
elt = 'BadNodes.ExcludeNodes.BadExit'
|
||||
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
|
||||
print(f"move to the {elt} section as a list")
|
||||
if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'):
|
||||
LOG.info('Add ControlSocket /run/tor/control for us')
|
||||
print('ControlSocket /run/tor/control GroupWritable RelaxDirModeCheck')
|
||||
if 'UseMicrodescriptors' not in keys or keys['UseMicrodescriptors'] != '1':
|
||||
LOG.info('Add UseMicrodescriptors 0 for us')
|
||||
print('UseMicrodescriptors 0')
|
||||
if 'AutomapHostsSuffixes' not in keys:
|
||||
LOG.info('Add AutomapHostsSuffixes for onions')
|
||||
print('AutomapHostsSuffixes .exit,.onion')
|
||||
if 'AutoMapHostsOnResolve' not in keys:
|
||||
LOG.info('Add AutoMapHostsOnResolve for onions')
|
||||
print('AutoMapHostsOnResolve 1')
|
||||
if 'VirtualAddrNetworkIPv4' not in keys:
|
||||
LOG.info('Add VirtualAddrNetworkIPv4 for onions')
|
||||
print('VirtualAddrNetworkIPv4 172.16.0.0/12')
|
||||
return 0
|
||||
|
||||
oGOOD_NODES = {}
|
||||
oGOOD_ROOT = 'GoodNodes'
|
||||
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
||||
|
|
@ -248,13 +230,14 @@ def lIntroductionPoints(controller, lOnions):
|
|||
l += [introduction_point.address]
|
||||
return l
|
||||
|
||||
lBAD_URLS = []
|
||||
tBAD_URLS = set()
|
||||
lATS = ['abuse', 'email']
|
||||
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
|
||||
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
|
||||
'sandbox', 'offlinemasterkey']
|
||||
def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050):
|
||||
global lBAD_URLS
|
||||
global tBAD_URLS
|
||||
global lKNOWN_NODNS
|
||||
# cleanups for yaml
|
||||
for elt in lINTS:
|
||||
if elt in a:
|
||||
|
|
@ -271,12 +254,17 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|||
|
||||
a.update({'fps': []})
|
||||
keys = list(a.keys())
|
||||
|
||||
# test the url for fps and add it to the array
|
||||
if 'proof' not in keys:
|
||||
LOG.warn(f"{fp} 'proof' not in {keys}")
|
||||
return a
|
||||
|
||||
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
|
||||
aCachedContact = aTRUST_DB_INDEX[fp]
|
||||
if aCachedContact['email'] = a['email']:
|
||||
return aCachedContact
|
||||
|
||||
|
||||
if 'url' not in keys:
|
||||
if 'uri' not in keys:
|
||||
a['url'] = ''
|
||||
|
|
@ -292,12 +280,21 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|||
a['url'] = 'https:' +a['url']
|
||||
|
||||
# domain should be a unique key for contacts
|
||||
domain = a['url'][8:]
|
||||
domain = a['url'][8:].strip('/')
|
||||
try:
|
||||
ip = sTorResolve(domain)
|
||||
except Exception as e:
|
||||
lpair = getaddrinfo(domain, 443)
|
||||
if lpait is None:
|
||||
try:
|
||||
lpair = getaddrinfo(domain, 443)
|
||||
except (socket.gaierror, ) as e:
|
||||
LOG.debug("{e}")
|
||||
lpair = None
|
||||
lKNOWN_NODNS.append(domain)
|
||||
except Exception as e:
|
||||
LOG.warn("{e}")
|
||||
lpair = None
|
||||
lKNOWN_NODNS.append(domain)
|
||||
if lpair is None:
|
||||
LOG.warn(f"TorResolv and getaddrinfo failed for {domain}")
|
||||
return a
|
||||
ip = lpair[0]
|
||||
|
|
@ -324,24 +321,24 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
|
|||
LOG.exception(f"AttributeError downloading from {domain} {e}")
|
||||
except CertificateError as e:
|
||||
LOG.warn(f"CertificateError downloading from {domain} {e}")
|
||||
lBAD_URLS += [a['url']]
|
||||
tBAD_URLS.add(a['url'])
|
||||
except TrustorError as e:
|
||||
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
|
||||
lBAD_URLS += [a['url']]
|
||||
tBAD_URLS.add(a['url'])
|
||||
except (BaseException ) as e:
|
||||
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
|
||||
else:
|
||||
if o.status_code >= 300:
|
||||
LOG.warn(f"Error downloading from {domain} {o.status_code} {o.reason}")
|
||||
# any reason retry?
|
||||
lBAD_URLS += [a['url']]
|
||||
tBAD_URLS.add(a['url'])
|
||||
return a
|
||||
|
||||
l = o.text.upper().strip().split('\n')
|
||||
if not l:
|
||||
# already squacked in lD
|
||||
LOG.warn(f"Downloading from {domain} empty for {fp}")
|
||||
lBAD_URLS += [a['url']]
|
||||
tBAD_URLS.add(a['url'])
|
||||
else:
|
||||
a['fps'] = [elt for elt in l if elt and len(elt) == 40
|
||||
and not elt.startswith('#')]
|
||||
|
|
@ -521,13 +518,23 @@ def iMain(lArgs):
|
|||
icheck_torrc(sFile, oArgs)
|
||||
|
||||
global aTRUST_DB
|
||||
global aTRUST_DB_INDEX
|
||||
|
||||
sFile = oArgs.proof_output
|
||||
if sFile and os.path.exists(sFile):
|
||||
try:
|
||||
with open(sFile, 'rt') as oFd:
|
||||
aTRUST_DB = yaml.safe_load(oFd)
|
||||
except:
|
||||
aTRUST_DB = {}
|
||||
# reverse lookup of fps to contacts
|
||||
# but...
|
||||
for k,v in aTRUST_DB:
|
||||
aTRUST_DB_INDEX[k] = v
|
||||
if 'fps' in aTRUST_DB[k].keys():
|
||||
for fp in aTRUST_DB[k]['fps']:
|
||||
aTRUST_DB_INDEX[fp] = v
|
||||
|
||||
except Exception as e:
|
||||
LOG.warn(f"Error reading YAML TrustDB {sFile} {e}")
|
||||
|
||||
if os.path.exists(oArgs.proxy_ctl):
|
||||
controller = oMakeController(sSock=oArgs.proxy_ctl)
|
||||
|
|
@ -615,7 +622,12 @@ def iMain(lArgs):
|
|||
i = c.find(' ')
|
||||
if i >=0:
|
||||
c = c[:i]
|
||||
LOG.info(f"{relay.fingerprint} skipping 'dns-rsa' {c} {sofar}")
|
||||
if c in lKNOWN_NODNS:
|
||||
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {c} {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
|
||||
LOG.info(f"skipping 'dns-rsa' {relay.fingerprint}.{c} {sofar}")
|
||||
iDnsContact += 1
|
||||
continue
|
||||
|
||||
|
|
@ -626,17 +638,18 @@ def iMain(lArgs):
|
|||
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
if 'url' in a and a['url'] and a['url'] in lBAD_URLS:
|
||||
# The fp is using a contact with a URL we know is bad
|
||||
LOG.info(f"{relay.fingerprint} skipping in lBAD_URLS {a['url']} {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
domain = a['url'][8:]
|
||||
if domain in lKNOWN_NODNS:
|
||||
# The fp is using a contact with a URL we know is nonexistent
|
||||
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
if 'url' in a and a['url']:
|
||||
if a['url'] in tBAD_URLS:
|
||||
# The fp is using a contact with a URL we know is bad
|
||||
LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']} {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
domain = a['url'].replace('https://', '').replace('http://', '')
|
||||
if domain in lKNOWN_NODNS:
|
||||
# The fp is using a contact with a URL we know is bogus
|
||||
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
|
||||
exit_excludelist.append(relay.fingerprint)
|
||||
continue
|
||||
|
||||
|
||||
b = aVerifyContact(list(a.values())[0],
|
||||
|
|
@ -661,9 +674,13 @@ def iMain(lArgs):
|
|||
iFakeContact += 1
|
||||
aBadContacts[relay.fingerprint] = b
|
||||
continue
|
||||
|
||||
|
||||
# great contact had good fps and we are in them
|
||||
lProofGoodFps += b['fps']
|
||||
if relay.fingerprint in aProofUri.keys():
|
||||
# a cached entry
|
||||
continue
|
||||
|
||||
LOG.info(f"{relay.fingerprint} verified {b['url']} {sofar}")
|
||||
# add our contact info to the trustdb
|
||||
aProofUri[relay.fingerprint] = b
|
||||
|
|
@ -706,7 +723,10 @@ def iMain(lArgs):
|
|||
|
||||
global oBAD_NODES
|
||||
oBAD_NODES['BadNodes']['ExcludeNodes']['BadExit'] = exit_excludelist
|
||||
global lKNOWN_NODNS
|
||||
o[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
|
||||
vwrite_badnodes(oArgs)
|
||||
|
||||
global oGOOD_NODES
|
||||
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = lProofGoodFps
|
||||
vwrite_goodnodes(oArgs)
|
||||
|
|
@ -739,7 +759,7 @@ def iMain(lArgs):
|
|||
except stem.SocketClosed as e:
|
||||
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
|
||||
retval += 1
|
||||
)
|
||||
|
||||
return retval
|
||||
|
||||
except InvalidRequest as e:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue