|
|
|
@ -17,7 +17,37 @@ or to relays that do not have ContactInfo that is verified to include them.
|
|
|
|
|
"""
|
|
|
|
|
__prolog__ = __doc__
|
|
|
|
|
|
|
|
|
|
__doc__ +="""But there's a problem, and your Tor notice.log will tell you about it:
|
|
|
|
|
sGOOD_NODES = """
|
|
|
|
|
---
|
|
|
|
|
GoodNodes:
|
|
|
|
|
EntryNodes: []
|
|
|
|
|
Relays:
|
|
|
|
|
# ExitNodes will be overwritten by this program
|
|
|
|
|
ExitNodes: []
|
|
|
|
|
IntroductionPoints: []
|
|
|
|
|
# use the Onions section to list onion services you want the
|
|
|
|
|
# Introduction Points whitelisted - these points may change daily
|
|
|
|
|
# Look in tor's notice.log for 'Every introduction point for service'
|
|
|
|
|
Onions: []
|
|
|
|
|
# use the Services list to list elays you want the whitelisted
|
|
|
|
|
# Look in tor's notice.log for 'Wanted to contact directory mirror'
|
|
|
|
|
Services: []
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
sBAD_NODES = """
|
|
|
|
|
BadNodes:
|
|
|
|
|
# list the internet domains you know are bad so you don't
|
|
|
|
|
# waste time trying to download contacts from them.
|
|
|
|
|
ExcludeDomains: []
|
|
|
|
|
ExcludeNodes:
|
|
|
|
|
# BadExit will be overwritten by this program
|
|
|
|
|
BadExit: []
|
|
|
|
|
# list MyBadExit in --bad_sections if you want it used, to exclude nodes
|
|
|
|
|
# or any others as a list separated by comma(,)
|
|
|
|
|
MyBadExit: []
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
__doc__ +=f"""But there's a problem, and your Tor notice.log will tell you about it:
|
|
|
|
|
you could exclude the relays needed to access hidden services or mirror
|
|
|
|
|
directories. So we need to add to the process the concept of a whitelist.
|
|
|
|
|
In addition, we may have our own blacklist of nodes we want to exclude,
|
|
|
|
@ -26,18 +56,16 @@ or use these lists for other applications like selektor.
|
|
|
|
|
So we make two files that are structured in YAML:
|
|
|
|
|
```
|
|
|
|
|
/etc/tor/yaml/torrc-goodnodes.yaml
|
|
|
|
|
GoodNodes:
|
|
|
|
|
Relays:
|
|
|
|
|
IntroductionPoints:
|
|
|
|
|
- NODEFINGERPRINT
|
|
|
|
|
...
|
|
|
|
|
{sGOOD_NODES}
|
|
|
|
|
|
|
|
|
|
By default all sections of the goodnodes.yaml are used as a whitelist.
|
|
|
|
|
|
|
|
|
|
Use the GoodNodes/Onions list to list onion services you want the
|
|
|
|
|
Introduction Points whitelisted - these points may change daily
|
|
|
|
|
Look in tor's notice.log for warnings of 'Every introduction point for service'
|
|
|
|
|
|
|
|
|
|
/etc/tor/yaml/torrc-badnodes.yaml
|
|
|
|
|
BadNodes:
|
|
|
|
|
ExcludeExitNodes:
|
|
|
|
|
BadExit:
|
|
|
|
|
- 0000000000000000000000000000000000000007
|
|
|
|
|
{sBAD_NODES}
|
|
|
|
|
```
|
|
|
|
|
That part requires [PyYAML](https://pyyaml.org/wiki/PyYAML)
|
|
|
|
|
https://github.com/yaml/pyyaml/ or ```ruamel```: do
|
|
|
|
@ -46,7 +74,7 @@ the advantage of the former is that it preserves comments.
|
|
|
|
|
|
|
|
|
|
(You may have to run this as the Tor user to get RW access to
|
|
|
|
|
/run/tor/control, in which case the directory for the YAML files must
|
|
|
|
|
be group Tor writeable, and its parents group Tor RX.)
|
|
|
|
|
be group Tor writeable, and its parent's directories group Tor RX.)
|
|
|
|
|
|
|
|
|
|
Because you don't want to exclude the introduction points to any onion
|
|
|
|
|
you want to connect to, ```--white_onions``` should whitelist the
|
|
|
|
@ -54,6 +82,13 @@ introduction points to a comma sep list of onions; we fixed stem to do this:
|
|
|
|
|
* https://github.com/torproject/stem/issues/96
|
|
|
|
|
* https://gitlab.torproject.org/legacy/trac/-/issues/25417
|
|
|
|
|
|
|
|
|
|
Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
|
|
|
|
|
you want the Introduction Points whitelisted - these points may change daily.
|
|
|
|
|
Look in tor's notice.log for 'Every introduction point for service'
|
|
|
|
|
|
|
|
|
|
```notice_log``` will parse the notice log for warnings about relays and
|
|
|
|
|
services that will then be whitelisted.
|
|
|
|
|
|
|
|
|
|
```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
|
|
|
|
|
|
|
|
|
|
```--good_contacts``` will write the contact info as a ciiss dictionary
|
|
|
|
@ -78,7 +113,7 @@ list of fingerprints to ```ExitNodes```, a whitelist of relays to use as exits.
|
|
|
|
|
3. clean relays that don't have "good' contactinfo. (implies 1)
|
|
|
|
|
```=Empty,NoEmail,NotGood```
|
|
|
|
|
|
|
|
|
|
The default is ```=Empty,NotGood``` ; ```NoEmail``` is inherently imperfect
|
|
|
|
|
The default is ```Empty,NoEmail,NotGood``` ; ```NoEmail``` is inherently imperfect
|
|
|
|
|
in that many of the contact-as-an-email are obfuscated, but we try anyway.
|
|
|
|
|
|
|
|
|
|
To be "good" the ContactInfo must:
|
|
|
|
@ -87,7 +122,8 @@ To be "good" the ContactInfo must:
|
|
|
|
|
3. must support getting the file with a valid SSL cert from a recognized authority
|
|
|
|
|
4. (not in the spec but added by Python) must use a TLS SSL > v1
|
|
|
|
|
5. must have a fingerprint list in the file
|
|
|
|
|
6. must have the FP that got us the contactinfo in the fingerprint list in the file,
|
|
|
|
|
6. must have the FP that got us the contactinfo in the fingerprint list in the file.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
For usage, do ```python3 exclude_badExits.py --help`
|
|
|
|
|
|
|
|
|
@ -175,28 +211,12 @@ sEXCLUDE_EXIT_GROUP = 'ExcludeNodes'
|
|
|
|
|
sINCLUDE_EXIT_KEY = 'ExitNodes'
|
|
|
|
|
|
|
|
|
|
oBAD_ROOT = 'BadNodes'
|
|
|
|
|
aBAD_NODES = safe_load("""
|
|
|
|
|
BadNodes:
|
|
|
|
|
ExcludeDomains: []
|
|
|
|
|
ExcludeNodes:
|
|
|
|
|
# BadExit will be overwritten
|
|
|
|
|
BadExit: []
|
|
|
|
|
# list MyBadExit in --bad_sections if you want it used
|
|
|
|
|
MyBadExit: []
|
|
|
|
|
""")
|
|
|
|
|
aBAD_NODES = safe_load(sBAD_NODES)
|
|
|
|
|
|
|
|
|
|
sGOOD_ROOT = 'GoodNodes'
|
|
|
|
|
sINCLUDE_GUARD_KEY = 'EntryNodes'
|
|
|
|
|
sEXCLUDE_DOMAINS = 'ExcludeDomains'
|
|
|
|
|
oGOOD_NODES = safe_load("""
|
|
|
|
|
GoodNodes:
|
|
|
|
|
EntryNodes: []
|
|
|
|
|
Relays:
|
|
|
|
|
ExitNodes: []
|
|
|
|
|
IntroductionPoints: []
|
|
|
|
|
Onions: []
|
|
|
|
|
Services: []
|
|
|
|
|
""")
|
|
|
|
|
aGOOD_NODES = safe_load(sGOOD_NODES)
|
|
|
|
|
|
|
|
|
|
lKNOWN_NODNS = []
|
|
|
|
|
tMAYBE_NODNS = set()
|
|
|
|
@ -230,13 +250,13 @@ def lYamlBadNodes(sFile,
|
|
|
|
|
return l
|
|
|
|
|
|
|
|
|
|
def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
|
|
|
|
|
global oGOOD_NODES
|
|
|
|
|
global aGOOD_NODES
|
|
|
|
|
l = []
|
|
|
|
|
if not yaml: return l
|
|
|
|
|
if os.path.exists(sFile):
|
|
|
|
|
with open(sFile, 'rt') as oFd:
|
|
|
|
|
o = safe_load(oFd)
|
|
|
|
|
oGOOD_NODES = o
|
|
|
|
|
aGOOD_NODES = o
|
|
|
|
|
if 'EntryNodes' in o[sGOOD_ROOT].keys():
|
|
|
|
|
l = o[sGOOD_ROOT]['EntryNodes']
|
|
|
|
|
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
|
|
|
|
@ -644,9 +664,9 @@ def oMainArgparser(_=None):
|
|
|
|
|
default='127.0.0.1',
|
|
|
|
|
help='proxy host')
|
|
|
|
|
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
|
|
|
|
|
help='proxy control port')
|
|
|
|
|
help='proxy socks port')
|
|
|
|
|
parser.add_argument('--proxy_ctl', '--proxy-ctl',
|
|
|
|
|
default='/run/tor/control' if os.path.exists('/run/tor/control') else 9051,
|
|
|
|
|
default='/run/tor/control' if os.path.exists('/run/tor/control') else '9051',
|
|
|
|
|
type=str,
|
|
|
|
|
help='control socket - or port')
|
|
|
|
|
|
|
|
|
@ -689,9 +709,12 @@ def oMainArgparser(_=None):
|
|
|
|
|
parser.add_argument('--torrc_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'torrc.new'),
|
|
|
|
|
help="Write the torrc configuration to a file")
|
|
|
|
|
parser.add_argument('--hs_dir', type=str,
|
|
|
|
|
default='/var/lib/tor',
|
|
|
|
|
help="Parse the files name hostname below this dir to find Hidden Services to whitelist")
|
|
|
|
|
parser.add_argument('--notice_log', type=str,
|
|
|
|
|
default='',
|
|
|
|
|
help="Parse the notice log for relays and services (not yet)")
|
|
|
|
|
help="Parse the notice log for relays and services")
|
|
|
|
|
parser.add_argument('--relays_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'relays.json'),
|
|
|
|
|
help="Write the download relays in json to a file")
|
|
|
|
@ -718,23 +741,23 @@ def vwrite_good_contacts(oargs):
|
|
|
|
|
yaml.dump(aBAD_CONTACTS_DB, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
|
|
|
|
|
def vwrite_badnodes(oargs, aBAD_NODES, slen):
|
|
|
|
|
def vwrite_badnodes(oargs, aBAD_NODES, slen, stag):
|
|
|
|
|
if not aBAD_NODES: return
|
|
|
|
|
tmp = oargs.bad_nodes +'.tmp'
|
|
|
|
|
bak = oargs.bad_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBAD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {slen} to {oargs.bad_nodes}")
|
|
|
|
|
LOG.info(f"Wrote {slen} to {stag} in {oargs.bad_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oargs.bad_nodes):
|
|
|
|
|
os.rename(oargs.bad_nodes, bak)
|
|
|
|
|
os.rename(tmp, oargs.bad_nodes)
|
|
|
|
|
|
|
|
|
|
def vwrite_goodnodes(oargs, oGOOD_NODES, ilen):
|
|
|
|
|
def vwrite_goodnodes(oargs, aGOOD_NODES, ilen):
|
|
|
|
|
tmp = oargs.good_nodes +'.tmp'
|
|
|
|
|
bak = oargs.good_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(oGOOD_NODES, oFYaml)
|
|
|
|
|
yaml.dump(aGOOD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {ilen} good relays to {oargs.good_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oargs.good_nodes):
|
|
|
|
@ -1022,23 +1045,31 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} EntryNodes from {oargs.good_nodes}")
|
|
|
|
|
|
|
|
|
|
t = set()
|
|
|
|
|
if sGOOD_ROOT in oGOOD_NODES and 'Relays' in oGOOD_NODES[sGOOD_ROOT] and \
|
|
|
|
|
'IntroductionPoints' in oGOOD_NODES[sGOOD_ROOT]['Relays'].keys():
|
|
|
|
|
t = set(oGOOD_NODES[sGOOD_ROOT]['Relays']['IntroductionPoints'])
|
|
|
|
|
if 'IntroductionPoints' in aGOOD_NODES[sGOOD_ROOT]['Relays'].keys():
|
|
|
|
|
t = set(aGOOD_NODES[sGOOD_ROOT]['Relays']['IntroductionPoints'])
|
|
|
|
|
|
|
|
|
|
if oargs.hs_dir and os.path.exists(oargs.hs_dir):
|
|
|
|
|
for (dirpath, dirnames, filenames,) in os.walk(oargs.hs_dir):
|
|
|
|
|
for f in filenames:
|
|
|
|
|
if f != 'hostname': continue
|
|
|
|
|
with open(os.path.join(dirpath, f), 'rt') as oFd:
|
|
|
|
|
son = oFd.read()
|
|
|
|
|
t.update(son)
|
|
|
|
|
LOG.info(f"Added {son} to the list for Introduction Points")
|
|
|
|
|
|
|
|
|
|
if oargs.notice_log and os.path.exists(oargs.notice_log):
|
|
|
|
|
tmp = tempfile.mktemp()
|
|
|
|
|
i = os.system(f"grep 'Every introduction point for service' {oargs.notice_log} |sed -e 's/.* service //' -e 's/ is .*//'|sort -u |sed -e '/ /d' > {tmp}")
|
|
|
|
|
if i:
|
|
|
|
|
with open(tmp, 'rt') as oFd:
|
|
|
|
|
lnew = oFd.readlines()
|
|
|
|
|
t.update(set(lnew))
|
|
|
|
|
tnew = {elt.strip() for elt in oFd.readlines()}
|
|
|
|
|
t.update(tnew)
|
|
|
|
|
LOG.info(f"Whitelist {len(lnew)} services from {oargs.notice_log}")
|
|
|
|
|
os.remove(tmp)
|
|
|
|
|
|
|
|
|
|
w = set()
|
|
|
|
|
if sGOOD_ROOT in oGOOD_NODES and 'Services' in oGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
w = set(oGOOD_NODES[sGOOD_ROOT]['Services'])
|
|
|
|
|
if sGOOD_ROOT in aGOOD_NODES and 'Services' in aGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
w = set(aGOOD_NODES[sGOOD_ROOT]['Services'])
|
|
|
|
|
if len(w) > 0:
|
|
|
|
|
LOG.info(f"Whitelist {len(w)} relays from {sGOOD_ROOT}/Services")
|
|
|
|
|
|
|
|
|
@ -1054,10 +1085,10 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
twhitelist_set.update(w)
|
|
|
|
|
|
|
|
|
|
w = set()
|
|
|
|
|
if 'Onions' in oGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
if 'Onions' in aGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
# Provides the descriptor for a hidden service. The **address** is the
|
|
|
|
|
# '.onion' address of the hidden service
|
|
|
|
|
w = set(oGOOD_NODES[sGOOD_ROOT]['Onions'])
|
|
|
|
|
w = set(aGOOD_NODES[sGOOD_ROOT]['Onions'])
|
|
|
|
|
if oargs.white_onions:
|
|
|
|
|
w.update(oargs.white_onions.split(','))
|
|
|
|
|
if oargs.points_timeout > 0:
|
|
|
|
@ -1088,7 +1119,7 @@ def iMain(lArgs):
|
|
|
|
|
global aGOOD_CONTACTS_FPS
|
|
|
|
|
global aBAD_CONTACTS_DB
|
|
|
|
|
global aBAD_NODES
|
|
|
|
|
global oGOOD_NODES
|
|
|
|
|
global aGOOD_NODES
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
global aRELAYS_DB
|
|
|
|
|
global aRELAYS_DB_INDEX
|
|
|
|
@ -1198,7 +1229,7 @@ def iMain(lArgs):
|
|
|
|
|
with open(oargs.torrc_output, 'wt') as oFTorrc:
|
|
|
|
|
oFTorrc.write(f"{sEXCLUDE_EXIT_GROUP} {','.join(texclude_set)}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aGOOD_CONTACTS_FPS.keys())}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
|
|
|
|
|
LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
|
|
|
|
|
oFTorrc.close()
|
|
|
|
|
|
|
|
|
@ -1214,12 +1245,13 @@ def iMain(lArgs):
|
|
|
|
|
aBAD_NODES[oBAD_ROOT][sEXCLUDE_EXIT_GROUP]['BadExit'] = list(texclude_set)
|
|
|
|
|
aBAD_NODES[oBAD_ROOT][sEXCLUDE_DOMAINS] = lKNOWN_NODNS
|
|
|
|
|
if oargs.bad_nodes:
|
|
|
|
|
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)))
|
|
|
|
|
stag = sEXCLUDE_EXIT_GROUP + '/BadExit'
|
|
|
|
|
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)), stag)
|
|
|
|
|
|
|
|
|
|
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aGOOD_CONTACTS_FPS.keys())
|
|
|
|
|
aGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aGOOD_CONTACTS_FPS.keys())
|
|
|
|
|
# EntryNodes are readony
|
|
|
|
|
if oargs.good_nodes:
|
|
|
|
|
vwrite_goodnodes(oargs, oGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
|
|
|
|
|
vwrite_goodnodes(oargs, aGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
|
|
|
|
|
|
|
|
|
|
vwritefinale(oargs)
|
|
|
|
|
|
|
|
|
@ -1245,15 +1277,15 @@ def iMain(lArgs):
|
|
|
|
|
LOG.debug(repr(l))
|
|
|
|
|
retval += 1
|
|
|
|
|
|
|
|
|
|
if 'EntryNodes' in oGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
if 'EntryNodes' in aGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
try:
|
|
|
|
|
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])} guard nodes")
|
|
|
|
|
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])} guard nodes")
|
|
|
|
|
# FixMe for now override StrictNodes it may be unusable otherwise
|
|
|
|
|
controller.set_conf(sINCLUDE_GUARD_KEY,
|
|
|
|
|
oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])
|
|
|
|
|
aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])
|
|
|
|
|
except (Exception, stem.InvalidRequest, stem.SocketClosed,) as e: # noqa
|
|
|
|
|
LOG.error(f"Failed setting {sINCLUDE_GUARD_KEY} guard nodes in Tor {e}")
|
|
|
|
|
LOG.debug(repr(list(oGOOD_NODES[sGOOD_ROOT]['EntryNodes'])))
|
|
|
|
|
LOG.debug(repr(list(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])))
|
|
|
|
|
retval += 1
|
|
|
|
|
|
|
|
|
|
cur = controller.get_conf('StrictNodes')
|
|
|
|
|