|
|
|
|
@ -1,3 +1,4 @@
|
|
|
|
|
#!/usr/local/bin/python3.sh
|
|
|
|
|
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
|
|
|
|
|
|
|
|
|
|
# https://github.com/nusenu/noContactInfo_Exit_Excluder
|
|
|
|
|
@ -64,8 +65,8 @@ Use the GoodNodes/Onions list to list onion services you want the
|
|
|
|
|
Introduction Points whitelisted - these points may change daily
|
|
|
|
|
Look in tor's notice.log for warnings of 'Every introduction point for service'
|
|
|
|
|
|
|
|
|
|
```--hs_dir``` ```default='/var/lib/tor'``` will make the program
|
|
|
|
|
parse the files named ```hostname``` below this dir to find
|
|
|
|
|
```--hs_dir``` ```default='/var/lib/tor'``` will make the program
|
|
|
|
|
parse the files named ```hostname``` below this dir to find
|
|
|
|
|
Hidden Services to whitelist.
|
|
|
|
|
|
|
|
|
|
The Introduction Points can change during the day, so you may want to
|
|
|
|
|
@ -97,7 +98,7 @@ Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
|
|
|
|
|
you want the Introduction Points whitelisted - these points may change daily.
|
|
|
|
|
Look in tor's notice.log for 'Every introduction point for service'
|
|
|
|
|
|
|
|
|
|
```notice_log``` will parse the notice log for warnings about relays and
|
|
|
|
|
```notice_log``` will parse the notice log for warnings about relays and
|
|
|
|
|
services that will then be whitelisted.
|
|
|
|
|
|
|
|
|
|
```--torrc``` will read a file like /etc/tor/torrc and make some
|
|
|
|
|
@ -154,19 +155,20 @@ See [exclude_badExits.txt](./exclude_badExits.txt)
|
|
|
|
|
# https://github.com/nusenu/trustor-example-trust-config/blob/main/trust_config
|
|
|
|
|
# https://github.com/nusenu/tor-relay-operator-ids-trust-information
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import os
|
|
|
|
|
import json
|
|
|
|
|
import re
|
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
|
|
|
|
import time
|
|
|
|
|
from io import StringIO
|
|
|
|
|
import logging
|
|
|
|
|
import warnings
|
|
|
|
|
|
|
|
|
|
import stem
|
|
|
|
|
from stem import InvalidRequest
|
|
|
|
|
from stem.connection import IncorrectPassword
|
|
|
|
|
from stem.util.tor_tools import is_valid_fingerprint
|
|
|
|
|
|
|
|
|
|
import urllib3
|
|
|
|
|
from urllib3.util.ssl_match_hostname import CertificateError
|
|
|
|
|
|
|
|
|
|
@ -192,7 +194,6 @@ except:
|
|
|
|
|
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
|
|
|
|
|
|
|
|
|
|
from support_onions import (bAreWeConnected, icheck_torrc, lIntroductionPoints,
|
|
|
|
|
oGetStemController, vwait_for_controller,
|
|
|
|
|
yKNOWN_NODNS, zResolveDomain)
|
|
|
|
|
|
|
|
|
|
from trustor_poc import TrustorError, idns_validate
|
|
|
|
|
@ -203,13 +204,6 @@ try:
|
|
|
|
|
except:
|
|
|
|
|
httpx = None
|
|
|
|
|
from trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
|
|
|
|
|
|
|
|
|
|
global LOG
|
|
|
|
|
import logging
|
|
|
|
|
import warnings
|
|
|
|
|
|
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
LOG = logging.getLogger()
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
from torcontactinfo import TorContactInfoParser
|
|
|
|
|
@ -217,9 +211,15 @@ try:
|
|
|
|
|
except ImportError:
|
|
|
|
|
oPARSER = None
|
|
|
|
|
|
|
|
|
|
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
|
|
|
|
|
|
|
|
|
|
ETC_DIR = '/usr/local/etc/tor/yaml'
|
|
|
|
|
from exclude_utils import (aCleanContact, sCleanEmail, aParseContact,
|
|
|
|
|
oStemController, oMainArgparser,
|
|
|
|
|
vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
|
|
|
|
|
vwritefinale, vsetup_logging)
|
|
|
|
|
|
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
global LOG
|
|
|
|
|
LOG = logging.getLogger()
|
|
|
|
|
|
|
|
|
|
aGOOD_CONTACTS_DB = {}
|
|
|
|
|
aGOOD_CONTACTS_FPS = {}
|
|
|
|
|
aBAD_CONTACTS_DB = {}
|
|
|
|
|
@ -242,6 +242,21 @@ aGOOD_NODES = safe_load(sGOOD_NODES)
|
|
|
|
|
|
|
|
|
|
lKNOWN_NODNS = []
|
|
|
|
|
tMAYBE_NODNS = set()
|
|
|
|
|
|
|
|
|
|
def tExcludeSet(oargs, sEXCLUDE_EXIT_GROUP):
|
|
|
|
|
texclude_set = set()
|
|
|
|
|
sections = {'BadExit'}
|
|
|
|
|
if oargs.bad_nodes and os.path.exists(oargs.bad_nodes):
|
|
|
|
|
if oargs.bad_sections:
|
|
|
|
|
sections.update(oargs.bad_sections.split(','))
|
|
|
|
|
texclude_set = set(lYamlBadNodes(oargs.bad_nodes,
|
|
|
|
|
tWanted=sections,
|
|
|
|
|
section=sEXCLUDE_EXIT_GROUP))
|
|
|
|
|
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
|
|
|
|
|
|
|
|
|
|
return texclude_set
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def lYamlBadNodes(sFile,
|
|
|
|
|
section=sEXCLUDE_EXIT_GROUP,
|
|
|
|
|
tWanted=None):
|
|
|
|
|
@ -314,7 +329,7 @@ lAT_REPS = ['[]', ' at ', '(at)', '[at]', '<at>', '(att)', '_at_',
|
|
|
|
|
]
|
|
|
|
|
lDOT_REPS = [' point ', ' dot ', '[dot]', '(dot)', '_dot_', '!dot!', '<.>',
|
|
|
|
|
'<:dot:>', '|dot--|', ' d07 ', '<dot=>', '(dot]', '{dot)',
|
|
|
|
|
'd.t', "'dot'", '(d)', '-dot-', ' adot ',
|
|
|
|
|
'd.t', "'dot'", '(d)', '-dot-', ' adot ',
|
|
|
|
|
'(d)', ' . ', '[punto]', '(point)', '"dot"', '{.}',
|
|
|
|
|
'--separator--', '|=dot|', ' period ', ')dot(',
|
|
|
|
|
]
|
|
|
|
|
@ -342,55 +357,7 @@ lNO_EMAIL = [
|
|
|
|
|
r'<nothing/at\\mail.de>',
|
|
|
|
|
]
|
|
|
|
|
#
|
|
|
|
|
lMORONS = ['hoster:Quintex Alliance Consulting ']
|
|
|
|
|
|
|
|
|
|
def sCleanEmail(s):
|
|
|
|
|
s = s.lower()
|
|
|
|
|
for elt in lAT_REPS:
|
|
|
|
|
if not elt.startswith(' '):
|
|
|
|
|
s = s.replace(' ' + elt + ' ', '@')
|
|
|
|
|
s = s.replace(elt, '@')
|
|
|
|
|
for elt in lDOT_REPS:
|
|
|
|
|
if not elt.startswith(' '):
|
|
|
|
|
s = s.replace(' ' + elt + ' ', '.')
|
|
|
|
|
s = s.replace(elt, '.')
|
|
|
|
|
s = s.replace('(dash)', '-')
|
|
|
|
|
s = s.replace('hyphen ', '-')
|
|
|
|
|
for elt in lNO_EMAIL:
|
|
|
|
|
s = s.replace(elt, '?')
|
|
|
|
|
return s
|
|
|
|
|
|
|
|
|
|
lEMAILS = ['abuse', 'email']
|
|
|
|
|
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
|
|
|
|
|
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
|
|
|
|
|
'sandbox', 'offlinemasterkey']
|
|
|
|
|
def aCleanContact(a):
|
|
|
|
|
# cleanups
|
|
|
|
|
for elt in lINTS:
|
|
|
|
|
if elt in a:
|
|
|
|
|
a[elt] = int(a[elt])
|
|
|
|
|
for elt in lBOOLS:
|
|
|
|
|
if elt not in a: continue
|
|
|
|
|
if a[elt] in ['y', 'yes', 'true', 'True']:
|
|
|
|
|
a[elt] = True
|
|
|
|
|
else:
|
|
|
|
|
a[elt] = False
|
|
|
|
|
for elt in lEMAILS:
|
|
|
|
|
if elt not in a: continue
|
|
|
|
|
a[elt] = sCleanEmail(a[elt])
|
|
|
|
|
if 'url' in a.keys():
|
|
|
|
|
a['url'] = a['url'].rstrip('/')
|
|
|
|
|
if a['url'].startswith('http://'):
|
|
|
|
|
domain = a['url'].replace('http://', '')
|
|
|
|
|
elif a['url'].startswith('https://'):
|
|
|
|
|
domain = a['url'].replace('https://', '')
|
|
|
|
|
else:
|
|
|
|
|
domain = a['url']
|
|
|
|
|
a['url'] = 'https://' + domain
|
|
|
|
|
a.update({'fps': []})
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def bVerifyContact(a=None, fp=None, https_cafile=None):
|
|
|
|
|
def bVerifyContact(lAT_REPS, lDOT_REPS, lNO_EMAIL, a, fp, https_cafile=None):
|
|
|
|
|
global aFP_EMAIL
|
|
|
|
|
global tBAD_URLS
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
@ -399,9 +366,9 @@ def bVerifyContact(a=None, fp=None, https_cafile=None):
|
|
|
|
|
assert a
|
|
|
|
|
assert fp
|
|
|
|
|
assert https_cafile
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
keys = list(a.keys())
|
|
|
|
|
a = aCleanContact(a)
|
|
|
|
|
a = aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL)
|
|
|
|
|
a['fp'] = fp
|
|
|
|
|
if 'email' not in keys:
|
|
|
|
|
a['email'] = ''
|
|
|
|
|
@ -455,7 +422,7 @@ def oVerifyUrl(url, domain, fp=None, https_cafile=None, timeout=20, host='127.0.
|
|
|
|
|
if url in tBAD_URLS:
|
|
|
|
|
LOG.debug(f"BC Known bad url from {domain} for {fp}")
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
o = None
|
|
|
|
|
try:
|
|
|
|
|
if httpx:
|
|
|
|
|
@ -464,7 +431,7 @@ def oVerifyUrl(url, domain, fp=None, https_cafile=None, timeout=20, host='127.0.
|
|
|
|
|
o = oDownloadUrl(url, https_cafile,
|
|
|
|
|
timeout=timeout, host=host, port=port,
|
|
|
|
|
content_type='text/plain')
|
|
|
|
|
else:
|
|
|
|
|
else:
|
|
|
|
|
LOG.debug(f"Downloading from {domain} for {fp}")
|
|
|
|
|
o = oDownloadUrl(url, https_cafile,
|
|
|
|
|
timeout=timeout, host=host, port=port,
|
|
|
|
|
@ -478,7 +445,7 @@ def oVerifyUrl(url, domain, fp=None, https_cafile=None, timeout=20, host='127.0.
|
|
|
|
|
tBAD_URLS.add(url)
|
|
|
|
|
except TrustorError as e:
|
|
|
|
|
if e.args == "HTTP Errorcode 404":
|
|
|
|
|
aFP_EMAIL[fp] = a['email']
|
|
|
|
|
#? aFP_EMAIL[fp] = a['email']
|
|
|
|
|
LOG.warn(f"BC TrustorError 404 from {domain} {e.args}")
|
|
|
|
|
else:
|
|
|
|
|
LOG.warn(f"BC TrustorError downloading from {domain} {e.args}")
|
|
|
|
|
@ -488,7 +455,7 @@ def oVerifyUrl(url, domain, fp=None, https_cafile=None, timeout=20, host='127.0.
|
|
|
|
|
# maybe offline - not bad
|
|
|
|
|
LOG.warn(f"BC MaxRetryError downloading from {domain} {e}")
|
|
|
|
|
except (BaseException) as e:
|
|
|
|
|
LOG.error(f"BC Exception {type(e)} downloading from {domain} {e}")
|
|
|
|
|
LOG.warn(f"BC Exception {type(e)} downloading from {domain} {e}")
|
|
|
|
|
else:
|
|
|
|
|
return o
|
|
|
|
|
return None
|
|
|
|
|
@ -499,13 +466,13 @@ def oVerifyUrl(url, domain, fp=None, https_cafile=None, timeout=20, host='127.0.
|
|
|
|
|
# If we paralelize the gathering of the URLs, we may have simultaneous
|
|
|
|
|
# gathers of the same URL from different relays, defeating the advantage
|
|
|
|
|
# of going parallel. The cache is global aDOMAIN_FPS.
|
|
|
|
|
def aVerifyContact(a=None, fp=None, https_cafile=None, timeout=20, host='127.0.0.1', port=9050, oargs=None):
|
|
|
|
|
def aVerifyContact(a, fp, https_cafile=None, timeout=20, host='127.0.0.1', port=9050, oargs=None):
|
|
|
|
|
global aFP_EMAIL
|
|
|
|
|
global tBAD_URLS
|
|
|
|
|
global lKNOWN_NODNS
|
|
|
|
|
global aDOMAIN_FPS
|
|
|
|
|
global aBAD_CONTACTS_DB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
assert a
|
|
|
|
|
assert fp
|
|
|
|
|
assert https_cafile
|
|
|
|
|
@ -515,10 +482,11 @@ def aVerifyContact(a=None, fp=None, https_cafile=None, timeout=20, host='127.0.0
|
|
|
|
|
if domain in aDOMAIN_FPS.keys():
|
|
|
|
|
a['fps'] = aDOMAIN_FPS[domain]
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
r = bVerifyContact(a=a, fp=fp, https_cafile=https_cafile)
|
|
|
|
|
|
|
|
|
|
r = bVerifyContact(lAT_REPS, lDOT_REPS, lNO_EMAIL, a, fp, https_cafile=https_cafile)
|
|
|
|
|
if r is not True:
|
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
if a['url'] in tBAD_URLS:
|
|
|
|
|
a['fps'] = []
|
|
|
|
|
return a
|
|
|
|
|
@ -549,10 +517,10 @@ def aVerifyContact(a=None, fp=None, https_cafile=None, timeout=20, host='127.0.0
|
|
|
|
|
elif a['proof'] == 'dns-rsa':
|
|
|
|
|
# well let the test of the URL be enough for now
|
|
|
|
|
LOG.debug(f"Downloaded from {url} ")
|
|
|
|
|
a['fps'] = [fp]
|
|
|
|
|
a['fps'] = [fp]
|
|
|
|
|
aDOMAIN_FPS[domain] = a['fps']
|
|
|
|
|
elif a['proof'] == 'uri-rsa':
|
|
|
|
|
a = aContactFps(oargs, a, o, domain)
|
|
|
|
|
a = aContactFps(oargs, a, fp, o, domain)
|
|
|
|
|
if a['fps']:
|
|
|
|
|
LOG.debug(f"Downloaded from {url} {len(a['fps'])} FPs for {fp}")
|
|
|
|
|
else:
|
|
|
|
|
@ -561,7 +529,7 @@ def aVerifyContact(a=None, fp=None, https_cafile=None, timeout=20, host='127.0.0
|
|
|
|
|
aDOMAIN_FPS[domain] = a['fps']
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def aContactFps(oargs, a, o, domain):
|
|
|
|
|
def aContactFps(oargs, a, fp, o, domain):
|
|
|
|
|
global aFP_EMAIL
|
|
|
|
|
global tBAD_URLS
|
|
|
|
|
global aDOMAIN_FPS
|
|
|
|
|
@ -594,7 +562,7 @@ def aContactFps(oargs, a, o, domain):
|
|
|
|
|
oFd.write(data)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"Error writing {sfile} {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
a['modified'] = int(time.time())
|
|
|
|
|
if not l:
|
|
|
|
|
LOG.warn(f"Downloaded from {domain} empty for {fp}")
|
|
|
|
|
@ -605,187 +573,6 @@ def aContactFps(oargs, a, o, domain):
|
|
|
|
|
LOG.info(f"Downloaded from {domain} {len(a['fps'])} FPs")
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def aParseContact(contact, fp):
|
|
|
|
|
"""
|
|
|
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
|
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
|
|
|
"""
|
|
|
|
|
a = {}
|
|
|
|
|
if not contact:
|
|
|
|
|
LOG.warn(f"BC null contact for {fp}")
|
|
|
|
|
LOG.debug(f"{fp} {contact}")
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
contact = contact.split(r'\n')[0]
|
|
|
|
|
for elt in lMORONS:
|
|
|
|
|
contact = contact.replace(elt, '')
|
|
|
|
|
m = oCONTACT_RE.match(contact)
|
|
|
|
|
# 450 matches!
|
|
|
|
|
if m and m.groups and len(m.groups(0)) > 2 and m.span()[1] > 0:
|
|
|
|
|
i = len(m.groups(0)[0]) + len(m.groups(0)[1])
|
|
|
|
|
contact = contact[i:]
|
|
|
|
|
|
|
|
|
|
# shlex?
|
|
|
|
|
lelts = contact.split(' ')
|
|
|
|
|
if not lelts:
|
|
|
|
|
LOG.warn(f"BC empty contact for {fp}")
|
|
|
|
|
LOG.debug(f"{fp} {contact}")
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
for elt in lelts:
|
|
|
|
|
if ':' not in elt:
|
|
|
|
|
# hoster:Quintex Alliance Consulting
|
|
|
|
|
LOG.warn(f"BC no : in {elt} for {contact} in {fp}")
|
|
|
|
|
# return {}
|
|
|
|
|
# try going with what we have
|
|
|
|
|
break
|
|
|
|
|
(key , val,) = elt.split(':', 1)
|
|
|
|
|
if key == '':
|
|
|
|
|
continue
|
|
|
|
|
key = key.rstrip(':')
|
|
|
|
|
a[key] = val
|
|
|
|
|
a = aCleanContact(a)
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def aParseContactYaml(contact, fp):
|
|
|
|
|
"""
|
|
|
|
|
See the Tor ContactInfo Information Sharing Specification v2
|
|
|
|
|
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
|
|
|
|
|
"""
|
|
|
|
|
l = [line for line in contact.strip().replace('"', '').split(' ')
|
|
|
|
|
if ':' in line]
|
|
|
|
|
LOG.debug(f"{fp} {len(l)} fields")
|
|
|
|
|
s = f'"{fp}":\n'
|
|
|
|
|
s += '\n'.join([f" {line}\"".replace(':', ': \"', 1)
|
|
|
|
|
for line in l])
|
|
|
|
|
oFd = StringIO(s)
|
|
|
|
|
a = safe_load(oFd)
|
|
|
|
|
return a
|
|
|
|
|
|
|
|
|
|
def oMainArgparser(_=None):
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
from OpenSSL import SSL
|
|
|
|
|
lCAfs = SSL._CERTIFICATE_FILE_LOCATIONS
|
|
|
|
|
except:
|
|
|
|
|
lCAfs = []
|
|
|
|
|
|
|
|
|
|
CAfs = []
|
|
|
|
|
for elt in lCAfs:
|
|
|
|
|
if os.path.exists(elt):
|
|
|
|
|
CAfs.append(elt)
|
|
|
|
|
if not CAfs:
|
|
|
|
|
CAfs = ['']
|
|
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(add_help=True,
|
|
|
|
|
epilog=__prolog__)
|
|
|
|
|
parser.add_argument('--https_cafile', type=str,
|
|
|
|
|
help="Certificate Authority file (in PEM)",
|
|
|
|
|
default=CAfs[0])
|
|
|
|
|
parser.add_argument('--proxy_host', '--proxy-host', type=str,
|
|
|
|
|
default='127.0.0.1',
|
|
|
|
|
help='proxy host')
|
|
|
|
|
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
|
|
|
|
|
help='proxy socks port')
|
|
|
|
|
parser.add_argument('--proxy_ctl', '--proxy-ctl',
|
|
|
|
|
default='/run/tor/control' if os.path.exists('/run/tor/control') else '9051',
|
|
|
|
|
type=str,
|
|
|
|
|
help='control socket - or port')
|
|
|
|
|
|
|
|
|
|
parser.add_argument('--torrc',
|
|
|
|
|
default='/etc/tor/torrc-defaults',
|
|
|
|
|
type=str,
|
|
|
|
|
help='torrc to check for suggestions')
|
|
|
|
|
parser.add_argument('--timeout', default=60, type=int,
|
|
|
|
|
help='proxy download connect timeout')
|
|
|
|
|
|
|
|
|
|
parser.add_argument('--good_nodes', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
|
|
|
|
|
help="Yaml file of good info that should not be excluded")
|
|
|
|
|
parser.add_argument('--bad_nodes', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'badnodes.yaml'),
|
|
|
|
|
help="Yaml file of bad nodes that should also be excluded")
|
|
|
|
|
parser.add_argument('--bad_on', type=str, default='Empty,NoEmail,NotGood',
|
|
|
|
|
help="comma sep list of conditions - Empty,NoEmail,NotGood")
|
|
|
|
|
parser.add_argument('--bad_contacts', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
|
|
|
|
|
help="Yaml file of bad contacts that bad FPs are using")
|
|
|
|
|
parser.add_argument('--saved_only', default=False,
|
|
|
|
|
action='store_true',
|
|
|
|
|
help="Just use the info in the last *.yaml files without querying the Tor controller")
|
|
|
|
|
parser.add_argument('--strict_nodes', type=str, default=0,
|
|
|
|
|
choices=['0', '1'],
|
|
|
|
|
help="Set StrictNodes: 1 is less anonymous but more secure, although some onion sites may be unreachable")
|
|
|
|
|
parser.add_argument('--wait_boot', type=int, default=120,
|
|
|
|
|
help="Seconds to wait for Tor to booststrap")
|
|
|
|
|
parser.add_argument('--points_timeout', type=int, default=0,
|
|
|
|
|
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
|
|
|
|
|
parser.add_argument('--log_level', type=int, default=20,
|
|
|
|
|
help="10=debug 20=info 30=warn 40=error")
|
|
|
|
|
parser.add_argument('--bad_sections', type=str,
|
|
|
|
|
default='BadExit',
|
|
|
|
|
help="sections of the badnodes.yaml to use, in addition to BadExit, comma separated")
|
|
|
|
|
parser.add_argument('--white_onions', type=str,
|
|
|
|
|
default='',
|
|
|
|
|
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
|
|
|
|
|
parser.add_argument('--torrc_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'torrc.new'),
|
|
|
|
|
help="Write the torrc configuration to a file")
|
|
|
|
|
parser.add_argument('--hs_dir', type=str,
|
|
|
|
|
default='/var/lib/tor',
|
|
|
|
|
help="Parse the files name hostname below this dir to find Hidden Services to whitelist")
|
|
|
|
|
parser.add_argument('--notice_log', type=str,
|
|
|
|
|
default='',
|
|
|
|
|
help="Parse the notice log for relays and services")
|
|
|
|
|
parser.add_argument('--relays_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'relays.json'),
|
|
|
|
|
help="Write the download relays in json to a file")
|
|
|
|
|
parser.add_argument('--wellknown_output', type=str,
|
|
|
|
|
default=os.path.join(ETC_DIR, 'https'),
|
|
|
|
|
help="Write the well-known files to a directory")
|
|
|
|
|
parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
|
|
|
|
|
help="Write the proof data of the included nodes to a YAML file")
|
|
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
def vwrite_good_contacts(oargs):
|
|
|
|
|
global aGOOD_CONTACTS_DB
|
|
|
|
|
good_contacts_tmp = oargs.good_contacts + '.tmp'
|
|
|
|
|
with open(good_contacts_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aGOOD_CONTACTS_DB, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oargs.good_contacts):
|
|
|
|
|
bak = oargs.good_contacts +'.bak'
|
|
|
|
|
os.rename(oargs.good_contacts, bak)
|
|
|
|
|
os.rename(good_contacts_tmp, oargs.good_contacts)
|
|
|
|
|
LOG.info(f"Wrote {len(list(aGOOD_CONTACTS_DB.keys()))} good contact details to {oargs.good_contacts}")
|
|
|
|
|
bad_contacts_tmp = good_contacts_tmp.replace('.tmp', '.bad')
|
|
|
|
|
with open(bad_contacts_tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBAD_CONTACTS_DB, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
|
|
|
|
|
def vwrite_badnodes(oargs, aBAD_NODES, slen, stag):
|
|
|
|
|
if not aBAD_NODES: return
|
|
|
|
|
tmp = oargs.bad_nodes +'.tmp'
|
|
|
|
|
bak = oargs.bad_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBAD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {slen} to {stag} in {oargs.bad_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oargs.bad_nodes):
|
|
|
|
|
os.rename(oargs.bad_nodes, bak)
|
|
|
|
|
os.rename(tmp, oargs.bad_nodes)
|
|
|
|
|
|
|
|
|
|
def vwrite_goodnodes(oargs, aGOOD_NODES, ilen):
|
|
|
|
|
tmp = oargs.good_nodes +'.tmp'
|
|
|
|
|
bak = oargs.good_nodes +'.bak'
|
|
|
|
|
with open(tmp, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aGOOD_NODES, oFYaml)
|
|
|
|
|
LOG.info(f"Wrote {ilen} good relays to {oargs.good_nodes}")
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
if os.path.exists(oargs.good_nodes):
|
|
|
|
|
os.rename(oargs.good_nodes, bak)
|
|
|
|
|
os.rename(tmp, oargs.good_nodes)
|
|
|
|
|
|
|
|
|
|
def lget_onionoo_relays(oargs):
|
|
|
|
|
import requests
|
|
|
|
|
adata = {}
|
|
|
|
|
@ -847,66 +634,6 @@ def lget_onionoo_relays(oargs):
|
|
|
|
|
lonionoo_relays = [r for r in adata["relays"] if 'fingerprint' in r.keys()]
|
|
|
|
|
return lonionoo_relays
|
|
|
|
|
|
|
|
|
|
def vsetup_logging(log_level, logfile='', stream=sys.stdout):
|
|
|
|
|
global LOG
|
|
|
|
|
add = True
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
|
|
|
|
|
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
|
|
|
|
# https://pypi.org/project/coloredlogs/
|
|
|
|
|
import coloredlogs
|
|
|
|
|
except ImportError:
|
|
|
|
|
coloredlogs = False
|
|
|
|
|
|
|
|
|
|
# stem fucks up logging
|
|
|
|
|
# from stem.util import log
|
|
|
|
|
logging.getLogger('stem').setLevel(30)
|
|
|
|
|
|
|
|
|
|
logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S')
|
|
|
|
|
logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S'
|
|
|
|
|
logging._defaultFormatter.default_msec_format = ''
|
|
|
|
|
|
|
|
|
|
kwargs = dict(level=log_level,
|
|
|
|
|
force=True,
|
|
|
|
|
format='%(levelname)s %(message)s')
|
|
|
|
|
|
|
|
|
|
if logfile:
|
|
|
|
|
add = logfile.startswith('+')
|
|
|
|
|
sub = logfile.startswith('-')
|
|
|
|
|
if add or sub:
|
|
|
|
|
logfile = logfile[1:]
|
|
|
|
|
kwargs['filename'] = logfile
|
|
|
|
|
|
|
|
|
|
if coloredlogs:
|
|
|
|
|
# https://pypi.org/project/coloredlogs/
|
|
|
|
|
aKw = dict(level=log_level,
|
|
|
|
|
logger=LOG,
|
|
|
|
|
stream=stream,
|
|
|
|
|
fmt='%(levelname)s %(message)s'
|
|
|
|
|
)
|
|
|
|
|
coloredlogs.install(**aKw)
|
|
|
|
|
if logfile:
|
|
|
|
|
oHandler = logging.FileHandler(logfile)
|
|
|
|
|
LOG.addHandler(oHandler)
|
|
|
|
|
LOG.info(f"CSetting log_level to {log_level} {stream}")
|
|
|
|
|
else:
|
|
|
|
|
logging.basicConfig(**kwargs)
|
|
|
|
|
if add and logfile:
|
|
|
|
|
oHandler = logging.StreamHandler(stream)
|
|
|
|
|
LOG.addHandler(oHandler)
|
|
|
|
|
LOG.info(f"SSetting log_level to {log_level!s}")
|
|
|
|
|
|
|
|
|
|
def vwritefinale(oargs):
|
|
|
|
|
global lNOT_IN_RELAYS_DB
|
|
|
|
|
|
|
|
|
|
if len(lNOT_IN_RELAYS_DB):
|
|
|
|
|
LOG.warn(f"{len(lNOT_IN_RELAYS_DB)} relays from stem were not in onionoo.torproject.org")
|
|
|
|
|
|
|
|
|
|
LOG.info(f"For info on a FP, use: https://nusenu.github.io/OrNetStats/w/relay/<FP>.html")
|
|
|
|
|
LOG.info(f"For info on relays, try: https://onionoo.torproject.org/details")
|
|
|
|
|
# https://onionoo.torproject.org/details
|
|
|
|
|
|
|
|
|
|
def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0):
|
|
|
|
|
global aGOOD_CONTACTS_DB
|
|
|
|
|
global aGOOD_CONTACTS_FPS
|
|
|
|
|
@ -936,16 +663,16 @@ def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0):
|
|
|
|
|
aGOOD_CONTACTS_FPS[elt] = b
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def bCheckFp(relay, sofar, lConds, texclude_set):
|
|
|
|
|
global aGOOD_CONTACTS_DB
|
|
|
|
|
global aGOOD_CONTACTS_FPS
|
|
|
|
|
global lNOT_IN_RELAYS_DB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if not is_valid_fingerprint(relay.fingerprint):
|
|
|
|
|
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fp = relay.fingerprint
|
|
|
|
|
if aRELAYS_DB and fp not in aRELAYS_DB.keys():
|
|
|
|
|
LOG.warn(f"{fp} not in aRELAYS_DB")
|
|
|
|
|
@ -969,44 +696,45 @@ def bCheckFp(relay, sofar, lConds, texclude_set):
|
|
|
|
|
|
|
|
|
|
# fail if the contact is empty
|
|
|
|
|
if ('Empty' in lConds and not relay.contact):
|
|
|
|
|
LOG.info(f"{fp} skipping empty contact - Empty {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} skipping empty contact - Empty {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
contact = sCleanEmail(relay.contact)
|
|
|
|
|
contact = sCleanEmail(relay.contact, lAT_REPS, lDOT_REPS, lNO_EMAIL)
|
|
|
|
|
# fail if the contact has no email - unreliable
|
|
|
|
|
if 'NoEmail' in lConds and relay.contact and \
|
|
|
|
|
('@' not in contact):
|
|
|
|
|
LOG.info(f"{fp} skipping contact - NoEmail {contact} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} skipping contact - NoEmail {contact} {sofar}")
|
|
|
|
|
# LOG.spam(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
# fail if the contact does not pass
|
|
|
|
|
if ('NotGood' in lConds and relay.contact and
|
|
|
|
|
('ciissversion:' not in relay.contact)):
|
|
|
|
|
LOG.info(f"{fp} skipping no ciissversion in contact {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} skipping no ciissversion in contact {sofar}")
|
|
|
|
|
# LOG.spam(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
# fail if the contact does not have url: to pass
|
|
|
|
|
if relay.contact and 'url' not in relay.contact:
|
|
|
|
|
LOG.info(f"{fp} skipping unfetchable contact - no url {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} skipping unfetchable contact - no url {sofar}")
|
|
|
|
|
# LOG.spam(f"{fp} {relay.contact} {sofar}")
|
|
|
|
|
if ('NotGood' in lConds): texclude_set.add(fp)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def oMainPreamble(lArgs):
|
|
|
|
|
global LOG
|
|
|
|
|
global aGOOD_CONTACTS_DB
|
|
|
|
|
global aGOOD_CONTACTS_FPS
|
|
|
|
|
|
|
|
|
|
parser = oMainArgparser()
|
|
|
|
|
|
|
|
|
|
parser = oMainArgparser( __prolog__= __prolog__)
|
|
|
|
|
oargs = parser.parse_args(lArgs)
|
|
|
|
|
|
|
|
|
|
vsetup_logging(oargs.log_level)
|
|
|
|
|
vsetup_logging(LOG, oargs.log_level, stream=sys.stdout)
|
|
|
|
|
if bAreWeConnected() is False:
|
|
|
|
|
raise SystemExit("we are not connected")
|
|
|
|
|
|
|
|
|
|
@ -1038,31 +766,9 @@ def oMainPreamble(lArgs):
|
|
|
|
|
|
|
|
|
|
return oargs
|
|
|
|
|
|
|
|
|
|
def oStemController(oargs):
|
|
|
|
|
if os.path.exists(oargs.proxy_ctl):
|
|
|
|
|
controller = oGetStemController(log_level=oargs.log_level, sock_or_pair=oargs.proxy_ctl)
|
|
|
|
|
else:
|
|
|
|
|
port =int(oargs.proxy_ctl)
|
|
|
|
|
controller = oGetStemController(log_level=oargs.log_level, sock_or_pair=port)
|
|
|
|
|
|
|
|
|
|
vwait_for_controller(controller, oargs.wait_boot)
|
|
|
|
|
|
|
|
|
|
elt = controller.get_conf('UseMicrodescriptors')
|
|
|
|
|
if elt != '0':
|
|
|
|
|
LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
|
|
|
|
|
controller.set_conf('UseMicrodescriptors', 0)
|
|
|
|
|
# does it work dynamically?
|
|
|
|
|
return 2
|
|
|
|
|
|
|
|
|
|
elt = controller.get_conf(sEXCLUDE_EXIT_GROUP)
|
|
|
|
|
if elt and elt != '{??}':
|
|
|
|
|
LOG.warn(f"{sEXCLUDE_EXIT_GROUP} is in use already")
|
|
|
|
|
|
|
|
|
|
return controller
|
|
|
|
|
|
|
|
|
|
def tWhitelistSet(oargs, controller):
|
|
|
|
|
twhitelist_set = set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
twhitelist_set.update(set(lYamlGoodNodes(oargs.good_nodes)))
|
|
|
|
|
LOG.info(f"lYamlGoodNodes {len(twhitelist_set)} EntryNodes from {oargs.good_nodes}")
|
|
|
|
|
|
|
|
|
|
@ -1077,7 +783,7 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
with open(os.path.join(dirpath, f), 'rt') as oFd:
|
|
|
|
|
son = oFd.read()
|
|
|
|
|
t.update(son)
|
|
|
|
|
LOG.info(f"Added {son} to the list for Introduction Points")
|
|
|
|
|
LOG.debug(f"Added {son} to the list for Introduction Points")
|
|
|
|
|
|
|
|
|
|
if oargs.notice_log and os.path.exists(oargs.notice_log):
|
|
|
|
|
tmp = tempfile.mktemp()
|
|
|
|
|
@ -1086,7 +792,7 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
with open(tmp, 'rt') as oFd:
|
|
|
|
|
tnew = {elt.strip() for elt in oFd.readlines()}
|
|
|
|
|
t.update(tnew)
|
|
|
|
|
LOG.info(f"Whitelist {len(lnew)} services from {oargs.notice_log}")
|
|
|
|
|
LOG.info(f"Whitelist {len(tnew)} services to {oargs.notice_log}")
|
|
|
|
|
os.remove(tmp)
|
|
|
|
|
|
|
|
|
|
w = set()
|
|
|
|
|
@ -1105,7 +811,7 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
LOG.info(f"Whitelist {len(lnew)} relays from {oargs.notice_log}")
|
|
|
|
|
os.remove(tmp)
|
|
|
|
|
twhitelist_set.update(w)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
w = set()
|
|
|
|
|
if 'Onions' in aGOOD_NODES[sGOOD_ROOT].keys():
|
|
|
|
|
# Provides the descriptor for a hidden service. The **address** is the
|
|
|
|
|
@ -1115,27 +821,16 @@ def tWhitelistSet(oargs, controller):
|
|
|
|
|
w.update(oargs.white_onions.split(','))
|
|
|
|
|
if oargs.points_timeout > 0:
|
|
|
|
|
LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
|
|
|
|
|
t.update(lIntroductionPoints(controller, w, itimeout=oargs.points_timeout))
|
|
|
|
|
t.update(lIntroductionPoints(controller, w, itimeout=oargs.points_timeout,
|
|
|
|
|
password=oargs.torctl_pass))
|
|
|
|
|
if len(t) > 0:
|
|
|
|
|
LOG.info(f"IntroductionPoints {len(t)} relays from {len(w)} IPs for onions")
|
|
|
|
|
twhitelist_set.update(t)
|
|
|
|
|
|
|
|
|
|
return twhitelist_set
|
|
|
|
|
|
|
|
|
|
def tExcludeSet(oargs):
|
|
|
|
|
texclude_set = set()
|
|
|
|
|
sections = {'BadExit'}
|
|
|
|
|
if oargs.bad_nodes and os.path.exists(oargs.bad_nodes):
|
|
|
|
|
if oargs.bad_sections:
|
|
|
|
|
sections.update(oargs.bad_sections.split(','))
|
|
|
|
|
texclude_set = set(lYamlBadNodes(oargs.bad_nodes,
|
|
|
|
|
tWanted=sections,
|
|
|
|
|
section=sEXCLUDE_EXIT_GROUP))
|
|
|
|
|
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
|
|
|
|
|
|
|
|
|
|
return texclude_set
|
|
|
|
|
|
|
|
|
|
# async
|
|
|
|
|
# async
|
|
|
|
|
def iMain(lArgs):
|
|
|
|
|
global aGOOD_CONTACTS_DB
|
|
|
|
|
global aGOOD_CONTACTS_FPS
|
|
|
|
|
@ -1147,12 +842,12 @@ def iMain(lArgs):
|
|
|
|
|
global aRELAYS_DB_INDEX
|
|
|
|
|
global tBAD_URLS
|
|
|
|
|
global lNOT_IN_RELAYS_DB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
oargs = oMainPreamble(lArgs)
|
|
|
|
|
controller = oStemController(oargs)
|
|
|
|
|
controller = oStemController(oargs, sEXCLUDE_EXIT_GROUP)
|
|
|
|
|
twhitelist_set = tWhitelistSet(oargs, controller)
|
|
|
|
|
texclude_set = tExcludeSet(oargs)
|
|
|
|
|
|
|
|
|
|
texclude_set = tExcludeSet(oargs, sEXCLUDE_EXIT_GROUP)
|
|
|
|
|
|
|
|
|
|
ttrust_db_index = aGOOD_CONTACTS_FPS.keys()
|
|
|
|
|
iFakeContact = 0
|
|
|
|
|
iTotalContacts = 0
|
|
|
|
|
@ -1175,13 +870,13 @@ def iMain(lArgs):
|
|
|
|
|
if r is not True: continue
|
|
|
|
|
# if it has a ciissversion in contact we count it in total
|
|
|
|
|
iTotalContacts += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# only proceed if 'NotGood' not in lConds:
|
|
|
|
|
if 'NotGood' not in lConds:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# fail if the contact does not have url: to pass
|
|
|
|
|
a = aParseContact(relay.contact, fp)
|
|
|
|
|
a = aParseContact(relay.contact, fp, lAT_REPS, lDOT_REPS, lNO_EMAIL)
|
|
|
|
|
if not a:
|
|
|
|
|
LOG.warn(f"{fp} BC contact did not parse {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
@ -1191,8 +886,8 @@ def iMain(lArgs):
|
|
|
|
|
if 'url' in a and a['url']:
|
|
|
|
|
# fail if the contact uses a url we already know is bad
|
|
|
|
|
if a['url'] in tBAD_URLS:
|
|
|
|
|
LOG.info(f"{fp} skipping in tBAD_URLS {a['url']} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} {a} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} skipping in tBAD_URLS {a['url']} {sofar}")
|
|
|
|
|
# LOG.spam(f"{fp} {a} {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
@ -1200,28 +895,28 @@ def iMain(lArgs):
|
|
|
|
|
# fail if the contact uses a domain we already know does not resolve
|
|
|
|
|
if domain in lKNOWN_NODNS:
|
|
|
|
|
# The fp is using a contact with a URL we know is bogus
|
|
|
|
|
LOG.info(f"{fp} BC skipping in lKNOWN_NODNS {a} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} {relay} {sofar}")
|
|
|
|
|
LOG.debug(f"{fp} BC skipping in lKNOWN_NODNS {a} {sofar}")
|
|
|
|
|
# LOG.spam(f"{fp} {relay} {sofar}")
|
|
|
|
|
texclude_set.add(fp)
|
|
|
|
|
aBAD_CONTACTS_DB[fp] = a
|
|
|
|
|
continue
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if 'proof' in a and a['proof'] in ['uri-rsa', 'dns-rsa']:
|
|
|
|
|
if domain in aDOMAIN_FPS.keys(): continue
|
|
|
|
|
if httpx:
|
|
|
|
|
a['fp'] = fp
|
|
|
|
|
lqueue.append(asyncio.create_task(
|
|
|
|
|
aVerifyContact(a=a,
|
|
|
|
|
fp=fp,
|
|
|
|
|
aVerifyContact(a,
|
|
|
|
|
fp,
|
|
|
|
|
https_cafile=oargs.https_cafile,
|
|
|
|
|
timeout=oargs.timeout,
|
|
|
|
|
host=oargs.proxy_host,
|
|
|
|
|
port=oargs.proxy_port,
|
|
|
|
|
oargs=oargs)))
|
|
|
|
|
else:
|
|
|
|
|
b = aVerifyContact(a=a,
|
|
|
|
|
fp=fp,
|
|
|
|
|
b = aVerifyContact(a,
|
|
|
|
|
fp,
|
|
|
|
|
https_cafile=oargs.https_cafile,
|
|
|
|
|
timeout=oargs.timeout,
|
|
|
|
|
host=oargs.proxy_host,
|
|
|
|
|
@ -1230,7 +925,7 @@ def iMain(lArgs):
|
|
|
|
|
r = bProcessContact(b, texclude_set, aBadContacts, iFakeContact)
|
|
|
|
|
if r is False:
|
|
|
|
|
iFakeContact += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if httpx:
|
|
|
|
|
# for b in asyncio.as_completed(lqueue):
|
|
|
|
|
for b in lqueue:
|
|
|
|
|
@ -1242,40 +937,61 @@ def iMain(lArgs):
|
|
|
|
|
elif r is True:
|
|
|
|
|
# iGoodContact += 1
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
|
|
|
|
|
texclude_set = texclude_set.difference(twhitelist_set)
|
|
|
|
|
LOG.info(f"{len(list(aGOOD_CONTACTS_DB.keys()))} good contacts out of {iTotalContacts}")
|
|
|
|
|
|
|
|
|
|
texclude_set = texclude_set.difference(twhitelist_set)
|
|
|
|
|
louts = []
|
|
|
|
|
if oargs.torrc_output and texclude_set:
|
|
|
|
|
with open(oargs.torrc_output, 'wt') as oFTorrc:
|
|
|
|
|
oFTorrc.write(f"{sEXCLUDE_EXIT_GROUP} {','.join(texclude_set)}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aGOOD_CONTACTS_FPS.keys())}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
|
|
|
|
|
LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
|
|
|
|
|
oFTorrc.close()
|
|
|
|
|
try:
|
|
|
|
|
with open(oargs.torrc_output, 'wt') as oFTorrc:
|
|
|
|
|
oFTorrc.write(f"{sEXCLUDE_EXIT_GROUP} {','.join(texclude_set)}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aGOOD_CONTACTS_FPS.keys())}\n")
|
|
|
|
|
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(aGOOD_NODES[sGOOD_ROOT]['EntryNodes'])}\n")
|
|
|
|
|
LOG.info(f"Wrote tor configuration to {oargs.torrc_output}")
|
|
|
|
|
oFTorrc.close()
|
|
|
|
|
louts += [oargs.torrc_output]
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"ERROR writing {oargs.torrc_output} {e}")
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
if oargs.bad_contacts and aBadContacts:
|
|
|
|
|
# for later analysis
|
|
|
|
|
with open(oargs.bad_contacts, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBadContacts, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
try:
|
|
|
|
|
# for later analysis
|
|
|
|
|
with open(oargs.bad_contacts, 'wt') as oFYaml:
|
|
|
|
|
yaml.dump(aBadContacts, oFYaml)
|
|
|
|
|
oFYaml.close()
|
|
|
|
|
louts += [oargs.bad_contacts]
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"ERROR writing {oargs.bad_contacts} {e}")
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
if oargs.good_contacts != '' and aGOOD_CONTACTS_DB:
|
|
|
|
|
vwrite_good_contacts(oargs)
|
|
|
|
|
try:
|
|
|
|
|
vwrite_good_contacts(oargs, aGOOD_CONTACTS_DB)
|
|
|
|
|
louts += [oargs.good_contacts]
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"ERROR writing vwrite_good_contacts {e}")
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
aBAD_NODES[oBAD_ROOT][sEXCLUDE_EXIT_GROUP]['BadExit'] = list(texclude_set)
|
|
|
|
|
aBAD_NODES[oBAD_ROOT][sEXCLUDE_DOMAINS] = lKNOWN_NODNS
|
|
|
|
|
if oargs.bad_nodes:
|
|
|
|
|
stag = sEXCLUDE_EXIT_GROUP + '/BadExit'
|
|
|
|
|
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)), stag)
|
|
|
|
|
try:
|
|
|
|
|
vwrite_badnodes(oargs, aBAD_NODES, str(len(texclude_set)), stag)
|
|
|
|
|
louts += [oargs.bad_nodes]
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"ERROR writing vwrite_badnodes {e}")
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
aGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aGOOD_CONTACTS_FPS.keys())
|
|
|
|
|
# EntryNodes are readony
|
|
|
|
|
if oargs.good_nodes:
|
|
|
|
|
vwrite_goodnodes(oargs, aGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
|
|
|
|
|
|
|
|
|
|
vwritefinale(oargs)
|
|
|
|
|
try:
|
|
|
|
|
vwrite_goodnodes(oargs, aGOOD_NODES, len(aGOOD_CONTACTS_FPS.keys()))
|
|
|
|
|
louts += [oargs.good_nodes]
|
|
|
|
|
except Exception as e:
|
|
|
|
|
LOG.warn(f"ERROR writing vwrite_goodnodes {e}")
|
|
|
|
|
# drop through
|
|
|
|
|
|
|
|
|
|
retval = 0
|
|
|
|
|
try:
|
|
|
|
|
@ -1318,10 +1034,17 @@ def iMain(lArgs):
|
|
|
|
|
LOG.warn(f"controller failed StrictNodes NOT {oargs.strict_nodes}")
|
|
|
|
|
else:
|
|
|
|
|
LOG.info(f"controller OVERRODE StrictNodes to {oargs.strict_nodes}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
LOG.info(f"controller StrictNodes is set to {cur}")
|
|
|
|
|
|
|
|
|
|
# final
|
|
|
|
|
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
|
|
|
|
|
LOG.info(f"{len(list(aGOOD_CONTACTS_DB.keys()))} good contacts out of {iTotalContacts}")
|
|
|
|
|
vwritefinale(oargs, lNOT_IN_RELAYS_DB)
|
|
|
|
|
elts='\n' + '\n'.join(louts)
|
|
|
|
|
LOG.info(f"The following files were written:{elts}")
|
|
|
|
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
|
return 0
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
emdee@spm.plastiras.org
4 months ago