# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- """ Reads a tox profile and prints out information on what's in there to stderr. Call it with one argument, the filename of the profile for the decrypt or info commands, or the filename of the nodes file for the nodes command. 4 commands are supported: --command info - default prints info about what's in the Tox profile to stderr --command nodes assumes you are reading a json nodes file instead of a profile --command decrypt decrypts the profile and writes to the result to stdout --command edits edits fields in a Tox profile with --output to a file """ """ --output Destination for info/decrypt/edit/nodes --info default='info', choices=[info, save, repr, yaml,json, pprint] with --info=info prints info about the profile to stderr yaml,json, pprint, repr - output format nmap_dht - test DHT nodes with nmap nmap_relay - test TCP_RELAY nodes with nmap nmap_path - test PATH_NODE nodes with nmap --indent for pprint/yaml/json default=2 --nodes choices=[select_tcp, select_udp, nmap_tcp, select_version, nmap_udp, check, download] select_udp - select udp nodes select_tcp - select tcp nodes nmap_udp - test UDP nodes with nmap nmap_tcp - test TCP nodes with nmap select_version - select nodes that are the latest version download - download nodes from --download_nodes_url check - check nodes from --download_nodes_url clean - check nodes and save them as --output --download_nodes_url https://nodes.tox.chat/json --edit help - print a summary of what fields can be edited section,num,key,val - edit the field section,num,key with val """ # originally from: # https://stackoverflow.com/questions/30901873/what-format-are-tox-files-stored-in import sys import os import time import struct from socket import inet_ntop, AF_INET6, AF_INET import logging import argparse from pprint import pprint import shutil import json import warnings warnings.filterwarnings('ignore') try: # https://pypi.org/project/msgpack/ import msgpack except ImportError as e: msgpack = None try: import yaml except ImportError as e: yaml = None try: # https://pypi.org/project/coloredlogs/ import coloredlogs if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ: os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' except ImportError as e: coloredlogs = False try: # https://git.plastiras.org/emdee/toxygen_wrapper from wrapper.toxencryptsave import ToxEncryptSave from wrapper_tests.support_http import download_url, bAreWeConnected from wrapper_tests.support_testing import sTorResolve except ImportError as e: print(f"Import Warning {e}") print("Download toxygen_wrapper to deal with encrypted tox files, from:") print("https://git.plastiras.org/emdee/toxygen_wrapper") print("Just put the parent of the wrapper directory on your PYTHONPATH") print("You also need to link your libtoxcore.so and libtoxav.so") print("and libtoxencryptsave.so into wrapper/../libs/") print("Link all 3 from libtoxcore.so if you have only libtoxcore.so") ToxEncryptSave = None download_url = None bAreWeConnected = None sTorResolve = None LOG = logging.getLogger('TSF') # Fix for Windows sDIR = os.environ.get('TMPDIR', '/tmp') sTOX_VERSION = "1000002018" sVER_MIN = "1000002013" # 3 months iOLD_SECS = 60*60*24*30*3 bHAVE_NMAP = shutil.which('nmap') bHAVE_TOR = shutil.which('tor') bHAVE_JQ = shutil.which('jq') bHAVE_BASH = shutil.which('bash') bMARK = b'\x00\x00\x00\x00\x1f\x1b\xed\x15' bDEBUG = 'DEBUG' in os.environ and os.environ['DEBUG'] != 0 def trace(s): LOG.log(LOG.level, '+ ' +s) LOG.trace = trace global bOUT, aOUT, sENC aOUT = {} bOUT = b'' sENC = sys.getdefaultencoding() # 'utf-8' lNULLS = ['', '[]', 'null'] # grep '#''#' logging_tox_savefile.py|sed -e 's/.* //' sEDIT_HELP = """ NAME,.,Nick_name,str STATUSMESSAGE,.,Status_message,str STATUS,.,Online_status,int NOSPAMKEYS,.,Nospam,hexstr NOSPAMKEYS,.,Public_key,hexstr NOSPAMKEYS,.,Private_key,hexstr DHT,.,DHTnode, TCP_RELAY,.,TCPnode, PATH_NODE,.,PATHnode, """ #messenger.c MESSENGER_STATE_TYPE_NOSPAMKEYS = 1 MESSENGER_STATE_TYPE_DHT = 2 MESSENGER_STATE_TYPE_FRIENDS = 3 MESSENGER_STATE_TYPE_NAME = 4 MESSENGER_STATE_TYPE_STATUSMESSAGE = 5 MESSENGER_STATE_TYPE_STATUS = 6 MESSENGER_STATE_TYPE_GROUPS = 7 MESSENGER_STATE_TYPE_TCP_RELAY = 10 MESSENGER_STATE_TYPE_PATH_NODE = 11 MESSENGER_STATE_TYPE_CONFERENCES = 20 MESSENGER_STATE_TYPE_END = 255 dSTATE_TYPE = { MESSENGER_STATE_TYPE_NOSPAMKEYS: "NOSPAMKEYS", MESSENGER_STATE_TYPE_DHT: "DHT", MESSENGER_STATE_TYPE_FRIENDS: "FRIENDS", MESSENGER_STATE_TYPE_NAME: "NAME", MESSENGER_STATE_TYPE_STATUSMESSAGE: "STATUSMESSAGE", MESSENGER_STATE_TYPE_STATUS: "STATUS", MESSENGER_STATE_TYPE_GROUPS: "GROUPS", MESSENGER_STATE_TYPE_TCP_RELAY: "TCP_RELAY", MESSENGER_STATE_TYPE_PATH_NODE: "PATH_NODE", MESSENGER_STATE_TYPE_CONFERENCES: "CONFERENCES", MESSENGER_STATE_TYPE_END: "END", } def decrypt_data(data): from getpass import getpass if not ToxEncryptSave: return data oToxES = ToxEncryptSave() if not oToxES.is_data_encrypted(data): LOG.debug('Not encrypted') return data assert data[:8] == b'toxEsave', data[:8] sys.stdout.flush() password = getpass('Password: ') assert password newData = oToxES.pass_decrypt(data, password) LOG.debug('Decrypted: ' +str(len(newData)) +' bytes') return newData def str_to_hex(raw_id, length=None): if length is None: length = len(raw_id) res = ''.join('{:02x}'.format(ord(raw_id[i])) for i in range(length)) return res.upper() def bin_to_hex(raw_id, length=None): if length is None: length = len(raw_id) res = ''.join('{:02x}'.format(raw_id[i]) for i in range(length)) return res.upper() def lProcessFriends(state, index, length, result): """Friend: The integers in this structure are stored in Big Endian format. Length Contents 1 uint8_t Status 32 Long term public key 1024 Friend request message as a byte string 1 PADDING 2 uint16_t Size of the friend request message 128 Name as a byte string 2 uint16_t Size of the name 1007 Status message as a byte string 1 PADDING 2 uint16_t Size of the status message 1 uint8_t User status (see also: USERSTATUS) 3 PADDING 4 uint32_t Nospam (only used for sending a friend request) 8 uint64_t Last seen time """ dStatus = { # Status Meaning 0: 'Not a friend', 1: 'Friend added', 2: 'Friend request sent', 3: 'Confirmed friend', 4: 'Friend online' } slen = 1+32+1024+1+2+128+2+1007+1+2+1+3+4+8 # 2216 assert length % slen == 0 lIN = [] for i in range(length // slen): delta = i*slen status = struct.unpack_from(">b", result, delta)[0] o = delta+1; l = 32 pk = bin_to_hex(result[o:o+l], l) o = delta+1+32+1024+1+2+128; l = 2 nsize = struct.unpack_from(">H", result, o)[0] o = delta+1+32+1024+1+2; l = 128 name = str(result[o:o+nsize], sENC) o = delta+1+32+1024+1+2+128+2+1007; l = 2 msize = struct.unpack_from(">H", result, o)[0] o = delta+1+32+1024+1+2+128+2; l = 1007 mame = str(result[o:o+msize], sENC) LOG.info(f"Friend #{i} {dStatus[status]} {name} {pk}") lIN += [{"Status": dStatus[status], "Name": name, "Pk": pk}] return lIN def lProcessGroups(state, index, length, result, label="GROUPS"): """ No GROUPS description in spec.html """ global sENC lIN = [] if not msgpack: LOG.warn(f"process_chunk Groups = NO msgpack bytes={length}") return [] try: groups = msgpack.loads(result, raw=True) LOG.info(f"{label} {len(groups)} groups") i = 0 for group in groups: assert len(group) == 7, group state_values, \ state_bin, \ topic_info, \ mod_list, \ keys, \ self_info, \ saved_peers, = group if state_values is None: LOG.warn(f"lProcessGroups #{i} state_values is None") else: assert len(state_values) == 8, state_values manually_disconnected, \ group_name_len, \ privacy_state, \ maxpeers, \ password_length, \ version, \ topic_lock, \ voice_state = state_values LOG.info(f"lProcessGroups #{i} version={version}") dBINS = {"Version": version, "Privacy_state": privacy_state} lIN += [{"State_values": dBINS}] if state_bin is None: LOG.warn(f"lProcessGroups #{i} state_bin is None") else: assert len(state_bin) == 5, state_bin shared_state_sig, \ founder_public_key, \ group_name_len, \ password_length, \ mod_list_hash = state_bin LOG.info(f"lProcessGroups #{i} founder_public_key={bin_to_hex(founder_public_key)}") dBINS = {"Founder_public_key": bin_to_hex(founder_public_key)} lIN += [{"State_bin": dBINS}] if topic_info is None: LOG.warn(f"lProcessGroups #{i} topic_info is None") else: assert len(topic_info) == 6, topic_info version, \ length, \ checksum, \ topic, \ public_sig_key, \ topic_sig = topic_info topic_info_topic = str(topic, sENC) LOG.info(f"lProcessGroups #{i} topic_info_topic={topic_info_topic}") dBINS = {"Topic_info_topic": topic_info_topic } lIN += [{"Topic_info": dBINS}] if mod_list is None: LOG.warn(f"lProcessGroups #{i} mod_list is None") else: assert len(mod_list) == 2, mod_list num_moderators = mod_list[0] LOG.info(f"lProcessGroups #{i} num moderators={mod_list[0]}") #define CRYPTO_SIGN_PUBLIC_KEY_SIZE 32 lMODS = [] if not num_moderators: LOG.warn(f"lProcessGroups #{i} num_moderators is 0") else: mods = mod_list[1] assert len(mods) % 32 == 0, len(mods) assert len(mods) == num_moderators * 32, len(mods) for j in range(num_moderators): mod = mods[j*32:j*32 + 32] LOG.info(f"lProcessGroups group#{i} mod#{j} sig_pk={bin_to_hex(mod)}") lMODS += [{"Sig_pk": bin_to_hex(mod)}] lIN += [{"Moderators": lMODS}] if keys is None: LOG.warn(f"lProcessGroups #{i} keys is None") else: assert len(keys) == 4, keys LOG.debug(f"lProcessGroups #{i} {repr(list(map(len, keys)))}") chat_public_key, \ chat_secret_key, \ self_public_key, \ self_secret_key = keys LOG.info(f"lProcessGroups #{i} chat_public_key={bin_to_hex(chat_public_key)}") lIN[0].update({"Chat_public_key": bin_to_hex(chat_public_key)}) if int(bin_to_hex(chat_secret_key), 16) != 0: # 192 * b'0' LOG.info(f"lProcessGroups #{i} chat_secret_key={bin_to_hex(chat_secret_key)}") lIN[0].update({"Chat_secret_key": bin_to_hex(chat_secret_key)}) LOG.info(f"lProcessGroups #{i} self_public_key={bin_to_hex(self_public_key)}") lIN[0].update({"Self_public_key": bin_to_hex(self_public_key)}) LOG.info(f"lProcessGroups #{i} self_secret_key={bin_to_hex(self_secret_key)}") lIN[0].update({"Self_secret_key": bin_to_hex(self_secret_key)}) if self_info is None: LOG.warn(f"lProcessGroups #{i} self_info is None") else: assert len(self_info) == 4, self_info self_nick_len, self_role, self_status, self_nick = self_info self_nick = str(self_nick, sENC) dBINS = {"Self_nick": self_nick, "Self_role": self_role, "Self_status": self_status, "Self_info": self_info, } LOG.info(f"lProcessGroups #{i} {repr(dBINS)}") lIN += [dBINS] if saved_peers is None: LOG.warn(f"lProcessGroups #{i} saved_peers is None") else: assert len(saved_peers) == 2, saved_peers i += 1 except Exception as e: LOG.warn(f"process_chunk Groups #{i} error={e}") return lIN def lProcessNodeInfo(state, index, length, result, label="DHTnode"): """Node Info (packed node format) The Node Info data structure contains a Transport Protocol, a Socket Address, and a Public Key. This is sufficient information to start communicating with that node. The binary representation of a Node Info is called the “packed node format”. Length Type Contents 1 bit Transport Protocol UDP = 0, TCP = 1 7 bit Address Family 2 = IPv4, 10 = IPv6 4 | 16 IP address 4 bytes for IPv4, 16 bytes for IPv6 2 Port Number Port number 32 Public Key Node ID """ delta = 0 relay = 0 lIN = [] while length > 0: status = struct.unpack_from(">B", result, delta)[0] if status >= 128: prot = 'TCP' af = status - 128 else: prot = 'UDP' af = status if af == 2: af = 'IPv4' alen = 4 ipaddr = inet_ntop(AF_INET, result[delta+1:delta+1+alen]) else: af = 'IPv6' alen = 16 ipaddr = inet_ntop(AF_INET6, result[delta+1:delta+1+alen]) total = 1 + alen + 2 + 32 port = int(struct.unpack_from(">H", result, delta+1+alen)[0]) pk = bin_to_hex(result[delta+1+alen+2:delta+1+alen+2+32], 32) LOG.info(f"{label} #{relay} bytes={length} status={status} prot={prot} af={af} ip={ipaddr} port={port} pk={pk}") lIN += [{"Bytes": length, "Status": status, "Prot": prot, "Af": af, "Ip": ipaddr, "Port": port, "Pk": pk}] relay += 1 delta += total length -= total return lIN def lProcessDHTnodes(state, index, length, result, label="DHTnode"): relay = 0 status = struct.unpack_from(" 0: slen = struct.unpack_from("B", result, offset+8)[0] assert status < 12 prot = 'UDP' if status == 2: af = 'IPv4' alen = 4 ipaddr = inet_ntop(AF_INET, result[offset+8+1:offset+8+1+alen]) else: af = 'IPv6' alen = 16 ipaddr = inet_ntop(AF_INET6, result[offset+8+1:offset+8+1+alen]) subtotal = 1 + alen + 2 + 32 port = int(struct.unpack_from(">H", result, offset+8+1+alen)[0]) pk = bin_to_hex(result[offset+8+1+alen+2:offset+8+1+alen+2+32], 32) LOG.info(f"{label} #{relay} status={status} ipaddr={ipaddr} port={port} {pk}") lIN += [{ "Status": status, "Prot": prot, "Af": af, "Ip": ipaddr, "Port": port, "Pk": pk}] offset += subtotal relay += 1 delta += total length -= total return lIN def process_chunk(index, state, oArgs=None): global bOUT, aOUT global sENC length = struct.unpack_from(" 0: LOG.warn(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}") elif bDEBUG: LOG.trace(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}") if data_type == MESSENGER_STATE_TYPE_NOSPAMKEYS: nospam = bin_to_hex(result[0:4]) public_key = bin_to_hex(result[4:36]) private_key = bin_to_hex(result[36:68]) LOG.info(f"{label} Nospam = {nospam}") LOG.info(f"{label} Public_key = {public_key}") LOG.info(f"{label} Private_key = {private_key}") aIN = {"Nospam": f"{nospam}", "Public_key": f"{public_key}", "Private_key": f"{private_key}"} aOUT.update({label: aIN}) if oArgs.command == 'edit' and section == label: ## NOSPAMKEYS,.,Nospam,hexstr if key == "Nospam": assert len(val) == 4*2, val result = bytes.fromhex (val) +result[4:] LOG.info(f"{label} {key} EDITED to {val}") ## NOSPAMKEYS,.,Public_key,hexstr elif key == "Public_key": assert len(val) == 32 * 2, val result = +result[0:4] +bytes.fromhex(val) +result[36:] LOG.info(f"{label} {key} EDITED to {val}") ## NOSPAMKEYS,.,Private_key,hexstr elif key == "Private_key": assert len(val) == 32 * 2, val result = +result[0:36] +bytes.fromhex(val) LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_DHT: LOG.debug(f"process_chunk {label} length={length}") if length > 4: lIN = lProcessDHTnodes(state, index, length, result, "DHTnode") else: lIN = [] LOG.info(f"NO {label}") aOUT.update({label: lIN}) if oArgs.command == 'edit' and section == label: ## DHT,.,DHTnode, if num == '.' and key == "DHTnode" and val in lNULLS: # 4 uint32_t (0x159000D) status = 0x159000D # FixMe - dunno result = struct.pack(" 0: lIN = lProcessFriends(state, index, length, result) else: lIN = [] LOG.info(f"NO {label}") aOUT.update({label: lIN}) elif data_type == MESSENGER_STATE_TYPE_NAME: name = str(result, sENC) LOG.info(f"{label} Nick_name = " +name) aIN = {"Nick_name": name} aOUT.update({label: aIN}) if oArgs.command == 'edit' and section == label: ## NAME,.,Nick_name,str if key == "Nick_name": result = bytes(val, sENC) length = len(result) LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_STATUSMESSAGE: mess = str(result, sENC) LOG.info(f"{label} StatusMessage = " +mess) aIN = {"Status_message": mess} aOUT.update({label: aIN}) if oArgs.command == 'edit' and section == label: ## STATUSMESSAGE,.,Status_message,str if key == "Status_message": result = bytes(val, sENC) length = len(result) LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_STATUS: # 1 uint8_t status (0 = online, 1 = away, 2 = busy) dStatus = {0: 'online', 1: 'away', 2: 'busy'} status = struct.unpack_from(">b", state, index)[0] status = dStatus[status] LOG.info(f"{label} = " +status) aIN = {f"Online_status": status} aOUT.update({label: aIN}) if oArgs.command == 'edit' and section == label: ## STATUS,.,Online_status,int if key == "Online_status": result = struct.pack(">b", int(val)) length = len(result) LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_GROUPS: if length > 0: lIN = lProcessGroups(state, index, length, result, label) else: lIN = [] LOG.info(f"NO {label}") aOUT.update({label: lIN}) elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY: if length > 0: lIN = lProcessNodeInfo(state, index, length, result, "TCPnode") LOG.info(f"TYPE_TCP_RELAY {len(lIN)} nodes {length} length") else: lIN = [] LOG.warn(f"NO {label} {length} length") aOUT.update({label: lIN}) if oArgs.command == 'edit' and section == label: ## TCP_RELAY,.,TCPnode, if num == '.' and key == "TCPnode" and val in lNULLS: result = b'' length = 0 LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_PATH_NODE: #define NUM_SAVED_PATH_NODES 8 if not length % 8 == 0: # this should be an assert? LOG.warn(f"process_chunk {label} mod={length % 8}") else: LOG.debug(f"process_chunk {label} bytes={length}") lIN = lProcessNodeInfo(state, index, length, result, "PATHnode") aOUT.update({label: lIN}) if oArgs.command == 'edit' and section == label: ## PATH_NODE,.,PATHnode, if num == '.' and key == "PATHnode" and val in lNULLS: result = b'' length = 0 LOG.info(f"{label} {key} EDITED to {val}") elif data_type == MESSENGER_STATE_TYPE_CONFERENCES: lIN = [] if length > 0: LOG.debug(f"TODO process_chunk {label} bytes={length}") else: LOG.info(f"NO {label}") aOUT.update({label: []}) elif data_type != MESSENGER_STATE_TYPE_END: LOG.error("UNRECOGNIZED datatype={datatype}") sys.exit(1) else: LOG.info("END") # That's all folks... # drop through # We repack as we read: or edit as we parse; simply edit result and length. # We'll add the results back to bOUT to see if we get what we started with. # Then will will be able to selectively null sections or selectively edit. assert length == len(result), length bOUT += struct.pack("= len(state): diff = len(bSAVE) - len(bOUT) if oArgs.command != 'edit' and diff > 0: # if short repacking as we read - tox_profile is padded with nulls LOG.warn(f"PROCESS_CHUNK bSAVE={len(bSAVE)} bOUT={len(bOUT)} delta={diff}") return process_chunk(new_index, state, oArgs) sNMAP_TCP = """#!/bin/bash ip="" declare -a ports jq '.|with_entries(select(.key|match("nodes"))).nodes[]|select(.status_tcp)|select(.ipv4|match("."))|.ipv4,.tcp_ports' | while read line ; do if [ -z "$ip" ] ; then ip=`echo $line|sed -e 's/"//g'` ports=() continue elif [ "$line" = '[' ] ; then continue elif [ "$line" = ']' ] ; then if ! route | grep -q ^def ; then echo ERROR no route exit 3 fi if [ "$ip" = '"NONE"' -o "$ip" = 'NONE' ] ; then : elif ping -c 1 $ip | grep '100% packet loss' ; then echo WARN failed ping $ip else echo INFO $ip "${ports[*]}" cmd="nmap -Pn -n -sT -p T:"`echo "${ports[*]}" |sed -e 's/ /,/g'` echo DBUG $cmd $ip $cmd $ip | grep /tcp fi ip="" continue else port=`echo $line|sed -e 's/,//'` ports+=($port) fi done""" def sBashFileNmapTcp(): assert bHAVE_JQ, "jq is required for this command" assert bHAVE_NMAP, "nmap is required for this command" assert bHAVE_BASH, "bash is required for this command" f = "NmapTcp.bash" sFile = os.path.join(sDIR, f) if not os.path.exists(sFile): with open(sFile, 'wt') as iFd: iFd.write(sNMAP_TCP) os.chmod(sFile, 0o0775) assert os.path.exists(sFile) return sFile def vBashFileNmapUdp(): assert bHAVE_JQ, "jq is required for this command" assert bHAVE_NMAP, "nmap is required for this command" assert bHAVE_BASH, "bash is required for this command" f = "NmapUdp.bash" sFile = os.path.join(sDIR, f) if not os.path.exists(sFile): with open(sFile, 'wt') as iFd: iFd.write(sNMAP_TCP. replace('nmap -Pn -n -sT -p T', 'nmap -Pn -n -sU -p U'). replace('tcp_ports','udp_ports'). replace('status_tcp','status_udp')) os.chmod(sFile, 0o0775) assert os.path.exists(sFile) return sFile def lParseNapOutput(sFile): lRet = [] for sLine in open(sFile, 'rt').readlines(): if sLine.startswith('Failed to resolve ') or \ 'Temporary failure in name resolution' in sLine or \ '/udp closed' in sLine '/tcp closed' in sLine: lRet += [sLine] return lRet sBLURB = """ I see you have a torrc. You can help the network by running a bootstrap daemon as a hidden service, or even using the --tcp_server option of your client. """ def lNodesCheckNodes(json_nodes, oArgs, bClean=False): """ Checking NODES.json """ lErrs = [] iErrs = 0 nth = 0 if bClean: lNew=[] # assert type(json_nodes) == dict for node in json_nodes: # new fields: if bClean: new_node = {} for key,val in node: if type(val) == bytes: new_node[key] = str(val, 'UTF-8') else: new_node[key] = val if 'onions' not in new_node: new_node['onions'] = [] for ipv in ['ipv4','ipv6']: if not node[ipv] in lNULLS: LOG.info(f"Checking {node[ipv]}") for fam in ["status_tcp", "status_udp"]: if node[ipv] in lNULLS \ and node[fam] in [True, "true"]: LOG.warn(f"{ipv} {node[ipv]} in [-, 'NONE'] but node[{fam}] is true") if bHAVE_NMAP and bAreWeConnected and bAreWeConnected() \ and not node[ipv] in lNULLS: # nmap test the ipv4/ipv6 lElts = [[node[host], node[port], node[key]]] ts.bootstrap_iNmapInfo(lElts, oArgs, bIS_LOCAL=False, iNODES=2) if node['ipv4'] in lNULLS and node['ipv6'] in lNULLS and \ not node['tcp_ports'] and not '.onion' in node['location']: LOG.warn("No ports to contact the daemon on") if node["version"] and node["version"] < "1000002013": lErrs += [nth] LOG.error(f"vulnerable version {node['version']} < 1000002013") elif node["version"] and node["version"] < sVER_MIN: LOG.warn(f"outdated version {node['version']} < {sVER_MIN}") # Put the onion address in the location after the country code if len(node["location"]) not in [2, 65]: LOG.warn(f"location {location} should be a 2 digit country code, or 'code onion'") elif len(node["location"]) == 65 and \ not node["location"].endswith('.onion') : LOG.warn(f"location {location} should be a 2 digit country code 'code onion'") elif len(node["location"]) == 65 and \ node["location"].endswith('.onion') and bHAVE_TOR: onion = node["location"][3:] if bHAVE_TOR and bAreWeConnected and bAreWeConnected() \ and (not node[ipv] in lNULLS and not node[ipv] in lNULLS ): # torresolve the onion # Fixme - see if tor is running try: s = sTorResolve(onion, verbose=False, sHost='127.0.0.1', iPort=9050) except: # assume tor isnt running pass else: if s: LOG.info(f"Found an onion that resolves to {s}") else: LOG.warn(f"Found an onion that resolves to {s}") if node['last_ping'] and time.time() - node['last_ping'] > iOLD_SECS: LOG.debug(f"node has not been pinged in more than 3 months") # suggestions YMMV if len(node['maintainer']) > 75 and len(node['motd']) < 75: pass # look for onion LOG.debug(f"Maybe put a ToxID: in motd so people can contact you.") if bClean and not nth in lErrs: lNew+=[new_node] nth += 1 # fixme look for /etc/tor/torrc but it may not be readable if bHAVE_TOR and os.path.exists('/etc/tor/torrc'): print(sBLURB) if bClean: return lNew else: return lErrs def iNodesFileCheck(sProOrNodes, oArgs, bClean=False): try: if not os.path.exists(sProOrNodes): raise RuntimeError("iNodesFileCheck file not found " +sProOrNodes) with open(sProOrNodes, 'rt') as fl: json_all = json.loads(fl.read()) json_nodes = json_all['nodes'] except Exception as e: LOG.exception(f"{oArgs.command} error reading {sProOrNodes}") return 1 LOG.info(f"iNodesFileCheck checking JSON") i = 0 try: al = lNodesCheckNodes(json_nodes, oArgs, bClean=bClean) if bClean == False: i = len(al) else: now = time.time() aOut = dict(last_scan=json_all['last_scan'], last_refresh=now, nodes=al) sOut = oArgs.output try: LOG.debug(f"iNodesFileClean saving to {sOut}") oStream = open(sOut, 'wt', encoding=sENC) json.dump(aOut, oStream, indent=oArgs.indent) if oStream.write('\n') > 0: i = 0 except Exception as e: LOG.exception(f"iNodesFileClean error dumping JSON to {sOUT}") return 3 except Exception as e: LOG.exception(f"iNodesFileCheck error checking JSON") i = -2 else: if i: LOG.error(f"iNodesFileCheck {i} errors in {sProOrNodes}") else: LOG.info(f"iNodesFileCheck NO errors in {sProOrNodes}") return i def iNodesFileClean(sProOrNodes): # unused return 0 f = "DHTNodes.clean" if not oArgs.output: sOut = os.path.join(sDIR, f) else: sOut = oArgs.output try: LOG.debug(f"iNodesFileClean saving to {sOut}") oStream = open(sOut, 'wt', encoding=sENC) json.dump(aOUT, oStream, indent=oArgs.indent) if oStream.write('\n') > 0: iRet = 0 except Exception as e: LOG.exception(f"iNodesFileClean error dumping JSON to {sOUT}") return 3 LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}") return 0 def iOsSystemNmapUdp(l, oArgs): iErrs = 0 for elt in l: cmd = f"sudo nmap -Pn -n -sU -p U:{elt['Port']} {elt['Ip']}" LOG.debug(f"{oArgs.info} {cmd} to {oArgs.output}") iErrs += os.system(cmd +f" >> {oArgs.output} 2>&1") if iErrs: LOG.warn(f"{oArgs.info} {iErrs} ERRORs to {oArgs.output}") else: LOG.info(f"{oArgs.info} NO errors to {oArgs.output}") lRet = lParseNapOutput(oArgs.output) if lRet: for sLine in lRet: LOG.warn(f"{oArgs.nodes} {sLine}") iErr = len(lRet) iErrs += iErr return iErrs def iOsSystemNmapTcp(l, oArgs): iErrs = 0 LOG.debug(f"{len(l)} nodes to {oArgs.output}") for elt in l: cmd = f"sudo nmap -Pn -n -sT -p T:{elt['Port']} {elt['Ip']}" LOG.debug(f"iOsSystemNmapTcp {cmd} to {oArgs.output}") iErr += os.system(cmd +f" >> {oArgs.output} 2>&1") if iErr: LOG.warn(f"iOsSystemNmapTcp {iErrs} ERRORs to {oArgs.output}") else: lRet = lParseNapOutput(oArgs.output) if lRet: for sLine in lRet: LOG.warn(f"{oArgs.nodes} {sLine}") iErr = len(lRet) iErrs += iErr return iErrs def vSetupLogging(loglevel=logging.DEBUG): global LOG if coloredlogs: aKw = dict(level=loglevel, logger=LOG, fmt='%(name)s %(levelname)s %(message)s') coloredlogs.install(**aKw) else: aKw = dict(level=loglevel, format='%(name)s %(levelname)-4s %(message)s') logging.basicConfig(**aKw) logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' logging._defaultFormatter.default_msec_format = '' def iMain(sProOrNodes, oArgs): global bOUT, aOUT, sENC global bSAVE assert os.path.isfile(sProOrNodes), sProOrNodes sENC = oArgs.encoding bSAVE = open(sProOrNodes, 'rb').read() if ToxEncryptSave and bSAVE[:8] == b'toxEsave': try: bSAVE = decrypt_data(bSAVE) except Exception as e: LOG.error(f"decrypting {sProOrNodes} - {e}") sys.exit(1) assert bSAVE LOG.debug(f"{oArgs.command} {len(bSAVE)} bytes") oStream = None if oArgs.command == 'decrypt': assert oArgs.output, "--output required for this command" oStream = open(oArgs.output, 'wb') iRet = oStream.write(bSAVE) LOG.info(f"Wrote {iRet} to {oArgs.output}") iRet = 0 elif oArgs.command == 'nodes': iRet = -1 ep_sec = str(int(time.time())) json_head = '{"last_scan":' +ep_sec \ +',"last_refresh":' +ep_sec \ +',"nodes":[' if oArgs.nodes == 'select_tcp': assert oArgs.output, "--output required for this command" assert bHAVE_JQ, "jq is required for this command" with open(oArgs.output, 'wt') as oFd: oFd.write(json_head) cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' " iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') elif oArgs.nodes == 'select_udp': assert oArgs.output, "--output required for this command" assert bHAVE_JQ, "jq is required for this command" with open(oArgs.output, 'wt') as oFd: oFd.write(json_head) cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' " iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') elif oArgs.nodes == 'select_version': assert bHAVE_JQ, "jq is required for this command" assert oArgs.output, "--output required for this command" with open(oArgs.output, 'wt') as oFd: oFd.write(json_head) cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))'" iRet = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') elif oArgs.nodes == 'nmap_tcp': assert oArgs.output, "--output required for this command" if not bAreWeConnected(): LOG.warn(f"{oArgs.nodes} we are not connected") else: cmd = sBashFileNmapTcp() cmd = f"bash {cmd} < '{sProOrNodes}' >'{oArgs.output}' 2>&1" LOG.debug(cmd) iRet = os.system(cmd) if iRet == 0: lRet = lParseNapOutput(oArgs.output) if lRet: for sLine in lRet: LOG.warn(f"{oArgs.nodes} {sLine}") iRet = len(lRet) elif oArgs.nodes == 'nmap_udp': assert oArgs.output, "--output required for this command" if not bAreWeConnected(): LOG.warn(f"{oArgs.nodes} we are not connected") elif bHAVE_TOR: LOG.warn(f"{oArgs.nodes} this wont work behind tor") cmd = vBashFileNmapUdp() cmd = f"bash {cmd} < '{sProOrNodes}'" +f" >'{oArgs.output}' 2>&1" LOG.debug(cmd) iRet = os.system(cmd) if iRet == 0: lRet = lParseNapOutput(oArgs.output) if lRet: for sLine in lRet: LOG.warn(f"{oArgs.nodes} {sLine}") iRet = len(lRet) elif oArgs.nodes == 'download' and download_url: if not bAreWeConnected(): LOG.warn(f"{oArgs.nodes} we are not connected") url = oArgs.download_nodes_url b = download_url(url) if not b: LOG.warn("failed downloading list of nodes") iRet = -1 else: if oArgs.output: oStream = open(oArgs.output, 'wb') oStream.write(b) else: oStream = sys.stdout oStream.write(str(b, sENC)) iRet = 0 LOG.info(f"downloaded list of nodes to {oStream}") elif oArgs.nodes == 'check': i = iNodesFileCheck(sProOrNodes, oArgs, bClean=False) iRet = i elif oArgs.nodes == 'clean': assert oArgs.output, "--output required for this command" i = iNodesFileCheck(sProOrNodes, oArgs, bClean=True) iRet = i if iRet > 0: LOG.warn(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}") elif iRet == 0: LOG.info(f"{oArgs.nodes} iRet={iRet} to {oArgs.output}") elif oArgs.command in ['info', 'edit']: if oArgs.command in ['edit']: assert oArgs.output, "--output required for this command" assert oArgs.edit != '', "--edit required for this command" elif oArgs.command == 'info': # assert oArgs.info != '', "--info required for this command" if oArgs.info in ['save', 'yaml', 'json', 'repr', 'pprint']: assert oArgs.output, "--output required for this command" # toxEsave assert bSAVE[:8] == bMARK, "Not a Tox profile" bOUT = bMARK iErrs = 0 process_chunk(len(bOUT), bSAVE, oArgs) if not bOUT: LOG.error(f"{oArgs.command} NO bOUT results") iRet = 1 else: oStream = None LOG.debug(f"command={oArgs.command} len bOUT={len(bOUT)} results") if oArgs.command in ['edit'] or oArgs.info in ['save']: LOG.debug(f"{oArgs.command} saving to {oArgs.output}") oStream = open(oArgs.output, 'wb', encoding=None) if oStream.write(bOUT) > 0: iRet = 0 LOG.info(f"{oArgs.info}ed iRet={iRet} to {oArgs.output}") elif oArgs.info == 'info': pass iRet = 0 elif oArgs.info == 'yaml': if not yaml: LOG.warn(f"{oArgs.command} no yaml support") iRet = -1 else: LOG.debug(f"{oArgs.command} saving to {oArgs.output}") oStream = open(oArgs.output, 'wt', encoding=sENC) try: assert aOUT yaml.dump(aOUT, stream=oStream, indent=oArgs.indent) except Exception as e: LOG.warn(f'WARN: {e}') else: oStream.write('\n') iRet = 0 LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}") elif oArgs.info == 'json': if not yaml: LOG.warn(f"{oArgs.command} no json support") iRet = -1 else: LOG.debug(f"{oArgs.command} saving to {oArgs.output}") oStream = open(oArgs.output, 'wt', encoding=sENC) try: json.dump(aOUT, oStream, indent=oArgs.indent, skipkeys=True) except: LOG.warn("There are somtimes problems with the json info dump of bytes keys: ```TypeError: Object of type bytes is not JSON serializable```") oStream.write('\n') > 0 iRet = 0 LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}") elif oArgs.info == 'repr': LOG.debug(f"{oArgs.command} saving to {oArgs.output}") oStream = open(oArgs.output, 'wt', encoding=sENC) if oStream.write(repr(bOUT)) > 0: iRet = 0 if oStream.write('\n') > 0: iRet = 0 LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}") elif oArgs.info == 'pprint': LOG.debug(f"{oArgs.command} saving to {oArgs.output}") oStream = open(oArgs.output, 'wt', encoding=sENC) pprint(aOUT, stream=oStream, indent=oArgs.indent, width=80) iRet = 0 LOG.info(f"{oArgs.info}ing iRet={iRet} to {oArgs.output}") elif oArgs.info == 'nmap_relay': assert bHAVE_NMAP, "nmap is required for this command" assert oArgs.output, "--output required for this command" if aOUT["TCP_RELAY"]: iRet = iOsSystemNmapTcp(aOUT["TCP_RELAY"], oArgs) else: LOG.warn(f"{oArgs.info} no TCP_RELAY") iRet = 0 elif oArgs.info == 'nmap_dht': assert bHAVE_NMAP, "nmap is required for this command" assert oArgs.output, "--output required for this command" if aOUT["DHT"]: iRet = iOsSystemNmapUdp(aOUT["DHT"], oArgs) else: LOG.warn(f"{oArgs.info} no DHT") iRet = 0 elif oArgs.info == 'nmap_path': assert bHAVE_NMAP, "nmap is required for this command" assert oArgs.output, "--output required for this command" if aOUT["PATH_NODE"]: iRet = iOsSystemNmapUdp(aOUT["PATH_NODE"], oArgs) else: LOG.warn(f"{oArgs.info} no PATH_NODE") iRet = 0 if oStream and oStream != sys.stdout and oStream != sys.stderr: oStream.close() return iRet def oMainArgparser(_=None): if not os.path.exists('/proc/sys/net/ipv6'): bIpV6 = 'False' else: bIpV6 = 'True' lIpV6Choices=[bIpV6, 'False'] parser = argparse.ArgumentParser(epilog=__doc__) # list(dSTATE_TYPE.values()) # ['nospamkeys', 'dht', 'friends', 'name', 'statusmessage', 'status', 'groups', 'tcp_relay', 'path_node', 'conferences'] parser.add_argument('--output', type=str, default='', help='Destination for info/decrypt - defaults to stderr') parser.add_argument('--command', type=str, default='info', choices=['info', 'decrypt', 'nodes', 'edit'], help='Action command - default: info') # nargs='+', parser.add_argument('--edit', type=str, default='', help='comma seperated SECTION,num,key,value - or help for ') parser.add_argument('--indent', type=int, default=2, help='Indent for yaml/json/pprint') choices=['info', 'save', 'repr', 'yaml','json', 'pprint'] if bHAVE_NMAP: choices += ['nmap_relay', 'nmap_dht', 'nmap_path'] parser.add_argument('--info', type=str, default='info', choices=choices, help='Format for info command') choices = ['check', 'clean'] if bHAVE_JQ: choices += ['select_tcp', 'select_udp', 'select_version'] if bHAVE_NMAP: choices += ['nmap_tcp', 'nmap_udp'] if download_url: choices += ['download'] parser.add_argument('--nodes', type=str, default='', choices=choices, help='Action for nodes command (requires jq)') parser.add_argument('--download_nodes_url', type=str, default='https://nodes.tox.chat/json') parser.add_argument('--encoding', type=str, default=sENC) parser.add_argument('lprofile', type=str, nargs='+', default=None, help='tox profile files - may be encrypted') return parser if __name__ == '__main__': lArgv = sys.argv[1:] parser = oMainArgparser() oArgs = parser.parse_args(lArgv) if oArgs.command in ['edit'] and oArgs.edit == 'help': l = list(dSTATE_TYPE.values()) l.remove('END') print('Available Sections: ' +repr(l)) print('Supported Quads: section,num,key,type ' +sEDIT_HELP) sys.exit(0) vSetupLogging() i = 0 for sProOrNodes in oArgs.lprofile: i = iMain(sProOrNodes, oArgs) sys.exit(i)