emdee/toxygen
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

update big NGC

Browse source
This commit is contained in:
emdee 2022-09-27 13:51:50 +00:00
parent 675bf1b2b9
commit 870e3125ad
122 changed files with 2945 additions and 768 deletions
Download patch file Download diff file Expand all files Collapse all files

320
toxygen/av/calls.py
View file

@ -1,14 +1,26 @@
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
import pyaudio
import time
import threading
from wrapper.toxav_enums import *
import cv2
import itertools
import numpy as np
from wrapper.toxav_enums import *
from av import screen_sharing
from av.call import Call
import common.tox_save
from utils import ui as util_ui
import tests.support_testing as ts
from middleware.threads import invoke_in_main_thread
from main import sleep
from middleware.threads import BaseThread
global LOG
import logging
LOG = logging.getLogger('app.'+__name__)
TIMER_TIMEOUT = 30.0
bSTREAM_CALLBACK = False
class AV(common.tox_save.ToxAvSave):
@ -16,6 +28,13 @@ class AV(common.tox_save.ToxAvSave):
super().__init__(toxav)
self._settings = settings
self._running = True
s = settings
if 'video' not in s:
LOG.warn("AV.__init__ 'video' not in s" )
LOG.debug(f"AV.__init__ {s!r}" )
elif 'device' not in s['video']:
LOG.warn("AV.__init__ 'device' not in s.video" )
LOG.debug(f"AV.__init__ {s['video']!r}" )
self._calls = {} # dict: key - friend number, value - Call instance
@ -25,17 +44,27 @@ class AV(common.tox_save.ToxAvSave):
self._audio_running = False
self._out_stream = None
self._audio_rate = 8000
self._audio_channels = 1
self._audio_duration = 60
self._audio_sample_count = self._audio_rate * self._audio_channels * self._audio_duration // 1000
self._audio_rate_pa = 48000
self._audio_rate_tox = 48000
self._audio_rate_pa = 48000
self._audio_krate_tox_audio = self._audio_rate_tox // 1000
self._audio_krate_tox_video = 5000
self._audio_sample_count_pa = self._audio_rate_pa * self._audio_channels * self._audio_duration // 1000
self._audio_sample_count_tox = self._audio_rate_tox * self._audio_channels * self._audio_duration // 1000
self._video = None
self._video_thread = None
self._video_running = False
self._video_width = 640
self._video_height = 480
self._video_width = 320
self._video_height = 240
iOutput = self._settings._args.audio['output']
self.lPaSampleratesO = ts.lSdSamplerates(iOutput)
global oPYA
oPYA = self._audio = pyaudio.PyAudio()
def stop(self):
self._running = False
@ -51,27 +80,70 @@ class AV(common.tox_save.ToxAvSave):
def __call__(self, friend_number, audio, video):
"""Call friend with specified number"""
self._toxav.call(friend_number, 32 if audio else 0, 5000 if video else 0)
if friend_number in self._calls:
LOG.warn(f"__call__ already has {friend_number}")
return
if self._audio_krate_tox_audio not in ts.lToxSampleratesK:
LOG.warn(f"__call__ {self._audio_krate_tox_audio} not in {ts.lToxSampleratesK}")
try:
self._toxav.call(friend_number,
self._audio_krate_tox_audio if audio else 0,
self._audio_krate_tox_video if video else 0)
except ArgumentError as e:
LOG.warn(f"_toxav.call already has {friend_number}")
return
self._calls[friend_number] = Call(audio, video)
threading.Timer(30.0, lambda: self.finish_not_started_call(friend_number)).start()
threading.Timer(TIMER_TIMEOUT,
lambda: self.finish_not_started_call(friend_number)).start()
def accept_call(self, friend_number, audio_enabled, video_enabled):
# obsolete
return call_accept_call(self, friend_number, audio_enabled, video_enabled)
def call_accept_call(self, friend_number, audio_enabled, video_enabled):
LOG.debug(f"call_accept_call from {friend_number} {self._running}" +
f"{audio_enabled} {video_enabled}")
# import pdb; pdb.set_trace() - gets into q Qt exec_ problem
# ts.trepan_handler()
if self._audio_krate_tox_audio not in ts.lToxSampleratesK:
LOG.warn(f"__call__ {self._audio_krate_tox_audio} not in {ts.lToxSampleratesK}")
if self._running:
self._calls[friend_number] = Call(audio_enabled, video_enabled)
self._toxav.answer(friend_number, 32 if audio_enabled else 0, 5000 if video_enabled else 0)
# audio_bit_rate: Audio bit rate in Kb/sec. Set this to 0 to disable audio sending.
# video_bit_rate: Video bit rate in Kb/sec. Set this to 0 to disable video sending.
try:
self._toxav.answer(friend_number,
self._audio_krate_tox_audio if audio_enabled else 0,
self._audio_krate_tox_video if video_enabled else 0)
except ArgumentError as e:
LOG.debug(f"AV accept_call error from {friend_number} {self._running}" +
f"{e}")
raise
if audio_enabled:
# may raise
self.start_audio_thread()
if video_enabled:
# may raise
self.start_video_thread()
def finish_call(self, friend_number, by_friend=False):
LOG.debug(f"finish_call {friend_number}")
if not by_friend:
self._toxav.call_control(friend_number, TOXAV_CALL_CONTROL['CANCEL'])
if friend_number in self._calls:
del self._calls[friend_number]
if not len(list(filter(lambda c: c.out_audio, self._calls))):
try:
# AttributeError: 'int' object has no attribute 'out_audio'
if not len(list(filter(lambda c: c.out_audio, self._calls))):
self.stop_audio_thread()
if not len(list(filter(lambda c: c.out_video, self._calls))):
self.stop_video_thread()
except Exception as e:
LOG.error(f"finish_call FixMe: {e}")
# dunno
self.stop_audio_thread()
if not len(list(filter(lambda c: c.out_video, self._calls))):
self.stop_video_thread()
def finish_not_started_call(self, friend_number):
@ -84,6 +156,7 @@ class AV(common.tox_save.ToxAvSave):
"""
New call state
"""
LOG.debug(f"toxav_call_state_cb {friend_number}")
call = self._calls[friend_number]
call.is_active = True
@ -107,31 +180,80 @@ class AV(common.tox_save.ToxAvSave):
"""
Start audio sending
"""
global oPYA
iInput = self._settings._args.audio['input']
if self._audio_thread is not None:
LOG.warn(f"start_audio_thread device={iInput}")
return
iInput = self._settings._args.audio['input']
LOG.debug(f"start_audio_thread device={iInput}")
lPaSamplerates = ts.lSdSamplerates(iInput)
if not(len(lPaSamplerates)):
e = f"No supported sample rates for device: audio[input]={iInput!r}"
LOG.error(f"No supported sample rates {e}")
raise RuntimeError(e)
if not self._audio_rate_pa in lPaSamplerates:
LOG.warn(f"{self._audio_rate_pa} not in {lPaSamplerates!r}")
if False:
self._audio_rate_pa = oPYA.get_device_info_by_index(iInput)['defaultSampleRate']
else:
LOG.warn(f"Setting audio_rate to: {lPaSamplerates[0]}")
self._audio_rate_pa = lPaSamplerates[0]
self._audio_running = True
try:
LOG.debug( f"start_audio_thread framerate: {self._audio_rate_pa}" \
+f" device: {iInput}"
+f" supported: {lPaSamplerates!r}")
if self._audio_rate_pa not in lPaSamplerates:
LOG.warn(f"PAudio sampling rate was {self._audio_rate_pa} changed to {lPaSamplerates[0]}")
self._audio_rate_pa = lPaSamplerates[0]
self._audio = pyaudio.PyAudio()
self._audio_stream = self._audio.open(format=pyaudio.paInt16,
rate=self._audio_rate,
channels=self._audio_channels,
input=True,
input_device_index=self._settings.audio['input'],
frames_per_buffer=self._audio_sample_count * 10)
if bSTREAM_CALLBACK:
self._audio_stream = oPYA.open(format=pyaudio.paInt16,
rate=self._audio_rate_pa,
channels=self._audio_channels,
input=True,
input_device_index=iInput,
frames_per_buffer=self._audio_sample_count_pa * 10,
stream_callback=self.send_audio_data)
self._audio_running = True
self._audio_stream.start_stream()
while self._audio_stream.is_active():
sleep(0.1)
self._audio_stream.stop_stream()
self._audio_stream.close()
self._audio_thread = threading.Thread(target=self.send_audio)
self._audio_thread.start()
else:
self._audio_stream = oPYA.open(format=pyaudio.paInt16,
rate=self._audio_rate_pa,
channels=self._audio_channels,
input=True,
input_device_index=iInput,
frames_per_buffer=self._audio_sample_count_pa * 10)
self._audio_running = True
self._audio_thread = BaseThread(target=self.send_audio,
name='_audio_thread')
self._audio_thread.start()
except Exception as e:
LOG.error(f"Starting self._audio.open {e}")
LOG.debug(repr(dict(format=pyaudio.paInt16,
rate=self._audio_rate_pa,
channels=self._audio_channels,
input=True,
input_device_index=iInput,
frames_per_buffer=self._audio_sample_count_pa * 10)))
# catcher in place in calls_manager
raise RuntimeError(e)
else:
LOG.debug(f"start_audio_thread {self._audio_stream!r}")
def stop_audio_thread(self):
if self._audio_thread is None:
return
self._audio_running = False
self._audio_thread.join()
self._audio_thread = None
self._audio_stream = None
self._audio = None
@ -144,21 +266,39 @@ class AV(common.tox_save.ToxAvSave):
def start_video_thread(self):
if self._video_thread is not None:
return
s = self._settings
if 'video' not in s:
LOG.warn("AV.__init__ 'video' not in s" )
LOG.debug(f"start_video_thread {s!r}" )
raise RuntimeError("start_video_thread not 'video' in s)" )
elif 'device' not in s['video']:
LOG.error("start_video_thread not 'device' in s['video']" )
LOG.debug(f"start_video_thread {s['video']!r}" )
raise RuntimeError("start_video_thread not 'device' ins s['video']" )
self._video_width = s['video']['width']
self._video_height = s['video']['height']
LOG.info("start_video_thread " \
+f" device: {s['video']['device']}" \
+f" supported: {s['video']['width']} {s['video']['height']}")
s['video']['device'] = -1
if s['video']['device'] == -1:
self._video = screen_sharing.DesktopGrabber(s['video']['x'],
s['video']['y'],
s['video']['width'],
s['video']['height'])
else:
with ts.ignoreStdout():
import cv2
self._video = cv2.VideoCapture(s['video']['device'])
self._video.set(cv2.CAP_PROP_FPS, 25)
self._video.set(cv2.CAP_PROP_FRAME_WIDTH, self._video_width)
self._video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._video_height)
self._video_running = True
self._video_width = s.video['width']
self._video_height = s.video['height']
if s.video['device'] == -1:
self._video = screen_sharing.DesktopGrabber(self._settings.video['x'], self._settings.video['y'],
self._settings.video['width'], self._settings.video['height'])
else:
self._video = cv2.VideoCapture(self._settings.video['device'])
self._video.set(cv2.CAP_PROP_FPS, 25)
self._video.set(cv2.CAP_PROP_FRAME_WIDTH, self._video_width)
self._video.set(cv2.CAP_PROP_FRAME_HEIGHT, self._video_height)
self._video_thread = threading.Thread(target=self.send_video)
self._video_thread = BaseThread(target=self.send_video,
name='_video_thread')
self._video_thread.start()
def stop_video_thread(self):
@ -166,7 +306,17 @@ class AV(common.tox_save.ToxAvSave):
return
self._video_running = False
self._video_thread.join()
i = 0
while i < ts.iTHREAD_JOINS:
self._video_thread.join(ts.iTHREAD_TIMEOUT)
try:
if not self._video_thread.is_alive(): break
except:
# AttributeError: 'NoneType' object has no attribute 'join'
break
i = i + 1
else:
LOG.warn("self._video_thread.is_alive BLOCKED")
self._video_thread = None
self._video = None
@ -180,58 +330,109 @@ class AV(common.tox_save.ToxAvSave):
"""
if self._out_stream is None:
self._out_stream = self._audio.open(format=pyaudio.paInt16,
channels=channels_count,
rate=rate,
output_device_index=self._settings.audio['output'],
output=True)
iOutput = self._settings._args.audio['output']
if not rate in self.lPaSampleratesO:
LOG.warn(f"{rate} not in {self.lPaSampleratesO!r}")
if False:
rate = oPYA.get_device_info_by_index(iOutput)['defaultSampleRate']
LOG.warn(f"Setting audio_rate to: {self.lPaSampleratesO[0]}")
rate = self.lPaSampleratesO[0]
try:
with ts.ignoreStderr():
self._out_stream = oPYA.open(format=pyaudio.paInt16,
channels=channels_count,
rate=rate,
output_device_index=iOutput,
output=True)
except Exception as e:
LOG.error(f"Error playing audio_chunk creating self._out_stream {e}")
LOG.debug(f"audio_chunk output_device_index={self._settings._args.audio['input']} rate={rate} channels={channels_count}")
invoke_in_main_thread(util_ui.message_box,
str(e),
util_ui.tr("Error Chunking audio"))
# dunno
self.stop()
return
self._out_stream.write(samples)
# -----------------------------------------------------------------------------------------------------------------
# AV sending
# -----------------------------------------------------------------------------------------------------------------
def send_audio_data(self, data, count, *largs, **kwargs):
pcm = data
# :param sampling_rate: Audio sampling rate used in this frame.
if self._toxav is None:
raise RuntimeError("_toxav not initialized")
if self._audio_rate_tox not in ts.lToxSamplerates:
LOG.warn(f"ToxAudio sampling rate was {self._audio_rate_tox} changed to {ts.lToxSamplerates[0]}")
self._audio_rate_tox = ts.lToxSamplerates[0]
for friend_num in self._calls:
if self._calls[friend_num].out_audio:
try:
# app.av.calls ERROR Error send_audio: One of the frame parameters was invalid. E.g. the resolution may be too small or too large, or the audio sampling rate may be unsupported
self._toxav.audio_send_frame(friend_num,
pcm,
count,
self._audio_channels,
self._audio_rate_tox)
except Exception as e:
LOG.error(f"Error send_audio audio_send_frame: {e}")
LOG.debug(f"send_audio self._audio_rate_tox={self._audio_rate_tox} self._audio_channels={self._audio_channels}")
invoke_in_main_thread(util_ui.message_box,
str(e),
util_ui.tr("Error send_audio audio_send_frame"))
pass
def send_audio(self):
"""
This method sends audio to friends
"""
i=0
count = self._audio_sample_count_tox
LOG.debug(f"send_audio stream={self._audio_stream}")
while self._audio_running:
try:
pcm = self._audio_stream.read(self._audio_sample_count)
if pcm:
for friend_num in self._calls:
if self._calls[friend_num].out_audio:
try:
self._toxav.audio_send_frame(friend_num, pcm, self._audio_sample_count,
self._audio_channels, self._audio_rate)
except:
pass
pcm = self._audio_stream.read(count, exception_on_overflow=False)
if not pcm:
sleep(0.1)
else:
self.send_audio_data(pcm, count)
except:
pass
time.sleep(0.01)
i += 1
LOG.debug(f"send_audio {i}")
sleep(0.01)
def send_video(self):
"""
This method sends video to friends
"""
LOG.debug(f"send_video thread={threading.current_thread()}"
+f" self._video_running={self._video_running}"
+f" device: {self._settings['video']['device']}" )
while self._video_running:
try:
result, frame = self._video.read()
if result:
LOG.warn(f"send_video video_send_frame _video.read")
else:
height, width, channels = frame.shape
for friend_num in self._calls:
if self._calls[friend_num].out_video:
try:
y, u, v = self.convert_bgr_to_yuv(frame)
self._toxav.video_send_frame(friend_num, width, height, y, u, v)
except:
except Exception as e:
LOG.debug(f"send_video video_send_frame ERROR {e}")
pass
except:
pass
time.sleep(0.01)
sleep(0.1)
def convert_bgr_to_yuv(self, frame):
"""
@ -264,11 +465,14 @@ class AV(common.tox_save.ToxAvSave):
Y, U, V can be extracted using slices and joined in one list using itertools.chain.from_iterable()
Function returns bytes(y), bytes(u), bytes(v), because it is required for ctypes
"""
with ts.ignoreStdout():
import cv2
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
y = frame[:self._video_height, :]
y = list(itertools.chain.from_iterable(y))
import numpy as np
u = np.zeros((self._video_height // 2, self._video_width // 2), dtype=np.int)
u[::2, :] = frame[self._video_height:self._video_height * 5 // 4, :self._video_width // 2]
u[1::2, :] = frame[self._video_height:self._video_height * 5 // 4, self._video_width // 2:]