3 #ifdef USE_VOICE_ASSISTANT 11 namespace voice_assistant {
13 static const char *
const TAG =
"voice_assistant";
19 static const size_t SAMPLE_RATE_HZ = 16000;
20 static const size_t INPUT_BUFFER_SIZE = 32 * SAMPLE_RATE_HZ / 1000;
21 static const size_t BUFFER_SIZE = 512 * SAMPLE_RATE_HZ / 1000;
22 static const size_t SEND_BUFFER_SIZE = INPUT_BUFFER_SIZE *
sizeof(int16_t);
23 static const size_t RECEIVE_SIZE = 1024;
24 static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE;
31 ESP_LOGE(TAG,
"Could not create socket");
36 int err = this->
socket_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &enable,
sizeof(
int));
38 ESP_LOGW(TAG,
"Socket unable to set reuseaddr: errno %d", err);
41 err = this->
socket_->setblocking(
false);
43 ESP_LOGE(TAG,
"Socket unable to set nonblocking mode: errno %d", err);
54 ESP_LOGE(TAG,
"Socket unable to set sockaddr: errno %d", errno);
61 ESP_LOGE(TAG,
"Socket unable to bind: errno %d", errno);
72 ESP_LOGCONFIG(TAG,
"Setting up Voice Assistant...");
87 ESP_LOGW(TAG,
"Could not allocate speaker buffer");
96 ESP_LOGW(TAG,
"Could not allocate input buffer");
106 ESP_LOGW(TAG,
"Could not allocate ring buffer");
113 ESP_LOGW(TAG,
"Could not allocate send buffer");
126 memset(this->
input_buffer_, 0, INPUT_BUFFER_SIZE *
sizeof(int16_t));
176 ESP_LOGD(TAG,
"reset conversation ID");
180 size_t bytes_read = 0;
183 if (bytes_read == 0) {
184 memset(this->
input_buffer_, 0, INPUT_BUFFER_SIZE *
sizeof(int16_t));
190 ESP_LOGD(TAG,
"microphone not running");
226 ESP_LOGD(TAG,
"Starting Microphone");
250 ESP_LOGD(TAG,
"Waiting for speech...");
256 if (bytes_read > 0) {
257 vad_state_t vad_state =
259 if (vad_state == VAD_SPEECH) {
263 ESP_LOGD(TAG,
"VAD detected speech");
280 ESP_LOGD(TAG,
"Requesting start...");
300 ESP_LOGW(TAG,
"Could not request start");
318 while (available >= SEND_BUFFER_SIZE) {
322 msg.
data.assign((
char *) this->send_buffer_, read_bytes);
332 sizeof(this->dest_addr_));
358 bool playing =
false;
361 ssize_t received_len = 0;
365 if (received_len > 0) {
371 ESP_LOGD(TAG,
"Receive buffer full");
381 ESP_LOGD(TAG,
"End of audio stream received");
390 #ifdef USE_MEDIA_PLAYER 417 ESP_LOGD(TAG,
"Speaker has finished outputting all audio");
445 this->speaker_buffer_size_ -= written;
449 ESP_LOGV(TAG,
"Speaker buffer full, trying again next loop");
458 ESP_LOGE(TAG,
"Client attempting to unsubscribe that is not the current API Client");
467 ESP_LOGE(TAG,
"Multiple API Clients attempting to connect to Voice Assistant");
477 static const LogString *voice_assistant_state_to_string(
State state) {
480 return LOG_STR(
"IDLE");
482 return LOG_STR(
"START_MICROPHONE");
484 return LOG_STR(
"STARTING_MICROPHONE");
486 return LOG_STR(
"WAIT_FOR_VAD");
488 return LOG_STR(
"WAITING_FOR_VAD");
490 return LOG_STR(
"START_PIPELINE");
492 return LOG_STR(
"STARTING_PIPELINE");
494 return LOG_STR(
"STREAMING_MICROPHONE");
496 return LOG_STR(
"STOP_MICROPHONE");
498 return LOG_STR(
"STOPPING_MICROPHONE");
500 return LOG_STR(
"AWAITING_RESPONSE");
502 return LOG_STR(
"STREAMING_RESPONSE");
504 return LOG_STR(
"RESPONSE_FINISHED");
506 return LOG_STR(
"UNKNOWN");
513 ESP_LOGD(TAG,
"State changed from %s to %s", LOG_STR_ARG(voice_assistant_state_to_string(old_state)),
514 LOG_STR_ARG(voice_assistant_state_to_string(state)));
520 ESP_LOGD(TAG,
"Desired state set to %s", LOG_STR_ARG(voice_assistant_state_to_string(desired_state)));
524 ESP_LOGE(TAG,
"Failed to start server. See Home Assistant logs for more details.");
525 this->
error_trigger_->
trigger(
"failed-to-start",
"Failed to start server. See Home Assistant logs for more details.");
535 ESP_LOGD(TAG,
"Client started, streaming microphone");
551 ESP_LOGD(TAG,
"Client started, streaming microphone");
577 ESP_LOGE(TAG,
"No API client connected");
630 ESP_LOGD(TAG,
"Signaling stop...");
637 ESP_LOGD(TAG,
"Event Type: %" PRId32, msg.
event_type);
640 ESP_LOGD(TAG,
"Assist Pipeline running");
646 ESP_LOGD(TAG,
"Wake word detected");
651 ESP_LOGD(TAG,
"STT started");
656 for (
auto arg : msg.
data) {
657 if (arg.name ==
"text") {
658 text = std::move(arg.value);
662 ESP_LOGW(TAG,
"No text in STT_END event");
665 ESP_LOGD(TAG,
"Speech recognised as: \"%s\"", text.c_str());
670 ESP_LOGD(TAG,
"Intent started");
674 for (
auto arg : msg.
data) {
675 if (arg.name ==
"conversation_id") {
684 for (
auto arg : msg.
data) {
685 if (arg.name ==
"text") {
686 text = std::move(arg.value);
690 ESP_LOGW(TAG,
"No text in TTS_START event");
693 ESP_LOGD(TAG,
"Response: \"%s\"", text.c_str());
694 this->
defer([
this, text]() {
706 for (
auto arg : msg.
data) {
707 if (arg.name ==
"url") {
708 url = std::move(arg.value);
712 ESP_LOGW(TAG,
"No url in TTS_END event");
715 ESP_LOGD(TAG,
"Response URL: \"%s\"", url.c_str());
716 this->
defer([
this, url]() {
717 #ifdef USE_MEDIA_PLAYER 729 ESP_LOGD(TAG,
"Assist Pipeline ended");
749 std::string code =
"";
750 std::string message =
"";
751 for (
auto arg : msg.
data) {
752 if (arg.name ==
"code") {
753 code = std::move(arg.value);
754 }
else if (arg.name ==
"message") {
755 message = std::move(arg.value);
758 if (code ==
"wake-word-timeout" || code ==
"wake_word_detection_aborted" || code ==
"no_wake_word") {
761 }
else if (code ==
"wake-provider-missing" || code ==
"wake-engine-missing") {
763 this->
defer([
this, code, message]() {
769 ESP_LOGE(TAG,
"Error: %s - %s", code.c_str(), message.c_str());
780 ESP_LOGD(TAG,
"TTS stream start");
788 ESP_LOGD(TAG,
"TTS stream end");
793 ESP_LOGD(TAG,
"Starting STT by VAD");
797 ESP_LOGD(TAG,
"STT by VAD end");
802 ESP_LOGD(TAG,
"Unhandled event type: %" PRId32, msg.
event_type);
808 #ifdef USE_SPEAKER // We should never get to this function if there is no speaker anyway 814 ESP_LOGV(TAG,
"Received audio: %u bytes from API", msg.
data.length());
816 ESP_LOGE(TAG,
"Cannot receive audio, buffer is full");
830 ESP_LOGD(TAG,
"Timer Event");
831 ESP_LOGD(TAG,
" Type: %" PRId32, msg.
event_type);
832 ESP_LOGD(TAG,
" %s", timer.
to_string().c_str());
861 std::vector<Timer> res;
862 res.reserve(this->
timers_.size());
863 for (
auto &pair : this->
timers_) {
864 auto &timer = pair.second;
865 if (timer.is_active && timer.seconds_left > 0) {
866 timer.seconds_left--;
868 res.push_back(timer);
874 #ifdef USE_MEDIA_PLAYER 890 #endif // USE_VOICE_ASSISTANT Trigger * tts_stream_start_trigger_
virtual size_t play(const uint8_t *data, size_t length)=0
std::string wake_word_phrase
Trigger< Timer > * timer_cancelled_trigger_
bool wait_for_stream_end_
void set_interval(const std::string &name, uint32_t interval, std::function< void()> &&f)
Set an interval function with a unique name.
const float AFTER_CONNECTION
For components that should be initialized after a data connection (API/MQTT) is connected.
std::unordered_map< std::string, Timer > timers_
bool cancel_timeout(const std::string &name)
Cancel a timeout function.
enums::VoiceAssistantTimerEvent event_type
void deallocate_buffers_()
Trigger * intent_end_trigger_
HighFrequencyLoopRequester high_freq_
VoiceAssistant * global_voice_assistant
std::string conversation_id_
socklen_t set_sockaddr_any(struct sockaddr *addr, socklen_t addrlen, uint16_t port)
Set a sockaddr to the any address and specified port for the IP version used by socket_ip().
std::string conversation_id
api::APIConnection * api_client_
Trigger< Timer > * timer_finished_trigger_
float get_setup_priority() const override
vad_handle_t vad_instance_
std::unique_ptr< socket::Socket > socket_
uint32_t noise_suppression_level
Trigger< std::string > * tts_start_trigger_
void set_timeout(const std::string &name, uint32_t timeout, std::function< void()> &&f)
Set a timeout function with a unique name.
bool cancel_interval(const std::string &name)
Cancel an interval function.
void defer(const std::string &name, std::function< void()> &&f)
Defer a callback to the next loop() call.
void deallocate(T *p, size_t n)
Trigger< std::string > * tts_end_trigger_
VoiceAssistantAudioSettings audio_settings
microphone::Microphone * mic_
Trigger< std::vector< Timer > > * timer_tick_trigger_
enums::VoiceAssistantEvent event_type
void client_subscription(api::APIConnection *client, bool subscribe)
virtual bool has_buffered_data() const =0
std::vector< VoiceAssistantEventData > data
std::string get_client_combined_info() const
uint32_t conversation_timeout_
size_t speaker_bytes_received_
uint8_t * speaker_buffer_
uint8_t noise_suppression_level_
void trigger(Ts... x)
Inform the parent automation that the event has triggered.
bool status_has_error() const
Trigger< Timer > * timer_started_trigger_
Trigger * listening_trigger_
void status_set_error(const char *message="unspecified")
media_player::MediaPlayer * media_player_
void start()
Start running the loop continuously.
Trigger * client_connected_trigger_
bool send_voice_assistant_request(const VoiceAssistantRequest &msg)
void reset_conversation_id()
speaker::Speaker * speaker_
void stop()
Stop running the loop continuously.
void set_state_(State state)
Trigger * stt_vad_start_trigger_
bool send_voice_assistant_announce_finished(const VoiceAssistantAnnounceFinished &msg)
std::unique_ptr< RingBuffer > ring_buffer_
bool send_voice_assistant_audio(const VoiceAssistantAudio &msg)
void on_audio(const api::VoiceAssistantAudio &msg)
Trigger * client_disconnected_trigger_
void status_clear_error()
void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg)
Trigger * stt_vad_end_trigger_
struct sockaddr_storage dest_addr_
virtual size_t read(int16_t *buf, size_t len)=0
Trigger * wake_word_detected_trigger_
Trigger * tts_stream_end_trigger_
virtual void mark_failed()
Mark this component as failed.
Implementation of SPI Controller mode.
size_t speaker_buffer_size_
std::string to_string() const
Trigger * intent_start_trigger_
void on_announce(const api::VoiceAssistantAnnounceRequest &msg)
static std::unique_ptr< RingBuffer > create(size_t len)
Trigger< Timer > * timer_updated_trigger_
size_t speaker_buffer_index_
void on_event(const api::VoiceAssistantEventResponse &msg)
Trigger< std::string > * stt_end_trigger_
void request_start(bool continuous, bool silence_detection)
Trigger< std::string, std::string > * error_trigger_
std::unique_ptr< Socket > socket(int domain, int type, int protocol)
Create a socket of the given domain, type and protocol.