VLM-Driven Spooky Sassy Jack-o’-Lantern on Arduino VENTUNO Q

1import os
2import time
3import queue
4import threading
5import random
6import hashlib
7import logging
8import itertools
9import numpy as np
10
11from arduino.app_utils import App, Bridge
12from arduino.app_peripherals.microphone import Microphone, MicrophoneOpenError
13from arduino.app_bricks.asr import (
14    AutomaticSpeechRecognition,
15    ASRBusyError,
16    ASRServiceBusyError,
17    ASRUnavailableError,
18    ASRError,
19)
20from arduino.app_bricks.vlm import VisionLanguageModel
21from arduino.app_bricks.tts import TTSBusyError, TTSError, TextToSpeech
22from arduino.app_bricks.object_detection import ObjectDetection
23from arduino.app_peripherals.camera import Camera
24from arduino.app_peripherals.speaker import Speaker
25from arduino.app_bricks.web_ui import WebUI
26from arduino.app_utils.image import compress_to_jpeg, resize, rotated, cropped, resized
27from requests.exceptions import ReadTimeout, ConnectionError, ChunkedEncodingError
28
29logging.basicConfig(
30    level=logging.INFO,
31    format="%(asctime)s.%(msecs)03d [%(threadName)s] %(name)s: %(message)s",
32    datefmt="%H:%M:%S",
33)
34log = logging.getLogger("app")
35
36# ---------------------------------------------------------------- tunables
37WAKE_CLASSES = {"face"}
38MIN_BOX_FRAC = 0.02
39WAKE_HITS = 3
40POLL_INTERVAL = 0.3
41LEAVE_AFTER = 25.0
42COOLDOWN = 15.0
43SESSION_READY_DELAY = 2.5
44LISTEN_WINDOW = 3
45IDLE_TIMEOUT = 0.6
46ASR_CLOSE_WAIT = 25.0
47TTS_WAIT_MAX = 30.0
48MIN_TTS_CHARS = 25
49FILLER_ENABLED = False     # live filler through the runner stalls it; cached fillers are used instead
50
51CACHE_DIR = "/tmp/pumpkin_tts_cache"   # delete this folder to force re-rendering
52CACHED_FILLERS = [
53    "Hmm... let me look.",
54    "Ohhh... one moment.",
55    "The shadows stir...",
56    "Let me peer closer...",
57    "Mmm, the candle flickers...",
58]
59MAX_FILLER_SECS = 2.2
60PCM_CACHE = {}      # text -> int16 array (greetings, goodbye, fillers)
61FILLER_PCM = []     # fillers short enough to play during the ASR close
62speaker_lock = threading.Lock()   # keeps a cached line and the answer from playing at once
63
64GREETINGS = [
65    "Ahh, a visitor approaches... speak, mortal.",
66    "Welcome, brave soul... what do you wish to ask?",
67    "I see you... come closer and speak.",
68]
69GOODBYE = "The night grows quiet... farewell, mortal."
70
71FILLER_LINES = [
72    "Ohhh... let me peer through the dark...",
73    "The shadows whisper... just a moment...",
74    "Mmm, flickering candlelight stirs now...",
75    "Patience, mortal... the spirits decide...",
76]
77filler_cycle = itertools.cycle(FILLER_LINES)
78
79S = {"mode": "IDLE", "hits": 0, "last_seen": 0.0, "cooldown_until": 0.0, "debug_logged": False}
80
81
82def light(state: str):
83    try:
84        Bridge.notify(f"set_state_{state}")
85    except Exception as e:
86        log.warning("light(%s) failed: %s", state, e)
87
88
89def wait_asr_idle(max_wait=ASR_CLOSE_WAIT) -> bool:
90    """Wait until no ASR session is open or closing (shared runner must not be hit meanwhile)."""
91    t0 = time.monotonic()
92    while asr.is_transcribing():
93        if time.monotonic() - t0 > max_wait:
94            return False
95        time.sleep(0.05)
96    return True
97
98
99def wait_tts(max_wait=TTS_WAIT_MAX) -> bool:
100    t0 = time.monotonic()
101    while text_queue.unfinished_tasks > 0:
102        if time.monotonic() - t0 > max_wait:
103            log.warning("TTS still busy after %.0fs, cancelling (unfinished=%d)",
104                        max_wait, text_queue.unfinished_tasks)
105            try:
106                tts.cancel()
107            except Exception:
108                pass
109            return False
110        time.sleep(0.02)
111    return True
112
113
114# ------------------------------------------------------------ cached audio
115def render_pcm(text):
116    """Synthesize once; reuse the disk copy on later runs."""
117    key = hashlib.md5(f"{text}|{speaker.sample_rate}".encode()).hexdigest()
118    path = os.path.join(CACHE_DIR, key + ".npy")
119    try:
120        if os.path.exists(path):
121            return np.load(path)
122    except Exception:
123        pass
124    pcm = np.frombuffer(tts.synthesize_pcm(text), dtype=np.int16)
125    try:
126        os.makedirs(CACHE_DIR, exist_ok=True)
127        np.save(path, pcm)
128    except Exception as e:
129        log.warning("could not save cache for %r: %s", text, e)
130    return pcm
131
132
133def prerender_all():
134    """Render everything the pumpkin says without the VLM, while the runner is idle."""
135    t0 = time.monotonic()
136    for line in GREETINGS + [GOODBYE] + CACHED_FILLERS:
137        try:
138            PCM_CACHE[line] = render_pcm(line)
139        except Exception as e:
140            log.warning("render failed for %r: %s", line, e)
141    for line in CACHED_FILLERS:
142        pcm = PCM_CACHE.get(line)
143        if pcm is None:
144            continue
145        secs = len(pcm) / (speaker.sample_rate * speaker.channels)
146        if secs <= MAX_FILLER_SECS:
147            FILLER_PCM.append(pcm)
148        else:
149            log.info("filler %r skipped (%.2fs too long)", line, secs)
150    log.info("cached %d lines (%d fillers) in %.1fs", len(PCM_CACHE), len(FILLER_PCM), time.monotonic() - t0)
151
152
153def play_pcm(pcm):
154    try:
155        with speaker_lock:
156            n = speaker.buffer_size * speaker.channels
157            for i in range(0, len(pcm), n):
158                speaker.play(pcm[i:i + n])
159    except Exception as e:
160        log.warning("cached playback failed: %s", e)
161
162
163def play_filler_async():
164    if FILLER_PCM:
165        threading.Thread(target=play_pcm, args=(random.choice(FILLER_PCM),),
166                         daemon=True, name="filler").start()
167
168
169def say_cached(text):
170    """Blocking. Uses the cached audio (no runner request); falls back to the TTS queue."""
171    pcm = PCM_CACHE.get(text)
172    if pcm is not None:
173        play_pcm(pcm)
174    else:
175        text_queue.put(text)
176        wait_tts()
177
178
179def tts_thread(tts, text_queue):
180    while True:
181        text = text_queue.get()
182        try:
183            if not text:
184                continue
185
186            t_w = time.monotonic()
187            if not wait_asr_idle():
188                log.warning("ASR close still pending after %.0fs, speaking anyway", ASR_CLOSE_WAIT)
189            waited = time.monotonic() - t_w
190            if waited > 0.05:
191                log.info("DIAG tts waited %.2fs for ASR close before speaking", waited)
192
193            with speaker_lock:   # let a cached line that is still playing finish first
194                pass
195
196            attempts = 0
197            while True:
198                t0 = time.monotonic()
199                try:
200                    tts.speak(text)
201                    log.info("tts.speak done in %.2fs (%d chars)", time.monotonic() - t0, len(text))
202                    break
203                except TTSBusyError as e:
204                    log.warning("TTS busy, retrying: %s", e)
205                    time.sleep(0.2)
206                except TTSError as e:
207                    log.error("TTS error, dropping chunk: %s", e)
208                    break
209                except (ConnectionError, ReadTimeout, ChunkedEncodingError) as e:
210                    attempts += 1
211                    log.warning("TTS backend failed after %.1fs (attempt %d): %s",
212                                time.monotonic() - t0, attempts, e)
213                    if attempts >= 3:
214                        log.error("Giving up on this chunk after 3 attempts")
215                        break
216                    time.sleep(1)
217                except Exception as e:
218                    log.exception("Unexpected TTS error after %.1fs, dropping chunk: %s",
219                                  time.monotonic() - t0, e)
220                    break
221        finally:
222            text_queue.task_done()
223
224
225def speech_recognition(idle_timeout=IDLE_TIMEOUT, max_duration=LISTEN_WINDOW):
226    t_start = time.monotonic()
227
228    final_text = ""
229    last_partial = ""
230    last_event = time.monotonic()
231    first_chunk_at = None
232    loop_end_at = None
233    cancel_sent_at = None
234    stop_watchdog = threading.Event()
235    got_enough = threading.Event()
236
237    def watchdog():
238        nonlocal cancel_sent_at
239        while not stop_watchdog.is_set():
240            time.sleep(0.2)
241            if (final_text or last_partial) and (time.monotonic() - last_event) > idle_timeout:
242                got_enough.set()
243                if loop_end_at is not None:
244                    log.info("DIAG watchdog: stream already ended at t+%.2fs, skipping cancel",
245                             loop_end_at - t_start)
246                    return
247                cancel_sent_at = time.monotonic()
248                log.info("DIAG watchdog: idle reached, sending cancel at t+%.2fs", cancel_sent_at - t_start)
249                asr.cancel()
250                return
251
252    def drain():
253        nonlocal final_text, last_partial, last_event, first_chunk_at, loop_end_at
254        try:
255            log.info("opening ASR session (max_duration=%ds)", max_duration)
256            with asr.transcribe_stream(duration=max_duration) as stream:
257                for chunk in stream:
258                    if not chunk.data.strip():
259                        continue
260                    if first_chunk_at is None:
261                        first_chunk_at = time.monotonic()
262                        log.info("first ASR chunk received after %.2fs", first_chunk_at - t_start)
263                    last_event = time.monotonic()
264                    match chunk.type:
265                        case "partial_text":
266                            last_partial = chunk.data
267                        case "full_text":
268                            final_text += chunk.data
269                            log.info("full_text segment at t+%.2fs: %r", time.monotonic() - t_start, chunk.data)
270                loop_end_at = time.monotonic()
271                log.info("DIAG stream loop ended at t+%.2fs (cancel_sent=%s)",
272                         loop_end_at - t_start,
273                         "no" if cancel_sent_at is None else "t+%.2fs" % (cancel_sent_at - t_start))
274            log.info("DIAG with-block exited (session closed) at t+%.2fs", time.monotonic() - t_start)
275        except ASRBusyError as e:
276            log.warning("ASR busy: %s", e)
277        except ASRServiceBusyError as e:
278            log.warning("ASR service busy: %s", e)
279        except ASRUnavailableError as e:
280            log.warning("ASR unavailable: %s", e)
281        except ASRError as e:
282            log.error("ASR error: %s", e)
283        except (ReadTimeout, ConnectionError) as e:
284            log.warning("ReadTimeout: %s", e)
285        except RuntimeError as e:
286            log.error("RuntimeError: %s", e)
287        except Exception as e:
288            log.exception("Unexpected exception during ASR: %s", e)
289        finally:
290            stop_watchdog.set()
291            got_enough.set()
292            log.info("drain thread finished at t+%.2fs (session closed)", time.monotonic() - t_start)
293
294    threading.Thread(target=watchdog, daemon=True, name="watchdog").start()
295    threading.Thread(target=drain, daemon=True, name="asr_drain").start()
296
297    got_enough.wait()
298
299    if not final_text and last_partial:
300        final_text = last_partial
301
302    log.info("speech_recognition() returning after %.2fs: %r", time.monotonic() - t_start, final_text)
303    return final_text
304
305
306# ------------------------------------------------------------ presence check
307def person_present() -> bool:
308    try:
309        frame = camera.capture()
310        if frame is None:
311            return False
312        h, w = frame.shape[:2]
313        out = object_detection.detect(compress_to_jpeg(frame).tobytes())
314    except Exception as e:
315        log.warning("detection failed: %s", e)
316        return False
317
318    if not S["debug_logged"]:
319        log.info("RAW detect() output: %r", out)
320        S["debug_logged"] = True
321
322    dets = out.get("detection", []) if isinstance(out, dict) else (out or [])
323    for d in dets:
324        if not isinstance(d, dict):
325            continue
326        name = str(d.get("class_name") or d.get("label") or d.get("name") or "").lower()
327        if name not in WAKE_CLASSES:
328            continue
329        box = d.get("bounding_box_xyxy") or d.get("bbox") or d.get("box")
330        if box and len(box) == 4:
331            x1, y1, x2, y2 = box
332            area = max(0.0, x2 - x1) * max(0.0, y2 - y1)
333            frac = area if max(box) <= 1.5 else area / float(w * h)
334            if frac < MIN_BOX_FRAC:
335                continue
336        return True
337    return False
338
339
340# ----------------------------------------------------------------- modes
341def idle_step():
342    if time.monotonic() < S["cooldown_until"]:
343        time.sleep(POLL_INTERVAL)
344        return
345
346    S["hits"] = S["hits"] + 1 if person_present() else 0
347    if S["hits"] < WAKE_HITS:
348        time.sleep(POLL_INTERVAL)
349        return
350
351    log.info(">>> person detected: conversation started")
352    S.update(mode="CONVERSATION", hits=0, last_seen=time.monotonic())
353    light("replying")
354    say_cached(random.choice(GREETINGS))
355
356
357def still_here() -> bool:
358    if person_present():
359        S["last_seen"] = time.monotonic()
360        return True
361    return time.monotonic() - S["last_seen"] <= LEAVE_AFTER
362
363
364def end_conversation():
365    log.info("<<< nobody in frame: conversation ended")
366    light("replying")
367    say_cached(GOODBYE)
368    S.update(mode="IDLE", hits=0, cooldown_until=time.monotonic() + COOLDOWN)
369    light("idle")
370
371
372def conversation_step():
373    t_loop = time.monotonic()
374
375    if not wait_asr_idle():
376        log.warning("previous ASR session still not closed after %.0fs", ASR_CLOSE_WAIT)
377
378    mic.start()
379    light("idle")   # candle flicker = session still opening, don't speak yet
380    ready_timer = threading.Timer(SESSION_READY_DELAY, light, args=("listening",))
381    ready_timer.start()   # solid = speak now
382    log.info("--- Start talking... ---")
383    prompt = speech_recognition()
384    ready_timer.cancel()
385    mic.stop()
386
387    log.info("prompt: %r (t+%.2fs since loop start)", prompt, time.monotonic() - t_loop)
388
389    if prompt:
390        S["last_seen"] = time.monotonic()
391    else:
392        if not still_here():
393            end_conversation()
394        return
395
396    if FILLER_ENABLED:
397        filler = next(filler_cycle)
398        text_queue.put(filler)
399        log.info("queued filler stall line: %r", filler)
400    play_filler_async()
401    light("thinking")
402
403    t_cam = time.monotonic()
404    frame = camera.capture()
405    if frame is None:
406        wait_tts()
407        return
408
409    frame = resize(frame, (224, 224))
410    frame = compress_to_jpeg(frame)
411    image = frame.tobytes()
412    log.info("camera.capture() took %.2fs", time.monotonic() - t_cam)
413
414    text = ""
415    t_vlm = time.monotonic()
416    first_token_at = None
417    replying = False
418
419    for chunk in vlm.chat_stream(prompt, images=[image]):
420        if first_token_at is None:
421            first_token_at = time.monotonic()
422            log.info("VLM first token after %.2fs", first_token_at - t_vlm)
423
424        text += chunk
425        if text and text[-1] in ".!?" and len(text.strip()) >= MIN_TTS_CHARS:
426            if not replying:
427                light("replying")
428                replying = True
429            text_queue.put(text)
430            log.info("queued TTS chunk: %r (t+%.2fs since VLM start)", text, time.monotonic() - t_vlm)
431            text = ""
432    if text.strip():
433        if not replying:
434            light("replying")
435            replying = True
436        text_queue.put(text)
437        log.info("queued final TTS chunk: %r", text)
438
439    log.info("VLM stream complete after %.2fs total", time.monotonic() - t_vlm)
440
441    t_join = time.monotonic()
442    wait_tts()
443    log.info("TTS wait took %.2fs", time.monotonic() - t_join)
444
445    if not still_here():
446        end_conversation()
447        return
448
449    log.info("=== loop() total: %.2fs ===", time.monotonic() - t_loop)
450
451
452def loop():
453    if S["mode"] == "IDLE":
454        idle_step()
455    else:
456        conversation_step()
457
458
459camera = mic = asr = tts = speaker = None
460try:
461    camera = Camera(fps=30, resolution=(640, 480), adjustments=rotated(90) | cropped(width=480, height=480, x=0, y=0) | resized(target_size=(320, 320)))
462    camera.start()
463
464    object_detection = ObjectDetection(confidence=0.6)
465
466    mic = Microphone("plughw:CARD=U0x46d0x825,DEV=0")
467    asr = AutomaticSpeechRecognition(mic, language="en")
468    asr.start()
469    text_queue = queue.Queue()
470
471    speaker = Speaker("plughw:CARD=Audio,DEV=0")
472    tts = TextToSpeech(speaker)
473
474    # Render cached lines in the background while the VLM warms up (different services).
475    cache_thread = threading.Thread(target=prerender_all, daemon=True, name="prerender")
476    cache_thread.start()
477
478    system_prompt = """
479    You are a sentient, eerie, glowing Jack-o'-Lantern carved on All Hallows' Eve. Speak with a spooky, theatrical, and wicked persona.
480    Keep answers to ONE short sentence, under 10 words. End every sentence with a period, exclamation point, or question mark — never trail off with ellipses or dashes.
481    """
482    vlm = VisionLanguageModel(system_prompt=system_prompt)
483
484    t_warmup = time.monotonic()
485    dummy_frame = compress_to_jpeg(np.zeros((1, 1, 3), dtype=np.uint8))
486    vlm.chat("Describe this image in exactly one word.", images=[dummy_frame.tobytes()])
487    log.info("VLM warmup took %.2fs", time.monotonic() - t_warmup)
488
489    cache_thread.join(timeout=60)
490    if cache_thread.is_alive():
491        log.warning("prerender still running after warmup; continuing (missing lines fall back to live TTS)")
492
493    th = threading.Thread(target=tts_thread, args=(tts, text_queue), daemon=True, name="tts_thread")
494    th.start()
495
496    ui = WebUI()
497    ui.expose_camera("/camera", camera)
498
499    light("idle")
500    App.run(user_loop=loop)
501except MicrophoneOpenError as e:
502    log.error("MicrophoneOpenError: %s", e)
503except Exception as e:
504    log.exception("Exception: %s", e)
505finally:
506    light("off")
507    if camera:
508        camera.stop()
509    if mic:
510        mic.stop()
511    if asr:
512        asr.stop()
513    if tts:
514        tts.stop()

1#include "Arduino_RouterBridge.h"
2
3#define FLASH_EN_PIN 6
4
5enum PumpkinState {
6  STATE_OFF,        
7  STATE_IDLE,       // Spooky candle flicker
8  STATE_LISTENING,  // Solid bright glow
9  STATE_THINKING,   // Smooth sine-wave breathing
10  STATE_REPLYING    // Speech cadence fluctuation
11};
12
13PumpkinState currentState = STATE_IDLE;
14unsigned long lastUpdate = 0;
15
16// 32-step sine wave lookup table for smooth breathing (0 to 255)
17const uint8_t sineBreathingTable[32] = {
18    0,   2,   9,  20,  36,  56,  80, 107,
19  135, 163, 189, 212, 230, 244, 252, 255,
20  255, 252, 244, 230, 212, 189, 163, 135,
21  107,  80,  56,  36,  20,   9,   2,   0
22};
23
24uint8_t sineIndex = 0;
25
26void switchState(PumpkinState newState);
27void updatePumpkinLighting();
28
29// Bridge RPC Provider Functions
30void set_state_off()       { switchState(STATE_OFF); }
31void set_state_idle()      { switchState(STATE_IDLE); }
32void set_state_listening() { switchState(STATE_LISTENING); }
33void set_state_thinking()  { switchState(STATE_THINKING); }
34void set_state_replying()  { switchState(STATE_REPLYING); }
35
36void setup() {
37  pinMode(FLASH_EN_PIN, OUTPUT);
38
39  // Initialize Router Bridge and expose state change endpoints
40  Bridge.begin();
41  Bridge.provide("set_state_off", set_state_off);
42  Bridge.provide("set_state_idle", set_state_idle);
43  Bridge.provide("set_state_listening", set_state_listening);
44  Bridge.provide("set_state_thinking", set_state_thinking);
45  Bridge.provide("set_state_replying", set_state_replying);
46
47  switchState(STATE_OFF);
48}
49
50void loop() {
51  updatePumpkinLighting();
52}
53
54void switchState(PumpkinState newState) {
55  currentState = newState;
56  lastUpdate = millis();
57
58  switch (currentState) {
59    case STATE_OFF:
60      analogWrite(FLASH_EN_PIN, 0);
61      break;
62
63    case STATE_IDLE:
64      break;
65
66    case STATE_LISTENING:
67      analogWrite(FLASH_EN_PIN, 255); // Peak brightness
68      break;
69
70    case STATE_THINKING:
71      sineIndex = 0;
72      break;
73
74    case STATE_REPLYING:
75      break;
76  }
77}
78
79void updatePumpkinLighting() {
80  unsigned long currentMillis = millis();
81
82  switch (currentState) {
83    case STATE_OFF:
84      break;
85
86    case STATE_IDLE:
87      if (currentMillis - lastUpdate >= random(60, 200)) {
88        lastUpdate = currentMillis;
89        uint8_t flickerVal = (random(0, 10) > 7) ? random(10, 60) : random(100, 190);
90        analogWrite(FLASH_EN_PIN, flickerVal);
91      }
92      break;
93
94    case STATE_LISTENING:
95      break;
96
97    case STATE_THINKING:
98      if (currentMillis - lastUpdate >= 80) {
99        lastUpdate = currentMillis;
100        analogWrite(FLASH_EN_PIN, sineBreathingTable[sineIndex]);
101        sineIndex = (sineIndex + 1) % 32;
102      }
103      break;
104
105    case STATE_REPLYING:
106      if (currentMillis - lastUpdate >= random(20, 50)) {
107        lastUpdate = currentMillis;
108        analogWrite(FLASH_EN_PIN, random(20, 255));
109      }
110      break;
111  }
112}