Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Add laughter/reaction detection channel (YAMNet ONNX, no-torch)

Nika Siradze379e7db7bdf8
diff --git a/backend/cli.py b/backend/cli.py
index f5c3f78..5d21292 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -395,6 +395,7 @@ def cmd_process(args):
from services.clip_generator import generate_clip
from services.transcript_parser import parse_speaker_transcript
from services.audio_analyzer import get_energy_profile
+ from services.audio_events import get_event_profile, is_available as audio_events_available
from services.encoder import get_encoder_info
from presets import get_preset, DEFAULT_PRESET, MIN_CLIP_DURATION, MAX_CLIP_DURATION, TARGET_CLIP_DURATION_MIN, TARGET_CLIP_DURATION_MAX
@@ -657,6 +658,7 @@ def cmd_process(args):
# ── Step 2: Analyze audio energy ──
energy_scores = None
+ reaction_scores = None
if config.get("energy_boost", True):
print(" [2/4] Analyzing audio energy...")
try:
@@ -665,6 +667,15 @@ def cmd_process(args):
print(f" {len(profile['peak_times'])} peak moments found")
except Exception as e:
print(f" Skipped (error: {e})")
+ if audio_events_available():
+ try:
+ reactions = get_event_profile(video_path, segments)
+ reaction_scores = reactions["segment_scores"]
+ n = len(reactions["reaction_times"])
+ if n:
+ print(f" {n} laughter/reaction moments found")
+ except Exception as e:
+ print(f" Reactions skipped (error: {e})")
else:
print(" [2/4] Audio analysis skipped (--no-energy)")
@@ -712,6 +723,7 @@ def cmd_process(args):
clips = _suggest_clips(
segments=segments,
energy_scores=energy_scores,
+ reaction_scores=reaction_scores,
top_n=top_n,
min_dur=config.get("min_clip_duration", MIN_CLIP_DURATION),
max_dur=config.get("max_clip_duration", MAX_CLIP_DURATION),
@@ -1607,6 +1619,7 @@ def _post_render_loop(
def _suggest_clips(
segments: list,
energy_scores: list | None = None,
+ reaction_scores: list | None = None,
top_n: int = 5,
min_dur: float = MIN_CLIP_DURATION,
max_dur: float = MAX_CLIP_DURATION,
@@ -1804,6 +1817,15 @@ def _suggest_clips(
if max_e > 7:
reasons.append("high_energy")
+ # ── 6b. Laughter / reactions (0-6 pts) ──
+ if reaction_scores:
+ seg_reactions = reaction_scores[snap_start : snap_end + 1]
+ if seg_reactions:
+ max_r = max(seg_reactions)
+ score += min(max_r, 6)
+ if max_r > 3:
+ reasons.append("laughter")
+
# ── 7. Density check — penalize sparse/rambling segments ──
words_per_sec = len(text.split()) / max(dur, 1)
if words_per_sec < 1.5:
diff --git a/backend/main.py b/backend/main.py
index 468669a..9355357 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -87,11 +87,19 @@ def handle_transcribe(task_id: str, params: dict):
except Exception:
pass # energy is a nice-to-have
+ events_data = None
+ try:
+ from services.audio_events import extract_audio_events
+ events_data = extract_audio_events(file_path)
+ except Exception:
+ pass # reactions are a nice-to-have
+
packed_path, packed_md = write_packed(
result,
cache_hash,
source_label=os.path.basename(file_path),
energy_data=energy_data,
+ events_data=events_data,
)
result["packed_path"] = packed_path
result["packed_size_bytes"] = len(packed_md.encode("utf-8"))
@@ -253,18 +261,27 @@ def handle_pack_transcript(task_id: str, params: dict):
return
energy_data = params.get("energy_data")
- if energy_data is None and params.get("file_path"):
- try:
- from services.audio_analyzer import extract_audio_energy
- energy_data = extract_audio_energy(params["file_path"])
- except Exception:
- pass
+ events_data = params.get("events_data")
+ if params.get("file_path"):
+ if energy_data is None:
+ try:
+ from services.audio_analyzer import extract_audio_energy
+ energy_data = extract_audio_energy(params["file_path"])
+ except Exception:
+ pass
+ if events_data is None:
+ try:
+ from services.audio_events import extract_audio_events
+ events_data = extract_audio_events(params["file_path"])
+ except Exception:
+ pass
path, md = write_packed(
transcript,
cache_hash,
source_label=params.get("source_label"),
energy_data=energy_data,
+ events_data=events_data,
)
emit_result(task_id, "success", data={
"packed_path": path,
diff --git a/backend/models/yamnet.onnx b/backend/models/yamnet.onnx
new file mode 100644
index 0000000..a799a1f
Binary files /dev/null and b/backend/models/yamnet.onnx differ
diff --git a/backend/models/yamnet_class_map.csv b/backend/models/yamnet_class_map.csv
new file mode 100644
index 0000000..9c07d81
--- /dev/null
+++ b/backend/models/yamnet_class_map.csv
@@ -0,0 +1,522 @@
+index,mid,display_name
+0,/m/09x0r,Speech
+1,/m/0ytgt,"Child speech, kid speaking"
+2,/m/01h8n0,Conversation
+3,/m/02qldy,"Narration, monologue"
+4,/m/0261r1,Babbling
+5,/m/0brhx,Speech synthesizer
+6,/m/07p6fty,Shout
+7,/m/07q4ntr,Bellow
+8,/m/07rwj3x,Whoop
+9,/m/07sr1lc,Yell
+10,/t/dd00135,Children shouting
+11,/m/03qc9zr,Screaming
+12,/m/02rtxlg,Whispering
+13,/m/01j3sz,Laughter
+14,/t/dd00001,Baby laughter
+15,/m/07r660_,Giggle
+16,/m/07s04w4,Snicker
+17,/m/07sq110,Belly laugh
+18,/m/07rgt08,"Chuckle, chortle"
+19,/m/0463cq4,"Crying, sobbing"
+20,/t/dd00002,"Baby cry, infant cry"
+21,/m/07qz6j3,Whimper
+22,/m/07qw_06,"Wail, moan"
+23,/m/07plz5l,Sigh
+24,/m/015lz1,Singing
+25,/m/0l14jd,Choir
+26,/m/01swy6,Yodeling
+27,/m/02bk07,Chant
+28,/m/01c194,Mantra
+29,/t/dd00005,Child singing
+30,/t/dd00006,Synthetic singing
+31,/m/06bxc,Rapping
+32,/m/02fxyj,Humming
+33,/m/07s2xch,Groan
+34,/m/07r4k75,Grunt
+35,/m/01w250,Whistling
+36,/m/0lyf6,Breathing
+37,/m/07mzm6,Wheeze
+38,/m/01d3sd,Snoring
+39,/m/07s0dtb,Gasp
+40,/m/07pyy8b,Pant
+41,/m/07q0yl5,Snort
+42,/m/01b_21,Cough
+43,/m/0dl9sf8,Throat clearing
+44,/m/01hsr_,Sneeze
+45,/m/07ppn3j,Sniff
+46,/m/06h7j,Run
+47,/m/07qv_x_,Shuffle
+48,/m/07pbtc8,"Walk, footsteps"
+49,/m/03cczk,"Chewing, mastication"
+50,/m/07pdhp0,Biting
+51,/m/0939n_,Gargling
+52,/m/01g90h,Stomach rumble
+53,/m/03q5_w,"Burping, eructation"
+54,/m/02p3nc,Hiccup
+55,/m/02_nn,Fart
+56,/m/0k65p,Hands
+57,/m/025_jnm,Finger snapping
+58,/m/0l15bq,Clapping
+59,/m/01jg02,"Heart sounds, heartbeat"
+60,/m/01jg1z,Heart murmur
+61,/m/053hz1,Cheering
+62,/m/028ght,Applause
+63,/m/07rkbfh,Chatter
+64,/m/03qtwd,Crowd
+65,/m/07qfr4h,"Hubbub, speech noise, speech babble"
+66,/t/dd00013,Children playing
+67,/m/0jbk,Animal
+68,/m/068hy,"Domestic animals, pets"
+69,/m/0bt9lr,Dog
+70,/m/05tny_,Bark
+71,/m/07r_k2n,Yip
+72,/m/07qf0zm,Howl
+73,/m/07rc7d9,Bow-wow
+74,/m/0ghcn6,Growling
+75,/t/dd00136,Whimper (dog)
+76,/m/01yrx,Cat
+77,/m/02yds9,Purr
+78,/m/07qrkrw,Meow
+79,/m/07rjwbb,Hiss
+80,/m/07r81j2,Caterwaul
+81,/m/0ch8v,"Livestock, farm animals, working animals"
+82,/m/03k3r,Horse
+83,/m/07rv9rh,Clip-clop
+84,/m/07q5rw0,"Neigh, whinny"
+85,/m/01xq0k1,"Cattle, bovinae"
+86,/m/07rpkh9,Moo
+87,/m/0239kh,Cowbell
+88,/m/068zj,Pig
+89,/t/dd00018,Oink
+90,/m/03fwl,Goat
+91,/m/07q0h5t,Bleat
+92,/m/07bgp,Sheep
+93,/m/025rv6n,Fowl
+94,/m/09b5t,"Chicken, rooster"
+95,/m/07st89h,Cluck
+96,/m/07qn5dc,"Crowing, cock-a-doodle-doo"
+97,/m/01rd7k,Turkey
+98,/m/07svc2k,Gobble
+99,/m/09ddx,Duck
+100,/m/07qdb04,Quack
+101,/m/0dbvp,Goose
+102,/m/07qwf61,Honk
+103,/m/01280g,Wild animals
+104,/m/0cdnk,"Roaring cats (lions, tigers)"
+105,/m/04cvmfc,Roar
+106,/m/015p6,Bird
+107,/m/020bb7,"Bird vocalization, bird call, bird song"
+108,/m/07pggtn,"Chirp, tweet"
+109,/m/07sx8x_,Squawk
+110,/m/0h0rv,"Pigeon, dove"
+111,/m/07r_25d,Coo
+112,/m/04s8yn,Crow
+113,/m/07r5c2p,Caw
+114,/m/09d5_,Owl
+115,/m/07r_80w,Hoot
+116,/m/05_wcq,"Bird flight, flapping wings"
+117,/m/01z5f,"Canidae, dogs, wolves"
+118,/m/06hps,"Rodents, rats, mice"
+119,/m/04rmv,Mouse
+120,/m/07r4gkf,Patter
+121,/m/03vt0,Insect
+122,/m/09xqv,Cricket
+123,/m/09f96,Mosquito
+124,/m/0h2mp,"Fly, housefly"
+125,/m/07pjwq1,Buzz
+126,/m/01h3n,"Bee, wasp, etc."
+127,/m/09ld4,Frog
+128,/m/07st88b,Croak
+129,/m/078jl,Snake
+130,/m/07qn4z3,Rattle
+131,/m/032n05,Whale vocalization
+132,/m/04rlf,Music
+133,/m/04szw,Musical instrument
+134,/m/0fx80y,Plucked string instrument
+135,/m/0342h,Guitar
+136,/m/02sgy,Electric guitar
+137,/m/018vs,Bass guitar
+138,/m/042v_gx,Acoustic guitar
+139,/m/06w87,"Steel guitar, slide guitar"
+140,/m/01glhc,Tapping (guitar technique)
+141,/m/07s0s5r,Strum
+142,/m/018j2,Banjo
+143,/m/0jtg0,Sitar
+144,/m/04rzd,Mandolin
+145,/m/01bns_,Zither
+146,/m/07xzm,Ukulele
+147,/m/05148p4,Keyboard (musical)
+148,/m/05r5c,Piano
+149,/m/01s0ps,Electric piano
+150,/m/013y1f,Organ
+151,/m/03xq_f,Electronic organ
+152,/m/03gvt,Hammond organ
+153,/m/0l14qv,Synthesizer
+154,/m/01v1d8,Sampler
+155,/m/03q5t,Harpsichord
+156,/m/0l14md,Percussion
+157,/m/02hnl,Drum kit
+158,/m/0cfdd,Drum machine
+159,/m/026t6,Drum
+160,/m/06rvn,Snare drum
+161,/m/03t3fj,Rimshot
+162,/m/02k_mr,Drum roll
+163,/m/0bm02,Bass drum
+164,/m/011k_j,Timpani
+165,/m/01p970,Tabla
+166,/m/01qbl,Cymbal
+167,/m/03qtq,Hi-hat
+168,/m/01sm1g,Wood block
+169,/m/07brj,Tambourine
+170,/m/05r5wn,Rattle (instrument)
+171,/m/0xzly,Maraca
+172,/m/0mbct,Gong
+173,/m/016622,Tubular bells
+174,/m/0j45pbj,Mallet percussion
+175,/m/0dwsp,"Marimba, xylophone"
+176,/m/0dwtp,Glockenspiel
+177,/m/0dwt5,Vibraphone
+178,/m/0l156b,Steelpan
+179,/m/05pd6,Orchestra
+180,/m/01kcd,Brass instrument
+181,/m/0319l,French horn
+182,/m/07gql,Trumpet
+183,/m/07c6l,Trombone
+184,/m/0l14_3,Bowed string instrument
+185,/m/02qmj0d,String section
+186,/m/07y_7,"Violin, fiddle"
+187,/m/0d8_n,Pizzicato
+188,/m/01xqw,Cello
+189,/m/02fsn,Double bass
+190,/m/085jw,"Wind instrument, woodwind instrument"
+191,/m/0l14j_,Flute
+192,/m/06ncr,Saxophone
+193,/m/01wy6,Clarinet
+194,/m/03m5k,Harp
+195,/m/0395lw,Bell
+196,/m/03w41f,Church bell
+197,/m/027m70_,Jingle bell
+198,/m/0gy1t2s,Bicycle bell
+199,/m/07n_g,Tuning fork
+200,/m/0f8s22,Chime
+201,/m/026fgl,Wind chime
+202,/m/0150b9,Change ringing (campanology)
+203,/m/03qjg,Harmonica
+204,/m/0mkg,Accordion
+205,/m/0192l,Bagpipes
+206,/m/02bxd,Didgeridoo
+207,/m/0l14l2,Shofar
+208,/m/07kc_,Theremin
+209,/m/0l14t7,Singing bowl
+210,/m/01hgjl,Scratching (performance technique)
+211,/m/064t9,Pop music
+212,/m/0glt670,Hip hop music
+213,/m/02cz_7,Beatboxing
+214,/m/06by7,Rock music
+215,/m/03lty,Heavy metal
+216,/m/05r6t,Punk rock
+217,/m/0dls3,Grunge
+218,/m/0dl5d,Progressive rock
+219,/m/07sbbz2,Rock and roll
+220,/m/05w3f,Psychedelic rock
+221,/m/06j6l,Rhythm and blues
+222,/m/0gywn,Soul music
+223,/m/06cqb,Reggae
+224,/m/01lyv,Country
+225,/m/015y_n,Swing music
+226,/m/0gg8l,Bluegrass
+227,/m/02x8m,Funk
+228,/m/02w4v,Folk music
+229,/m/06j64v,Middle Eastern music
+230,/m/03_d0,Jazz
+231,/m/026z9,Disco
+232,/m/0ggq0m,Classical music
+233,/m/05lls,Opera
+234,/m/02lkt,Electronic music
+235,/m/03mb9,House music
+236,/m/07gxw,Techno
+237,/m/07s72n,Dubstep
+238,/m/0283d,Drum and bass
+239,/m/0m0jc,Electronica
+240,/m/08cyft,Electronic dance music
+241,/m/0fd3y,Ambient music
+242,/m/07lnk,Trance music
+243,/m/0g293,Music of Latin America
+244,/m/0ln16,Salsa music
+245,/m/0326g,Flamenco
+246,/m/0155w,Blues
+247,/m/05fw6t,Music for children
+248,/m/02v2lh,New-age music
+249,/m/0y4f8,Vocal music
+250,/m/0z9c,A capella
+251,/m/0164x2,Music of Africa
+252,/m/0145m,Afrobeat
+253,/m/02mscn,Christian music
+254,/m/016cjb,Gospel music
+255,/m/028sqc,Music of Asia
+256,/m/015vgc,Carnatic music
+257,/m/0dq0md,Music of Bollywood
+258,/m/06rqw,Ska
+259,/m/02p0sh1,Traditional music
+260,/m/05rwpb,Independent music
+261,/m/074ft,Song
+262,/m/025td0t,Background music
+263,/m/02cjck,Theme music
+264,/m/03r5q_,Jingle (music)
+265,/m/0l14gg,Soundtrack music
+266,/m/07pkxdp,Lullaby
+267,/m/01z7dr,Video game music
+268,/m/0140xf,Christmas music
+269,/m/0ggx5q,Dance music
+270,/m/04wptg,Wedding music
+271,/t/dd00031,Happy music
+272,/t/dd00033,Sad music
+273,/t/dd00034,Tender music
+274,/t/dd00035,Exciting music
+275,/t/dd00036,Angry music
+276,/t/dd00037,Scary music
+277,/m/03m9d0z,Wind
+278,/m/09t49,Rustling leaves
+279,/t/dd00092,Wind noise (microphone)
+280,/m/0jb2l,Thunderstorm
+281,/m/0ngt1,Thunder
+282,/m/0838f,Water
+283,/m/06mb1,Rain
+284,/m/07r10fb,Raindrop
+285,/t/dd00038,Rain on surface
+286,/m/0j6m2,Stream
+287,/m/0j2kx,Waterfall
+288,/m/05kq4,Ocean
+289,/m/034srq,"Waves, surf"
+290,/m/06wzb,Steam
+291,/m/07swgks,Gurgling
+292,/m/02_41,Fire
+293,/m/07pzfmf,Crackle
+294,/m/07yv9,Vehicle
+295,/m/019jd,"Boat, Water vehicle"
+296,/m/0hsrw,"Sailboat, sailing ship"
+297,/m/056ks2,"Rowboat, canoe, kayak"
+298,/m/02rlv9,"Motorboat, speedboat"
+299,/m/06q74,Ship
+300,/m/012f08,Motor vehicle (road)
+301,/m/0k4j,Car
+302,/m/0912c9,"Vehicle horn, car horn, honking"
+303,/m/07qv_d5,Toot
+304,/m/02mfyn,Car alarm
+305,/m/04gxbd,"Power windows, electric windows"
+306,/m/07rknqz,Skidding
+307,/m/0h9mv,Tire squeal
+308,/t/dd00134,Car passing by
+309,/m/0ltv,"Race car, auto racing"
+310,/m/07r04,Truck
+311,/m/0gvgw0,Air brake
+312,/m/05x_td,"Air horn, truck horn"
+313,/m/02rhddq,Reversing beeps
+314,/m/03cl9h,"Ice cream truck, ice cream van"
+315,/m/01bjv,Bus
+316,/m/03j1ly,Emergency vehicle
+317,/m/04qvtq,Police car (siren)
+318,/m/012n7d,Ambulance (siren)
+319,/m/012ndj,"Fire engine, fire truck (siren)"
+320,/m/04_sv,Motorcycle
+321,/m/0btp2,"Traffic noise, roadway noise"
+322,/m/06d_3,Rail transport
+323,/m/07jdr,Train
+324,/m/04zmvq,Train whistle
+325,/m/0284vy3,Train horn
+326,/m/01g50p,"Railroad car, train wagon"
+327,/t/dd00048,Train wheels squealing
+328,/m/0195fx,"Subway, metro, underground"
+329,/m/0k5j,Aircraft
+330,/m/014yck,Aircraft engine
+331,/m/04229,Jet engine
+332,/m/02l6bg,"Propeller, airscrew"
+333,/m/09ct_,Helicopter
+334,/m/0cmf2,"Fixed-wing aircraft, airplane"
+335,/m/0199g,Bicycle
+336,/m/06_fw,Skateboard
+337,/m/02mk9,Engine
+338,/t/dd00065,Light engine (high frequency)
+339,/m/08j51y,"Dental drill, dentist's drill"
+340,/m/01yg9g,Lawn mower
+341,/m/01j4z9,Chainsaw
+342,/t/dd00066,Medium engine (mid frequency)
+343,/t/dd00067,Heavy engine (low frequency)
+344,/m/01h82_,Engine knocking
+345,/t/dd00130,Engine starting
+346,/m/07pb8fc,Idling
+347,/m/07q2z82,"Accelerating, revving, vroom"
+348,/m/02dgv,Door
+349,/m/03wwcy,Doorbell
+350,/m/07r67yg,Ding-dong
+351,/m/02y_763,Sliding door
+352,/m/07rjzl8,Slam
+353,/m/07r4wb8,Knock
+354,/m/07qcpgn,Tap
+355,/m/07q6cd_,Squeak
+356,/m/0642b4,Cupboard open or close
+357,/m/0fqfqc,Drawer open or close
+358,/m/04brg2,"Dishes, pots, and pans"
+359,/m/023pjk,"Cutlery, silverware"
+360,/m/07pn_8q,Chopping (food)
+361,/m/0dxrf,Frying (food)
+362,/m/0fx9l,Microwave oven
+363,/m/02pjr4,Blender
+364,/m/02jz0l,"Water tap, faucet"
+365,/m/0130jx,Sink (filling or washing)
+366,/m/03dnzn,Bathtub (filling or washing)
+367,/m/03wvsk,Hair dryer
+368,/m/01jt3m,Toilet flush
+369,/m/012xff,Toothbrush
+370,/m/04fgwm,Electric toothbrush
+371,/m/0d31p,Vacuum cleaner
+372,/m/01s0vc,Zipper (clothing)
+373,/m/03v3yw,Keys jangling
+374,/m/0242l,Coin (dropping)
+375,/m/01lsmm,Scissors
+376,/m/02g901,"Electric shaver, electric razor"
+377,/m/05rj2,Shuffling cards
+378,/m/0316dw,Typing
+379,/m/0c2wf,Typewriter
+380,/m/01m2v,Computer keyboard
+381,/m/081rb,Writing
+382,/m/07pp_mv,Alarm
+383,/m/07cx4,Telephone
+384,/m/07pp8cl,Telephone bell ringing
+385,/m/01hnzm,Ringtone
+386,/m/02c8p,"Telephone dialing, DTMF"
+387,/m/015jpf,Dial tone
+388,/m/01z47d,Busy signal
+389,/m/046dlr,Alarm clock
+390,/m/03kmc9,Siren
+391,/m/0dgbq,Civil defense siren
+392,/m/030rvx,Buzzer
+393,/m/01y3hg,"Smoke detector, smoke alarm"
+394,/m/0c3f7m,Fire alarm
+395,/m/04fq5q,Foghorn
+396,/m/0l156k,Whistle
+397,/m/06hck5,Steam whistle
+398,/t/dd00077,Mechanisms
+399,/m/02bm9n,"Ratchet, pawl"
+400,/m/01x3z,Clock
+401,/m/07qjznt,Tick
+402,/m/07qjznl,Tick-tock
+403,/m/0l7xg,Gears
+404,/m/05zc1,Pulleys
+405,/m/0llzx,Sewing machine
+406,/m/02x984l,Mechanical fan
+407,/m/025wky1,Air conditioning
+408,/m/024dl,Cash register
+409,/m/01m4t,Printer
+410,/m/0dv5r,Camera
+411,/m/07bjf,Single-lens reflex camera
+412,/m/07k1x,Tools
+413,/m/03l9g,Hammer
+414,/m/03p19w,Jackhammer
+415,/m/01b82r,Sawing
+416,/m/02p01q,Filing (rasp)
+417,/m/023vsd,Sanding
+418,/m/0_ksk,Power tool
+419,/m/01d380,Drill
+420,/m/014zdl,Explosion
+421,/m/032s66,"Gunshot, gunfire"
+422,/m/04zjc,Machine gun
+423,/m/02z32qm,Fusillade
+424,/m/0_1c,Artillery fire
+425,/m/073cg4,Cap gun
+426,/m/0g6b5,Fireworks
+427,/g/122z_qxw,Firecracker
+428,/m/07qsvvw,"Burst, pop"
+429,/m/07pxg6y,Eruption
+430,/m/07qqyl4,Boom
+431,/m/083vt,Wood
+432,/m/07pczhz,Chop
+433,/m/07pl1bw,Splinter
+434,/m/07qs1cx,Crack
+435,/m/039jq,Glass
+436,/m/07q7njn,"Chink, clink"
+437,/m/07rn7sz,Shatter
+438,/m/04k94,Liquid
+439,/m/07rrlb6,"Splash, splatter"
+440,/m/07p6mqd,Slosh
+441,/m/07qlwh6,Squish
+442,/m/07r5v4s,Drip
+443,/m/07prgkl,Pour
+444,/m/07pqc89,"Trickle, dribble"
+445,/t/dd00088,Gush
+446,/m/07p7b8y,Fill (with liquid)
+447,/m/07qlf79,Spray
+448,/m/07ptzwd,Pump (liquid)
+449,/m/07ptfmf,Stir
+450,/m/0dv3j,Boiling
+451,/m/0790c,Sonar
+452,/m/0dl83,Arrow
+453,/m/07rqsjt,"Whoosh, swoosh, swish"
+454,/m/07qnq_y,"Thump, thud"
+455,/m/07rrh0c,Thunk
+456,/m/0b_fwt,Electronic tuner
+457,/m/02rr_,Effects unit
+458,/m/07m2kt,Chorus effect
+459,/m/018w8,Basketball bounce
+460,/m/07pws3f,Bang
+461,/m/07ryjzk,"Slap, smack"
+462,/m/07rdhzs,"Whack, thwack"
+463,/m/07pjjrj,"Smash, crash"
+464,/m/07pc8lb,Breaking
+465,/m/07pqn27,Bouncing
+466,/m/07rbp7_,Whip
+467,/m/07pyf11,Flap
+468,/m/07qb_dv,Scratch
+469,/m/07qv4k0,Scrape
+470,/m/07pdjhy,Rub
+471,/m/07s8j8t,Roll
+472,/m/07plct2,Crushing
+473,/t/dd00112,"Crumpling, crinkling"
+474,/m/07qcx4z,Tearing
+475,/m/02fs_r,"Beep, bleep"
+476,/m/07qwdck,Ping
+477,/m/07phxs1,Ding
+478,/m/07rv4dm,Clang
+479,/m/07s02z0,Squeal
+480,/m/07qh7jl,Creak
+481,/m/07qwyj0,Rustle
+482,/m/07s34ls,Whir
+483,/m/07qmpdm,Clatter
+484,/m/07p9k1k,Sizzle
+485,/m/07qc9xj,Clicking
+486,/m/07rwm0c,Clickety-clack
+487,/m/07phhsh,Rumble
+488,/m/07qyrcz,Plop
+489,/m/07qfgpx,"Jingle, tinkle"
+490,/m/07rcgpl,Hum
+491,/m/07p78v5,Zing
+492,/t/dd00121,Boing
+493,/m/07s12q4,Crunch
+494,/m/028v0c,Silence
+495,/m/01v_m0,Sine wave
+496,/m/0b9m1,Harmonic
+497,/m/0hdsk,Chirp tone
+498,/m/0c1dj,Sound effect
+499,/m/07pt_g0,Pulse
+500,/t/dd00125,"Inside, small room"
+501,/t/dd00126,"Inside, large room or hall"
+502,/t/dd00127,"Inside, public space"
+503,/t/dd00128,"Outside, urban or manmade"
+504,/t/dd00129,"Outside, rural or natural"
+505,/m/01b9nn,Reverberation
+506,/m/01jnbd,Echo
+507,/m/096m7z,Noise
+508,/m/06_y0by,Environmental noise
+509,/m/07rgkc5,Static
+510,/m/06xkwv,Mains hum
+511,/m/0g12c5,Distortion
+512,/m/08p9q4,Sidetone
+513,/m/07szfh9,Cacophony
+514,/m/0chx_,White noise
+515,/m/0cj0r,Pink noise
+516,/m/07p_0gm,Throbbing
+517,/m/01jwx6,Vibration
+518,/m/07c52,Television
+519,/m/06bz3,Radio
+520,/m/07hvw1,Field recording
diff --git a/backend/requirements-runtime.txt b/backend/requirements-runtime.txt
index 8112b0e..6e1b4f3 100644
--- a/backend/requirements-runtime.txt
+++ b/backend/requirements-runtime.txt
@@ -4,6 +4,9 @@
# via backend/requirements.txt.
opencv-python-headless>=4.8.1.78
numpy>=1.24.0
+# Audio-event detection (YAMNet laughter/reaction channel) runs on ONNX Runtime —
+# no torch/TF, so it stays on the hermetic native path.
+onnxruntime>=1.16.0
Pillow>=10.0.0
questionary>=2.0.0
python-dotenv>=1.2.2
diff --git a/backend/requirements.txt b/backend/requirements.txt
index bb6ae7b..7532eb6 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -12,6 +12,9 @@ openai-whisper>=20231117
opencv-python-headless>=4.8.1.78
numpy>=1.24.0
+# Audio-event detection (laughter/reaction channel via YAMNet ONNX — no torch/TF)
+onnxruntime>=1.16.0
+
# Thumbnails
Pillow>=10.0.0
diff --git a/backend/services/audio_events.py b/backend/services/audio_events.py
new file mode 100644
index 0000000..ce1a265
--- /dev/null
+++ b/backend/services/audio_events.py
@@ -0,0 +1,222 @@
+"""
+Audio-event detection (laughter, cheering, applause, screaming) for reaction-based
+clip scoring.
+
+Runs YAMNet (AudioSet, 521 classes) as a self-contained ONNX graph via onnxruntime,
+which imports neither torch nor tensorflow — so this belongs on the native /
+whisper.cpp runtime path. Audio is extracted at 16 kHz mono (the same format
+whisper.cpp uses) and fed to the graph as a raw waveform; the log-mel front-end is
+baked into the model, so there is no librosa dependency.
+
+A laugh is the strongest language-independent "something funny happened" signal, and
+it anti-correlates with speech at the frame level, which makes it a reliable anchor
+for extending a clip backwards to the moment that caused the reaction.
+"""
+
+import csv
+import subprocess
+import tempfile
+import wave
+from pathlib import Path
+from typing import Optional, Callable
+
+import numpy as np
+
+from utils.proc import run as proc_run
+
+try:
+ import onnxruntime as ort
+ _ORT_AVAILABLE = True
+except ImportError:
+ _ORT_AVAILABLE = False
+
+_MODELS_DIR = Path(__file__).resolve().parents[1] / "models"
+_MODEL_PATH = _MODELS_DIR / "yamnet.onnx"
+_CLASS_MAP_PATH = _MODELS_DIR / "yamnet_class_map.csv"
+
+# AudioSet display names, grouped into the reaction channels we score on. Laughter is
+# spread across fine-grained children in the ontology, so we collapse the whole family.
+_LAUGHTER_KEYS = ("laugh", "giggle", "chuckle", "snicker", "chortle")
+_CHEER_NAMES = ("Cheering", "Applause", "Whoop")
+_SCREAM_NAMES = ("Screaming",)
+_SPEECH_NAMES = ("Speech",)
+
+_session = None
+_class_index: dict[str, list[int]] = {}
+
+
+def _load_class_index() -> dict[str, list[int]]:
+ names: dict[int, str] = {}
+ with open(_CLASS_MAP_PATH, newline="") as f:
+ reader = csv.reader(f)
+ next(reader) # header: index,mid,display_name
+ for row in reader:
+ names[int(row[0])] = row[2]
+
+ def by_substr(keys) -> list[int]:
+ return [i for i, n in names.items() if any(k in n.lower() for k in keys)]
+
+ def by_name(wanted) -> list[int]:
+ return [i for i, n in names.items() if n in wanted]
+
+ return {
+ "laughter": by_substr(_LAUGHTER_KEYS),
+ "cheering": by_name(_CHEER_NAMES),
+ "screaming": by_name(_SCREAM_NAMES),
+ "speech": by_name(_SPEECH_NAMES),
+ }
+
+
+def _get_session():
+ global _session, _class_index
+ if not _ORT_AVAILABLE or not _MODEL_PATH.exists():
+ return None
+ if _session is None:
+ _session = ort.InferenceSession(
+ str(_MODEL_PATH), providers=["CPUExecutionProvider"]
+ )
+ _class_index = _load_class_index()
+ return _session
+
+
+def is_available() -> bool:
+ """True if onnxruntime and the YAMNet model are both present."""
+ return _ORT_AVAILABLE and _MODEL_PATH.exists()
+
+
+def _read_waveform_16k_mono(video_path: str) -> Optional[np.ndarray]:
+ """Extract audio as a float32 [-1, 1] mono waveform at 16 kHz via ffmpeg."""
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp:
+ cmd = [
+ "ffmpeg", "-y", "-i", video_path,
+ "-vn", "-ar", "16000", "-ac", "1",
+ "-f", "wav", tmp.name, "-loglevel", "error",
+ ]
+ result = proc_run(cmd, timeout=600, check=False)
+ if result.returncode != 0:
+ return None
+ with wave.open(tmp.name) as w:
+ frames = w.readframes(w.getnframes())
+ if not frames:
+ return None
+ return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
+
+
+def extract_audio_events(video_path: str) -> list[dict]:
+ """
+ Run YAMNet over a video's audio and return per-frame reaction probabilities.
+
+ Returns a list of {time, laughter, cheering, screaming, speech} dicts, one per
+ YAMNet frame (~0.48 s hop). Empty list if the model/runtime is unavailable or the
+ audio can't be read, so callers degrade gracefully.
+ """
+ session = _get_session()
+ if session is None:
+ return []
+
+ waveform = _read_waveform_16k_mono(video_path)
+ if waveform is None or waveform.size == 0:
+ return []
+
+ input_name = session.get_inputs()[0].name
+ scores = session.run(None, {input_name: waveform})[0] # [frames, 521]
+ n_frames = scores.shape[0]
+ if n_frames == 0:
+ return []
+
+ duration = waveform.size / 16000.0
+ hop = duration / n_frames
+
+ def channel_max(keys: list[int]) -> np.ndarray:
+ return scores[:, keys].max(axis=1) if keys else np.zeros(n_frames)
+
+ laughter = channel_max(_class_index["laughter"])
+ cheering = channel_max(_class_index["cheering"])
+ screaming = channel_max(_class_index["screaming"])
+ speech = channel_max(_class_index["speech"])
+
+ return [
+ {
+ "time": round(i * hop, 2),
+ "laughter": round(float(laughter[i]), 3),
+ "cheering": round(float(cheering[i]), 3),
+ "screaming": round(float(screaming[i]), 3),
+ "speech": round(float(speech[i]), 3),
+ }
+ for i in range(n_frames)
+ ]
+
+
+def _reaction_level(event: dict) -> float:
+ """Combined reaction strength for one frame: the loudest reaction channel."""
+ return max(event["laughter"], event["cheering"], event["screaming"])
+
+
+def compute_event_scores(
+ events_data: list[dict],
+ segments: list[dict],
+) -> list[float]:
+ """
+ Per-segment reaction score (0-10) from the peak reaction within each segment.
+
+ Unlike audio energy (RMS in dB, only meaningful relative to a video's own
+ baseline), YAMNet probabilities are absolute-calibrated in [0, 1], so the score is
+ a direct scaling of the peak rather than a z-score. A brief chuckle registers
+ ~0.25, a hearty laugh higher; peaks stand out sharply against a ~0 baseline.
+ """
+ if not events_data:
+ return [0.0] * len(segments)
+
+ scores = []
+ for seg in segments:
+ seg_start = seg.get("start", 0)
+ seg_end = seg.get("end", 0)
+ levels = [
+ _reaction_level(e)
+ for e in events_data
+ if seg_start <= e["time"] <= seg_end
+ ]
+ peak = max(levels) if levels else 0.0
+ scores.append(round(min(10.0, peak * 12.0), 2))
+
+ return scores
+
+
+def get_event_profile(
+ video_path: str,
+ segments: list[dict],
+ progress_callback: Optional[Callable] = None,
+ reaction_threshold: float = 0.15,
+) -> dict:
+ """
+ Full pipeline: detect audio events and score all segments.
+
+ Returns {events_data, segment_scores, reaction_times} where reaction_times are the
+ timestamps of frames whose reaction level clears reaction_threshold — the anchors a
+ reaction-based detector expands backwards from.
+ """
+ if not is_available():
+ return {"events_data": [], "segment_scores": [0.0] * len(segments), "reaction_times": []}
+
+ if progress_callback:
+ progress_callback(0, "Detecting laughter and reactions...")
+
+ events_data = extract_audio_events(video_path)
+
+ if progress_callback:
+ progress_callback(70, "Scoring segments by reaction...")
+
+ segment_scores = compute_event_scores(events_data, segments)
+
+ reaction_times = [
+ e["time"] for e in events_data if _reaction_level(e) >= reaction_threshold
+ ]
+
+ if progress_callback:
+ progress_callback(100, "Reaction analysis complete")
+
+ return {
+ "events_data": events_data,
+ "segment_scores": segment_scores,
+ "reaction_times": reaction_times,
+ }
diff --git a/backend/services/profiles.py b/backend/services/profiles.py
new file mode 100644
index 0000000..c5450da
--- /dev/null
+++ b/backend/services/profiles.py
@@ -0,0 +1,80 @@
+"""Content profiles — which signals drive moment detection, per content type.
+
+A ContentProfile is orthogonal to FormatSpec: FormatSpec is a render-time aspect-ratio
+decision, a ContentProfile is a detection-time decision about which signal channels
+matter and how candidate moments are generated. A podcast is selected transcript-first
+(the LLM reads dialogue); a party video has no useful transcript and is driven by
+laughter/energy peaks. Both are the same fusion engine with different channel weights.
+
+Weights are starting points, tuned with real footage. `candidate_source` picks how
+moments are generated: "llm" keeps the existing transcript-first selector (podcast),
+"saliency" peak-picks a fused signal curve (party/action, works with no transcript).
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ContentProfile:
+ name: str
+ candidate_source: str # "llm" | "saliency"
+ channel_weights: dict[str, float]
+ # Reaction-based expansion: anchor on a laugh/cheer onset and extend backwards.
+ reaction_lookback_sec: float
+ reaction_payoff_sec: float
+ # Peak-pick tuning for the saliency candidate source (per-video normalized units).
+ peak_min_gap_sec: float
+ peak_top_percentile: float
+
+
+_CHANNELS = ("transcript_semantic", "prosody", "audio_event", "energy", "motion", "face_reaction")
+
+
+def _weights(**kw) -> dict[str, float]:
+ return {ch: float(kw.get(ch, 0.0)) for ch in _CHANNELS}
+
+
+PROFILES = {
+ "podcast": ContentProfile(
+ name="podcast",
+ candidate_source="llm",
+ channel_weights=_weights(
+ transcript_semantic=0.5, prosody=0.2, audio_event=0.15, energy=0.1, face_reaction=0.05
+ ),
+ reaction_lookback_sec=8.0,
+ reaction_payoff_sec=2.0,
+ peak_min_gap_sec=15.0,
+ peak_top_percentile=0.15,
+ ),
+ "party": ContentProfile(
+ name="party",
+ candidate_source="saliency",
+ channel_weights=_weights(
+ audio_event=0.4, prosody=0.2, energy=0.2, motion=0.1, face_reaction=0.1
+ ),
+ reaction_lookback_sec=8.0,
+ reaction_payoff_sec=2.0,
+ peak_min_gap_sec=8.0,
+ peak_top_percentile=0.15,
+ ),
+ "action": ContentProfile(
+ name="action",
+ candidate_source="saliency",
+ channel_weights=_weights(
+ audio_event=0.3, motion=0.2, prosody=0.2, energy=0.2, transcript_semantic=0.1
+ ),
+ reaction_lookback_sec=6.0,
+ reaction_payoff_sec=2.0,
+ peak_min_gap_sec=8.0,
+ peak_top_percentile=0.15,
+ ),
+}
+
+DEFAULT_PROFILE = "podcast"
+
+
+def get_profile(name: str | None) -> ContentProfile:
+ import sys
+ if name is not None and name not in PROFILES:
+ print(f"[profiles] unknown profile {name!r}; using {DEFAULT_PROFILE}", file=sys.stderr)
+ return PROFILES.get(name or DEFAULT_PROFILE, PROFILES[DEFAULT_PROFILE])
diff --git a/backend/services/transcript_packer.py b/backend/services/transcript_packer.py
index f311496..cd88f48 100644
--- a/backend/services/transcript_packer.py
+++ b/backend/services/transcript_packer.py
@@ -38,6 +38,8 @@ SILENCE_GAP_REPORT_SEC = 0.6 # list gaps >= this in the silence section
PHRASE_MAX_SEC = 12.0
PHRASE_MAX_CHARS = 160
ENERGY_PEAKS_TO_REPORT = 20
+REACTIONS_TO_REPORT = 20
+REACTION_REPORT_THRESHOLD = 0.15
def compute_cache_hash(video_path: str) -> str:
@@ -272,6 +274,7 @@ def pack_transcript(
transcript: dict,
source_label: str,
energy_data: Optional[list[dict]] = None,
+ events_data: Optional[list[dict]] = None,
) -> str:
"""Render a packed markdown view of a transcription JSON."""
duration = float(transcript.get("duration", 0.0))
@@ -348,6 +351,31 @@ def pack_transcript(
lines.append(f"[{_fmt_ts(t)}] {rms:.1f}dB{snippet}")
lines.append("")
+ # Laughter & reactions (optional) — the moment BEFORE a laugh is usually the
+ # clippable one, so these timestamps flag where to look for the setup.
+ if events_data:
+ def level(e):
+ return max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0))
+ reactions = [e for e in events_data if level(e) >= REACTION_REPORT_THRESHOLD]
+ reactions.sort(key=level, reverse=True)
+ reactions = reactions[:REACTIONS_TO_REPORT]
+ reactions.sort(key=lambda e: e.get("time", 0))
+ if reactions:
+ lines.append(f"## Laughter & reactions (top {len(reactions)})")
+ for e in reactions:
+ t = float(e.get("time", 0))
+ kind = max(
+ ("laughter", "cheering", "screaming"),
+ key=lambda k: e.get(k, 0),
+ )
+ near = _phrase_for_time(phrases, t)
+ snippet = ""
+ if near:
+ txt = near["text"]
+ snippet = f' near: "{txt[:70]}{"…" if len(txt) > 70 else ""}"'
+ lines.append(f"[{_fmt_ts(t)}] {kind} {e.get(kind, 0):.2f}{snippet}")
+ lines.append("")
+
return "\n".join(lines)
@@ -372,10 +400,11 @@ def write_packed(
cache_hash: str,
source_label: Optional[str] = None,
energy_data: Optional[list[dict]] = None,
+ events_data: Optional[list[dict]] = None,
) -> tuple[str, str]:
"""Pack a transcript dict and write to .podcli/packed/<hash>.md."""
label = source_label or cache_hash
- md = pack_transcript(transcript, label, energy_data=energy_data)
+ md = pack_transcript(transcript, label, energy_data=energy_data, events_data=events_data)
os.makedirs(_packed_dir(), exist_ok=True)
out_path = packed_path_for(cache_hash)
with open(out_path, "w", encoding="utf-8") as f:
diff --git a/plans/moment-detection.md b/plans/moment-detection.md
new file mode 100644
index 0000000..4665bd6
--- /dev/null
+++ b/plans/moment-detection.md
@@ -0,0 +1,110 @@
+# podcli → Generalized moment detection (multi-signal saliency engine)
+
+> Goal: one detector that works across content types. The podcast clip selector and a new "auto-cut the funny/interesting moments" highlighter (party videos, vlogs, action, streams) become **two profiles of one engine**, not two features. Anchor use case that surfaced the need: a folder of party clips, "cut me the funny bits" — any language, no labels. Podcast selection is preserved as the default profile.
+
+## North star
+
+```text
+one long video (any content, any language)
+ → extract N signal channels (transcript, energy, laughter, prosody, motion, face)
+ → per-video-normalize each into a 0..1 saliency curve on a common time grid
+ → per-PROFILE weighted fusion → one interestingness curve
+ → peak-pick (NMS + prominence) → reaction-expand → snap to natural boundaries
+ → candidate moments → (render to formats via the existing fan-out)
+```
+
+## The one decision everything hangs on
+
+**Content profile is orthogonal to output format.** `FormatSpec` (9:16 / 16:9 / 1:1) is a *render-time* aspect-ratio decision. A **`ContentProfile`** (podcast / party / action / auto) is a *detection-time* decision about **which signals matter and how candidates are generated**. A moment is detected under a profile, then rendered to one or more formats. Do not overload `FormatSpec.score_key` for this — mirror its pattern in a separate `ContentProfile` spec.
+
+The existing transcript-LLM selector is **one channel** in this engine (the `transcript_semantic` channel), not the whole thing. That is the reframe: today the LLM *is* selection; tomorrow it is the dominant channel for the podcast profile and near-zero for party.
+
+## Why this generalizes (research-backed)
+
+Multi-signal highlight detection is a solved shape: compute a per-modality saliency curve, normalize **per-video** (never global), fuse (weighted sum with soft-OR bias, audio weighted heaviest for human-centric video), peak-pick, and for reaction-driven content extend **backwards from the reaction** to the cause. FunnyNet-W: a funny moment is "an n-second clip *followed by* laughter," audio carries >50% of the decision, optimal look-back ≈ 8s. Cai/Lu/Cai built laughter+applause+cheer detection *for home videos* at ~93% recall. All of it is unsupervised — no labeled data, only pretrained off-the-shelf models. See the audio-model and fusion research threads (this session) for citations.
+
+## Profiles as concrete weight vectors
+
+Channels each emit a per-video-normalized 0..1 curve; a profile is a weight vector + candidate-source + peak params.
+
+| Channel | podcast | party | action | Source status |
+|---|---|---|---|---|
+| `transcript_semantic` (LLM dialogue/quotability) | **0.5** | 0.0 | 0.1 | REUSE `claude_suggest.py` |
+| `prosody` (pitch/energy/rate spikes) | 0.2 | 0.2 | 0.2 | NEW (numpy, no librosa) |
+| `audio_event` (laughter/cheer/applause/scream) | 0.15 | **0.4** | **0.3** | NEW (YAMNet-ONNX, no torch) |
+| `energy` (RMS, peak-weighted) | 0.1 | 0.2 | 0.2 | REUSE `audio_analyzer.py` |
+| `motion` (optical flow + scene-cut density) | 0.0 | 0.1 | **0.2** | PARTIAL (cuts exist; flow NEW) |
+| `face_reaction` (mouth activity / smile proxy) | 0.05 | 0.1 | 0.0 | REUSE `face_analysis.py` (crude) |
+| **candidate source** | `llm` | `saliency` | `saliency` | — |
+
+Weights are starting points, tuned later. `auto` = a cheap first pass picks the profile: clean speech + 1-2 stable faces → podcast; sparse speech + high motion + crowd/laughter audio → party; sustained high motion → action.
+
+## Two candidate-generation modes (the key to not regressing podcast)
+
+- **`llm` (podcast):** the existing `claude_suggest` flow generates candidates over the whole transcript, exactly as today. The fused signal score blends in at the ranking seam. **When party channels have weight 0, output is byte-for-byte the current behavior.** This is the safety property.
+- **`saliency` (party / action):** the fused curve drives candidate generation directly — `find_peaks` → reaction-expand → boundary-snap → candidate windows. Works with **no usable transcript**. The LLM is optional here, used only to title/caption the already-chosen windows.
+
+## Architecture — new modules
+
+1. **`backend/services/audio_events.py`** (NEW, no-torch)
+ - YAMNet exported to a **self-contained ONNX graph** (raw 16 kHz waveform in, log-mel baked into the graph → **no librosa needed**). ~15 MB, Apache-2.0. AudioSet classes `Laughter`, `Cheering`, `Applause`, `Screaming` are native outputs (+ Giggle/Belly-laugh/Whoop children).
+ - Reuse `transcription_whispercpp._extract_wav` (`-ar 16000 -ac 1`) for audio; **do not import `speaker_detection`** (drags in torch).
+ - Returns per-window (≈1 Hz, matching energy) class probabilities → one 0..1 curve per trigger class.
+ - Runtime: `onnxruntime`. **First test whether `cv2.dnn` (already loads YuNet `.onnx`) can run yamnet.onnx** — if yes, zero new deps.
+ - Ship `backend/models/yamnet.onnx` + a **dev-only** `scripts/export_yamnet.py` (uses tf2onnx; never shipped or imported at runtime).
+
+2. **`backend/services/saliency.py`** (NEW)
+ - `normalize_per_video(curve)` — robust z-score via numpy median/MAD, or percentile-rank. **Never global.**
+ - `fuse(channels, profile)` — weighted sum with soft-OR bias.
+ - `pick_peaks(curve, profile)` — **numpy reimplementation of `find_peaks`** (~40 lines: height floor `mean + kσ`, min-distance NMS by descending height, prominence filter, min-width). Avoids adding scipy. Keep peaks above the top ~10-15th percentile of the video's **own** prominence distribution (adaptive count).
+ - `reaction_expand(peak, events)` — if the peak's driver is an `audio_event` class, move anchor to reaction onset, extend ~8 s back, keep 1-3 s of payoff.
+ - `snap_boundaries(window)` — snap start/end to nearest {Whisper sentence-end, silence gap >300 ms, scene cut, motion-quiet local min} within ±500 ms.
+
+3. **`backend/services/profiles.py`** (NEW) — `ContentProfile` dataclass (name, `channel_weights`, `candidate_source`, peak params, reaction-expand params, boundary sources) mirroring `FormatSpec`; `get_profile(name)`; `auto_detect(signals)`.
+
+4. **`motion` channel** (Phase 3) — `cv2.calcOpticalFlowFarneback` on sampled frames (**OpenCV already present**) + reuse `local_reframe.count_scene_cuts` / `clip_generator._detect_scene_cuts`.
+
+5. **`prosody` channel** (Phase 2/3) — numpy F0 (autocorrelation) + short-term energy + speech-rate vs speaker baseline. No librosa.
+
+## Dependency deltas (no-torch native path)
+
+- **ADD** `onnxruntime` to `requirements-runtime.txt` — the only new hard dep. **Settled by spike:** OpenCV-DNN *cannot* run the YAMNet graph (rejects the dynamic-shape mel front-end: `dynamic 'zero' shapes are not supported`), so the zero-new-dep route is out. `onnxruntime` still pulls in no torch/TF.
+- **NO** librosa (self-contained waveform-in graph, confirmed), **NO** torch/TF at runtime.
+- `scipy` is present in the dev venv (so `find_peaks` works there) but is **not** in `requirements-runtime.txt` — either add it or keep the numpy peak-pick reimpl for the hermetic runtime. Decide at Phase 2.
+- Optical flow + face already covered by `opencv-python-headless`.
+- `yamnet.onnx` weight (~16 MB) committed to `backend/models/`; no export step needed — a self-contained waveform-in export already exists at HF `zeropointnine/yamnet-onnx` (input `waveform` [dynamic], outputs `[frames,521]` scores + `[frames,1024]` embeddings + `[frames,64]` mel). Verify its license before committing; keep a dev-only `scripts/export_yamnet.py` (tf2onnx) as the reproducible fallback.
+
+## Spike results (validated this session)
+
+Ran the self-contained YAMNet ONNX on real podcli audio (a 38 s Deeptech Decoded clip, extracted with podcli's exact `-ar 16000 -ac 1`):
+- **No-torch confirmed:** `torch`/`tensorflow` absent from the process before and after inference (onnxruntime only).
+- **Correct classification:** Speech 0.774 (dominant, correct for a talking clip), Silence 0.186, and Laughter/Cheering/Applause/Screaming all exactly 0.000 — correct negative control (serious monologue, no laughter). Mel front-end baked in the graph → featurization provably correct, no code to own.
+- **Negligible cost:** 98 ms for 38 s of audio (~380× realtime); a 3 h video ≈ 28 s of audio-event analysis.
+- **Positive reaction detection confirmed on podcast content.** Swept 32 real clips: the detector ranked the one comedic show + two chuckle moments above the dry technical majority (which scored exactly 0.00). Zooming in, laughter fires as a **sharp 1-2 frame spike anti-correlated with Speech** (e.g. laugh=0.23 while speech drops 1.00→0.11 at 13.5s, then back to speech) — the exact reaction signature the 8s-backward-expansion relies on. Absolute values are modest (0.23-0.25 for brief chuckles in pre-cut clips) but that is a non-issue: a spike on a 0.00 baseline is trivially caught by **prominence-based peak-picking**, which is why per-video normalization (not absolute thresholds) is mandated above.
+- **Still worth a check (needs real party footage):** amplitude/robustness on noisy handheld party audio with music and overlapping speech — YAMNet's known soft spot. If weak, AST ONNX (`onnx-community`, pre-exported, mAP 0.485) is the drop-in fallback; needs a 128-bin log-mel numpy front-end.
+
+## Integration seams (from the code map)
+
+1. **Detect-once hub** — `backend/cli.py` ~658-720 (and MCP twin `backend/main.py:handle_suggest_clips` ~358). Energy is *already* computed once here (`get_energy_profile`) and `face_map` extracted. All new channels compute here; fusion produces the saliency curve.
+2. **Scoring merge** — `claude_suggest.suggest_with_claude` normalization (~999) + `_select_top_by_score` (~687). Blend the LLM score in as the `transcript_semantic` channel; key the weights off the active `ContentProfile`.
+3. **Transcript-packer co-location** — `transcript_packer.pack_transcript` (~274) already flags top-RMS moments for the in-chat MCP agent. Add laughter/event/motion flags so the conversational path also sees them.
+
+## Contract-change tax
+
+A new `profile` param threads the **same ~12 hops the `format` field did**: `suggest-clips.handler.ts` inputSchema, `src/models/index.ts` types, `src/server.ts` Zod (dual-declaration), `src/ui/web-server.ts` `styledClips` whitelist, `python-executor` stdin, `main.py` params, and `cli.py`'s `generate_clip` call sites — or it silently reverts to the default. Follow the `format` precedent exactly.
+
+## Phasing
+
+- **Phase 0 — profile scaffolding, zero behavior change.** Add `ContentProfile` abstraction; thread the `profile` param through the ~12 hops; `default = podcast` reproduces current selection exactly. **Gate: existing test suite green; same clips out for a fixed transcript.**
+- **Phase 1 — audio-event channel (the isolated valuable core).** YAMNet-ONNX laughter/cheer/applause/scream computed in the detect-once hub. Feeds podcast ranking as a labeled signal (laughs already spike energy; now they're *named*) and lays the party foundation. **Gate: laughter timestamps validated on a sample clip; podcast output unchanged unless the channel is given weight.**
+- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** `saliency.py` fusion + numpy peak-pick + reaction-expand (8 s) + boundary-snap. Party profile = energy + audio_event + prosody, no transcript, no motion. **Party videos auto-clip end to end. Gate: demo on real party footage.**
+- **Phase 3 — visual channels + action profile + multi-file pooling.** Optical flow (OpenCV) + face-reaction channels; action profile; pool peaks across a *folder* of clips and rank globally ("best 15 bits from tonight" across 80 phone videos). **Gate: catches a silent visual gag; folder-level ranking works.**
+- **Phase 4 (optional) — highlight reel renderer.** Ordering, pacing, optional music-bed ducking, transitions — a thin renderer atop the detected moments, reusing the clip-render stack.
+
+## Guardrails
+
+- **Per-video normalization is mandatory** — party clips vary wildly in level/room/camera; global normalization is wrong.
+- **Podcast profile must reproduce current behavior** when party channels are weight 0. Ship behind the profile param; default is safe. This is the non-regression contract.
+- **`find_peaks` `distance` param is the NMS / min-gap** — it deletes lower peaks within the gap automatically. Adaptive clip count = top percentile of the video's own prominence distribution, not an absolute N.
+- **YAMNet ONNX must never import torch/TF at runtime.** The export step is dev/CI only; the native runtime installs `onnxruntime` (or reuses OpenCV-DNN) and feeds raw 16 kHz samples.
+- **Reaction-expansion only fires for saliency candidates** whose trigger is an audio-event class — never rewrite LLM-chosen podcast windows.
diff --git a/tests/test_audio_events.py b/tests/test_audio_events.py
new file mode 100644
index 0000000..cea4e63
--- /dev/null
+++ b/tests/test_audio_events.py
@@ -0,0 +1,80 @@
+"""Tests for backend.services.audio_events scoring and profile scaffolding."""
+
+import os
+import sys
+import unittest
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import audio_events as ae
+from services import profiles
+
+
+def _event(t, laughter=0.0, cheering=0.0, screaming=0.0, speech=0.0):
+ return {
+ "time": t,
+ "laughter": laughter,
+ "cheering": cheering,
+ "screaming": screaming,
+ "speech": speech,
+ }
+
+
+class ComputeEventScoresTests(unittest.TestCase):
+ def test_empty_events_returns_zeros(self):
+ segments = [{"start": 0, "end": 10}, {"start": 10, "end": 20}]
+ self.assertEqual(ae.compute_event_scores([], segments), [0.0, 0.0])
+
+ def test_empty_segments_returns_empty(self):
+ self.assertEqual(ae.compute_event_scores([_event(1, laughter=0.5)], []), [])
+
+ def test_segment_with_laugh_scores_higher_than_silent(self):
+ events = [_event(2, laughter=0.5, speech=0.1), _event(30, speech=1.0)]
+ scores = ae.compute_event_scores(
+ events, [{"start": 0, "end": 10}, {"start": 20, "end": 40}]
+ )
+ self.assertGreater(scores[0], scores[1])
+ self.assertEqual(scores[1], 0.0)
+
+ def test_score_uses_loudest_reaction_channel(self):
+ # cheering should count even with zero laughter
+ events = [_event(1, cheering=0.5)]
+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])
+ self.assertGreater(scores[0], 0.0)
+
+ def test_score_is_clamped_to_ten(self):
+ events = [_event(1, laughter=1.0)]
+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])
+ self.assertLessEqual(scores[0], 10.0)
+
+ def test_speech_alone_is_not_a_reaction(self):
+ events = [_event(1, speech=1.0)]
+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])
+ self.assertEqual(scores[0], 0.0)
+
+
+class ProfileTests(unittest.TestCase):
+ def test_default_is_podcast_and_llm_sourced(self):
+ p = profiles.get_profile(None)
+ self.assertEqual(p.name, "podcast")
+ self.assertEqual(p.candidate_source, "llm")
+
+ def test_unknown_profile_falls_back_to_default(self):
+ self.assertEqual(profiles.get_profile("nonsense").name, profiles.DEFAULT_PROFILE)
+
+ def test_party_is_saliency_sourced_and_ignores_transcript(self):
+ p = profiles.get_profile("party")
+ self.assertEqual(p.candidate_source, "saliency")
+ self.assertEqual(p.channel_weights["transcript_semantic"], 0.0)
+ self.assertGreater(p.channel_weights["audio_event"], 0.0)
+
+ def test_podcast_weights_are_transcript_dominant(self):
+ w = profiles.get_profile("podcast").channel_weights
+ self.assertEqual(max(w, key=w.get), "transcript_semantic")
+
+
+if __name__ == "__main__":
+ unittest.main()