Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Add laughter/reaction detection channel (YAMNet ONNX, no-torch)
Nika Siradze379e7db7bdf8
diff --git a/backend/cli.py b/backend/cli.pyindex f5c3f78..5d21292 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -395,6 +395,7 @@ def cmd_process(args):from services.clip_generator import generate_clipfrom services.transcript_parser import parse_speaker_transcriptfrom services.audio_analyzer import get_energy_profile+ from services.audio_events import get_event_profile, is_available as audio_events_availablefrom services.encoder import get_encoder_infofrom presets import get_preset, DEFAULT_PRESET, MIN_CLIP_DURATION, MAX_CLIP_DURATION, TARGET_CLIP_DURATION_MIN, TARGET_CLIP_DURATION_MAX@@ -657,6 +658,7 @@ def cmd_process(args):# ── Step 2: Analyze audio energy ──energy_scores = None+ reaction_scores = Noneif config.get("energy_boost", True):print(" [2/4] Analyzing audio energy...")try:@@ -665,6 +667,15 @@ def cmd_process(args):print(f" {len(profile['peak_times'])} peak moments found")except Exception as e:print(f" Skipped (error: {e})")+ if audio_events_available():+ try:+ reactions = get_event_profile(video_path, segments)+ reaction_scores = reactions["segment_scores"]+ n = len(reactions["reaction_times"])+ if n:+ print(f" {n} laughter/reaction moments found")+ except Exception as e:+ print(f" Reactions skipped (error: {e})")else:print(" [2/4] Audio analysis skipped (--no-energy)")@@ -712,6 +723,7 @@ def cmd_process(args):clips = _suggest_clips(segments=segments,energy_scores=energy_scores,+ reaction_scores=reaction_scores,top_n=top_n,min_dur=config.get("min_clip_duration", MIN_CLIP_DURATION),max_dur=config.get("max_clip_duration", MAX_CLIP_DURATION),@@ -1607,6 +1619,7 @@ def _post_render_loop(def _suggest_clips(segments: list,energy_scores: list | None = None,+ reaction_scores: list | None = None,top_n: int = 5,min_dur: float = MIN_CLIP_DURATION,max_dur: float = MAX_CLIP_DURATION,@@ -1804,6 +1817,15 @@ def _suggest_clips(if max_e > 7:reasons.append("high_energy")+ # ── 6b. Laughter / reactions (0-6 pts) ──+ if reaction_scores:+ seg_reactions = reaction_scores[snap_start : snap_end + 1]+ if seg_reactions:+ max_r = max(seg_reactions)+ score += min(max_r, 6)+ if max_r > 3:+ reasons.append("laughter")+# ── 7. Density check — penalize sparse/rambling segments ──words_per_sec = len(text.split()) / max(dur, 1)if words_per_sec < 1.5:diff --git a/backend/main.py b/backend/main.pyindex 468669a..9355357 100644--- a/backend/main.py+++ b/backend/main.py@@ -87,11 +87,19 @@ def handle_transcribe(task_id: str, params: dict):except Exception:pass # energy is a nice-to-have+ events_data = None+ try:+ from services.audio_events import extract_audio_events+ events_data = extract_audio_events(file_path)+ except Exception:+ pass # reactions are a nice-to-have+packed_path, packed_md = write_packed(result,cache_hash,source_label=os.path.basename(file_path),energy_data=energy_data,+ events_data=events_data,)result["packed_path"] = packed_pathresult["packed_size_bytes"] = len(packed_md.encode("utf-8"))@@ -253,18 +261,27 @@ def handle_pack_transcript(task_id: str, params: dict):returnenergy_data = params.get("energy_data")- if energy_data is None and params.get("file_path"):- try:- from services.audio_analyzer import extract_audio_energy- energy_data = extract_audio_energy(params["file_path"])- except Exception:- pass+ events_data = params.get("events_data")+ if params.get("file_path"):+ if energy_data is None:+ try:+ from services.audio_analyzer import extract_audio_energy+ energy_data = extract_audio_energy(params["file_path"])+ except Exception:+ pass+ if events_data is None:+ try:+ from services.audio_events import extract_audio_events+ events_data = extract_audio_events(params["file_path"])+ except Exception:+ passpath, md = write_packed(transcript,cache_hash,source_label=params.get("source_label"),energy_data=energy_data,+ events_data=events_data,)emit_result(task_id, "success", data={"packed_path": path,diff --git a/backend/models/yamnet.onnx b/backend/models/yamnet.onnxnew file mode 100644index 0000000..a799a1fBinary files /dev/null and b/backend/models/yamnet.onnx differdiff --git a/backend/models/yamnet_class_map.csv b/backend/models/yamnet_class_map.csvnew file mode 100644index 0000000..9c07d81--- /dev/null+++ b/backend/models/yamnet_class_map.csv@@ -0,0 +1,522 @@+index,mid,display_name+0,/m/09x0r,Speech+1,/m/0ytgt,"Child speech, kid speaking"+2,/m/01h8n0,Conversation+3,/m/02qldy,"Narration, monologue"+4,/m/0261r1,Babbling+5,/m/0brhx,Speech synthesizer+6,/m/07p6fty,Shout+7,/m/07q4ntr,Bellow+8,/m/07rwj3x,Whoop+9,/m/07sr1lc,Yell+10,/t/dd00135,Children shouting+11,/m/03qc9zr,Screaming+12,/m/02rtxlg,Whispering+13,/m/01j3sz,Laughter+14,/t/dd00001,Baby laughter+15,/m/07r660_,Giggle+16,/m/07s04w4,Snicker+17,/m/07sq110,Belly laugh+18,/m/07rgt08,"Chuckle, chortle"+19,/m/0463cq4,"Crying, sobbing"+20,/t/dd00002,"Baby cry, infant cry"+21,/m/07qz6j3,Whimper+22,/m/07qw_06,"Wail, moan"+23,/m/07plz5l,Sigh+24,/m/015lz1,Singing+25,/m/0l14jd,Choir+26,/m/01swy6,Yodeling+27,/m/02bk07,Chant+28,/m/01c194,Mantra+29,/t/dd00005,Child singing+30,/t/dd00006,Synthetic singing+31,/m/06bxc,Rapping+32,/m/02fxyj,Humming+33,/m/07s2xch,Groan+34,/m/07r4k75,Grunt+35,/m/01w250,Whistling+36,/m/0lyf6,Breathing+37,/m/07mzm6,Wheeze+38,/m/01d3sd,Snoring+39,/m/07s0dtb,Gasp+40,/m/07pyy8b,Pant+41,/m/07q0yl5,Snort+42,/m/01b_21,Cough+43,/m/0dl9sf8,Throat clearing+44,/m/01hsr_,Sneeze+45,/m/07ppn3j,Sniff+46,/m/06h7j,Run+47,/m/07qv_x_,Shuffle+48,/m/07pbtc8,"Walk, footsteps"+49,/m/03cczk,"Chewing, mastication"+50,/m/07pdhp0,Biting+51,/m/0939n_,Gargling+52,/m/01g90h,Stomach rumble+53,/m/03q5_w,"Burping, eructation"+54,/m/02p3nc,Hiccup+55,/m/02_nn,Fart+56,/m/0k65p,Hands+57,/m/025_jnm,Finger snapping+58,/m/0l15bq,Clapping+59,/m/01jg02,"Heart sounds, heartbeat"+60,/m/01jg1z,Heart murmur+61,/m/053hz1,Cheering+62,/m/028ght,Applause+63,/m/07rkbfh,Chatter+64,/m/03qtwd,Crowd+65,/m/07qfr4h,"Hubbub, speech noise, speech babble"+66,/t/dd00013,Children playing+67,/m/0jbk,Animal+68,/m/068hy,"Domestic animals, pets"+69,/m/0bt9lr,Dog+70,/m/05tny_,Bark+71,/m/07r_k2n,Yip+72,/m/07qf0zm,Howl+73,/m/07rc7d9,Bow-wow+74,/m/0ghcn6,Growling+75,/t/dd00136,Whimper (dog)+76,/m/01yrx,Cat+77,/m/02yds9,Purr+78,/m/07qrkrw,Meow+79,/m/07rjwbb,Hiss+80,/m/07r81j2,Caterwaul+81,/m/0ch8v,"Livestock, farm animals, working animals"+82,/m/03k3r,Horse+83,/m/07rv9rh,Clip-clop+84,/m/07q5rw0,"Neigh, whinny"+85,/m/01xq0k1,"Cattle, bovinae"+86,/m/07rpkh9,Moo+87,/m/0239kh,Cowbell+88,/m/068zj,Pig+89,/t/dd00018,Oink+90,/m/03fwl,Goat+91,/m/07q0h5t,Bleat+92,/m/07bgp,Sheep+93,/m/025rv6n,Fowl+94,/m/09b5t,"Chicken, rooster"+95,/m/07st89h,Cluck+96,/m/07qn5dc,"Crowing, cock-a-doodle-doo"+97,/m/01rd7k,Turkey+98,/m/07svc2k,Gobble+99,/m/09ddx,Duck+100,/m/07qdb04,Quack+101,/m/0dbvp,Goose+102,/m/07qwf61,Honk+103,/m/01280g,Wild animals+104,/m/0cdnk,"Roaring cats (lions, tigers)"+105,/m/04cvmfc,Roar+106,/m/015p6,Bird+107,/m/020bb7,"Bird vocalization, bird call, bird song"+108,/m/07pggtn,"Chirp, tweet"+109,/m/07sx8x_,Squawk+110,/m/0h0rv,"Pigeon, dove"+111,/m/07r_25d,Coo+112,/m/04s8yn,Crow+113,/m/07r5c2p,Caw+114,/m/09d5_,Owl+115,/m/07r_80w,Hoot+116,/m/05_wcq,"Bird flight, flapping wings"+117,/m/01z5f,"Canidae, dogs, wolves"+118,/m/06hps,"Rodents, rats, mice"+119,/m/04rmv,Mouse+120,/m/07r4gkf,Patter+121,/m/03vt0,Insect+122,/m/09xqv,Cricket+123,/m/09f96,Mosquito+124,/m/0h2mp,"Fly, housefly"+125,/m/07pjwq1,Buzz+126,/m/01h3n,"Bee, wasp, etc."+127,/m/09ld4,Frog+128,/m/07st88b,Croak+129,/m/078jl,Snake+130,/m/07qn4z3,Rattle+131,/m/032n05,Whale vocalization+132,/m/04rlf,Music+133,/m/04szw,Musical instrument+134,/m/0fx80y,Plucked string instrument+135,/m/0342h,Guitar+136,/m/02sgy,Electric guitar+137,/m/018vs,Bass guitar+138,/m/042v_gx,Acoustic guitar+139,/m/06w87,"Steel guitar, slide guitar"+140,/m/01glhc,Tapping (guitar technique)+141,/m/07s0s5r,Strum+142,/m/018j2,Banjo+143,/m/0jtg0,Sitar+144,/m/04rzd,Mandolin+145,/m/01bns_,Zither+146,/m/07xzm,Ukulele+147,/m/05148p4,Keyboard (musical)+148,/m/05r5c,Piano+149,/m/01s0ps,Electric piano+150,/m/013y1f,Organ+151,/m/03xq_f,Electronic organ+152,/m/03gvt,Hammond organ+153,/m/0l14qv,Synthesizer+154,/m/01v1d8,Sampler+155,/m/03q5t,Harpsichord+156,/m/0l14md,Percussion+157,/m/02hnl,Drum kit+158,/m/0cfdd,Drum machine+159,/m/026t6,Drum+160,/m/06rvn,Snare drum+161,/m/03t3fj,Rimshot+162,/m/02k_mr,Drum roll+163,/m/0bm02,Bass drum+164,/m/011k_j,Timpani+165,/m/01p970,Tabla+166,/m/01qbl,Cymbal+167,/m/03qtq,Hi-hat+168,/m/01sm1g,Wood block+169,/m/07brj,Tambourine+170,/m/05r5wn,Rattle (instrument)+171,/m/0xzly,Maraca+172,/m/0mbct,Gong+173,/m/016622,Tubular bells+174,/m/0j45pbj,Mallet percussion+175,/m/0dwsp,"Marimba, xylophone"+176,/m/0dwtp,Glockenspiel+177,/m/0dwt5,Vibraphone+178,/m/0l156b,Steelpan+179,/m/05pd6,Orchestra+180,/m/01kcd,Brass instrument+181,/m/0319l,French horn+182,/m/07gql,Trumpet+183,/m/07c6l,Trombone+184,/m/0l14_3,Bowed string instrument+185,/m/02qmj0d,String section+186,/m/07y_7,"Violin, fiddle"+187,/m/0d8_n,Pizzicato+188,/m/01xqw,Cello+189,/m/02fsn,Double bass+190,/m/085jw,"Wind instrument, woodwind instrument"+191,/m/0l14j_,Flute+192,/m/06ncr,Saxophone+193,/m/01wy6,Clarinet+194,/m/03m5k,Harp+195,/m/0395lw,Bell+196,/m/03w41f,Church bell+197,/m/027m70_,Jingle bell+198,/m/0gy1t2s,Bicycle bell+199,/m/07n_g,Tuning fork+200,/m/0f8s22,Chime+201,/m/026fgl,Wind chime+202,/m/0150b9,Change ringing (campanology)+203,/m/03qjg,Harmonica+204,/m/0mkg,Accordion+205,/m/0192l,Bagpipes+206,/m/02bxd,Didgeridoo+207,/m/0l14l2,Shofar+208,/m/07kc_,Theremin+209,/m/0l14t7,Singing bowl+210,/m/01hgjl,Scratching (performance technique)+211,/m/064t9,Pop music+212,/m/0glt670,Hip hop music+213,/m/02cz_7,Beatboxing+214,/m/06by7,Rock music+215,/m/03lty,Heavy metal+216,/m/05r6t,Punk rock+217,/m/0dls3,Grunge+218,/m/0dl5d,Progressive rock+219,/m/07sbbz2,Rock and roll+220,/m/05w3f,Psychedelic rock+221,/m/06j6l,Rhythm and blues+222,/m/0gywn,Soul music+223,/m/06cqb,Reggae+224,/m/01lyv,Country+225,/m/015y_n,Swing music+226,/m/0gg8l,Bluegrass+227,/m/02x8m,Funk+228,/m/02w4v,Folk music+229,/m/06j64v,Middle Eastern music+230,/m/03_d0,Jazz+231,/m/026z9,Disco+232,/m/0ggq0m,Classical music+233,/m/05lls,Opera+234,/m/02lkt,Electronic music+235,/m/03mb9,House music+236,/m/07gxw,Techno+237,/m/07s72n,Dubstep+238,/m/0283d,Drum and bass+239,/m/0m0jc,Electronica+240,/m/08cyft,Electronic dance music+241,/m/0fd3y,Ambient music+242,/m/07lnk,Trance music+243,/m/0g293,Music of Latin America+244,/m/0ln16,Salsa music+245,/m/0326g,Flamenco+246,/m/0155w,Blues+247,/m/05fw6t,Music for children+248,/m/02v2lh,New-age music+249,/m/0y4f8,Vocal music+250,/m/0z9c,A capella+251,/m/0164x2,Music of Africa+252,/m/0145m,Afrobeat+253,/m/02mscn,Christian music+254,/m/016cjb,Gospel music+255,/m/028sqc,Music of Asia+256,/m/015vgc,Carnatic music+257,/m/0dq0md,Music of Bollywood+258,/m/06rqw,Ska+259,/m/02p0sh1,Traditional music+260,/m/05rwpb,Independent music+261,/m/074ft,Song+262,/m/025td0t,Background music+263,/m/02cjck,Theme music+264,/m/03r5q_,Jingle (music)+265,/m/0l14gg,Soundtrack music+266,/m/07pkxdp,Lullaby+267,/m/01z7dr,Video game music+268,/m/0140xf,Christmas music+269,/m/0ggx5q,Dance music+270,/m/04wptg,Wedding music+271,/t/dd00031,Happy music+272,/t/dd00033,Sad music+273,/t/dd00034,Tender music+274,/t/dd00035,Exciting music+275,/t/dd00036,Angry music+276,/t/dd00037,Scary music+277,/m/03m9d0z,Wind+278,/m/09t49,Rustling leaves+279,/t/dd00092,Wind noise (microphone)+280,/m/0jb2l,Thunderstorm+281,/m/0ngt1,Thunder+282,/m/0838f,Water+283,/m/06mb1,Rain+284,/m/07r10fb,Raindrop+285,/t/dd00038,Rain on surface+286,/m/0j6m2,Stream+287,/m/0j2kx,Waterfall+288,/m/05kq4,Ocean+289,/m/034srq,"Waves, surf"+290,/m/06wzb,Steam+291,/m/07swgks,Gurgling+292,/m/02_41,Fire+293,/m/07pzfmf,Crackle+294,/m/07yv9,Vehicle+295,/m/019jd,"Boat, Water vehicle"+296,/m/0hsrw,"Sailboat, sailing ship"+297,/m/056ks2,"Rowboat, canoe, kayak"+298,/m/02rlv9,"Motorboat, speedboat"+299,/m/06q74,Ship+300,/m/012f08,Motor vehicle (road)+301,/m/0k4j,Car+302,/m/0912c9,"Vehicle horn, car horn, honking"+303,/m/07qv_d5,Toot+304,/m/02mfyn,Car alarm+305,/m/04gxbd,"Power windows, electric windows"+306,/m/07rknqz,Skidding+307,/m/0h9mv,Tire squeal+308,/t/dd00134,Car passing by+309,/m/0ltv,"Race car, auto racing"+310,/m/07r04,Truck+311,/m/0gvgw0,Air brake+312,/m/05x_td,"Air horn, truck horn"+313,/m/02rhddq,Reversing beeps+314,/m/03cl9h,"Ice cream truck, ice cream van"+315,/m/01bjv,Bus+316,/m/03j1ly,Emergency vehicle+317,/m/04qvtq,Police car (siren)+318,/m/012n7d,Ambulance (siren)+319,/m/012ndj,"Fire engine, fire truck (siren)"+320,/m/04_sv,Motorcycle+321,/m/0btp2,"Traffic noise, roadway noise"+322,/m/06d_3,Rail transport+323,/m/07jdr,Train+324,/m/04zmvq,Train whistle+325,/m/0284vy3,Train horn+326,/m/01g50p,"Railroad car, train wagon"+327,/t/dd00048,Train wheels squealing+328,/m/0195fx,"Subway, metro, underground"+329,/m/0k5j,Aircraft+330,/m/014yck,Aircraft engine+331,/m/04229,Jet engine+332,/m/02l6bg,"Propeller, airscrew"+333,/m/09ct_,Helicopter+334,/m/0cmf2,"Fixed-wing aircraft, airplane"+335,/m/0199g,Bicycle+336,/m/06_fw,Skateboard+337,/m/02mk9,Engine+338,/t/dd00065,Light engine (high frequency)+339,/m/08j51y,"Dental drill, dentist's drill"+340,/m/01yg9g,Lawn mower+341,/m/01j4z9,Chainsaw+342,/t/dd00066,Medium engine (mid frequency)+343,/t/dd00067,Heavy engine (low frequency)+344,/m/01h82_,Engine knocking+345,/t/dd00130,Engine starting+346,/m/07pb8fc,Idling+347,/m/07q2z82,"Accelerating, revving, vroom"+348,/m/02dgv,Door+349,/m/03wwcy,Doorbell+350,/m/07r67yg,Ding-dong+351,/m/02y_763,Sliding door+352,/m/07rjzl8,Slam+353,/m/07r4wb8,Knock+354,/m/07qcpgn,Tap+355,/m/07q6cd_,Squeak+356,/m/0642b4,Cupboard open or close+357,/m/0fqfqc,Drawer open or close+358,/m/04brg2,"Dishes, pots, and pans"+359,/m/023pjk,"Cutlery, silverware"+360,/m/07pn_8q,Chopping (food)+361,/m/0dxrf,Frying (food)+362,/m/0fx9l,Microwave oven+363,/m/02pjr4,Blender+364,/m/02jz0l,"Water tap, faucet"+365,/m/0130jx,Sink (filling or washing)+366,/m/03dnzn,Bathtub (filling or washing)+367,/m/03wvsk,Hair dryer+368,/m/01jt3m,Toilet flush+369,/m/012xff,Toothbrush+370,/m/04fgwm,Electric toothbrush+371,/m/0d31p,Vacuum cleaner+372,/m/01s0vc,Zipper (clothing)+373,/m/03v3yw,Keys jangling+374,/m/0242l,Coin (dropping)+375,/m/01lsmm,Scissors+376,/m/02g901,"Electric shaver, electric razor"+377,/m/05rj2,Shuffling cards+378,/m/0316dw,Typing+379,/m/0c2wf,Typewriter+380,/m/01m2v,Computer keyboard+381,/m/081rb,Writing+382,/m/07pp_mv,Alarm+383,/m/07cx4,Telephone+384,/m/07pp8cl,Telephone bell ringing+385,/m/01hnzm,Ringtone+386,/m/02c8p,"Telephone dialing, DTMF"+387,/m/015jpf,Dial tone+388,/m/01z47d,Busy signal+389,/m/046dlr,Alarm clock+390,/m/03kmc9,Siren+391,/m/0dgbq,Civil defense siren+392,/m/030rvx,Buzzer+393,/m/01y3hg,"Smoke detector, smoke alarm"+394,/m/0c3f7m,Fire alarm+395,/m/04fq5q,Foghorn+396,/m/0l156k,Whistle+397,/m/06hck5,Steam whistle+398,/t/dd00077,Mechanisms+399,/m/02bm9n,"Ratchet, pawl"+400,/m/01x3z,Clock+401,/m/07qjznt,Tick+402,/m/07qjznl,Tick-tock+403,/m/0l7xg,Gears+404,/m/05zc1,Pulleys+405,/m/0llzx,Sewing machine+406,/m/02x984l,Mechanical fan+407,/m/025wky1,Air conditioning+408,/m/024dl,Cash register+409,/m/01m4t,Printer+410,/m/0dv5r,Camera+411,/m/07bjf,Single-lens reflex camera+412,/m/07k1x,Tools+413,/m/03l9g,Hammer+414,/m/03p19w,Jackhammer+415,/m/01b82r,Sawing+416,/m/02p01q,Filing (rasp)+417,/m/023vsd,Sanding+418,/m/0_ksk,Power tool+419,/m/01d380,Drill+420,/m/014zdl,Explosion+421,/m/032s66,"Gunshot, gunfire"+422,/m/04zjc,Machine gun+423,/m/02z32qm,Fusillade+424,/m/0_1c,Artillery fire+425,/m/073cg4,Cap gun+426,/m/0g6b5,Fireworks+427,/g/122z_qxw,Firecracker+428,/m/07qsvvw,"Burst, pop"+429,/m/07pxg6y,Eruption+430,/m/07qqyl4,Boom+431,/m/083vt,Wood+432,/m/07pczhz,Chop+433,/m/07pl1bw,Splinter+434,/m/07qs1cx,Crack+435,/m/039jq,Glass+436,/m/07q7njn,"Chink, clink"+437,/m/07rn7sz,Shatter+438,/m/04k94,Liquid+439,/m/07rrlb6,"Splash, splatter"+440,/m/07p6mqd,Slosh+441,/m/07qlwh6,Squish+442,/m/07r5v4s,Drip+443,/m/07prgkl,Pour+444,/m/07pqc89,"Trickle, dribble"+445,/t/dd00088,Gush+446,/m/07p7b8y,Fill (with liquid)+447,/m/07qlf79,Spray+448,/m/07ptzwd,Pump (liquid)+449,/m/07ptfmf,Stir+450,/m/0dv3j,Boiling+451,/m/0790c,Sonar+452,/m/0dl83,Arrow+453,/m/07rqsjt,"Whoosh, swoosh, swish"+454,/m/07qnq_y,"Thump, thud"+455,/m/07rrh0c,Thunk+456,/m/0b_fwt,Electronic tuner+457,/m/02rr_,Effects unit+458,/m/07m2kt,Chorus effect+459,/m/018w8,Basketball bounce+460,/m/07pws3f,Bang+461,/m/07ryjzk,"Slap, smack"+462,/m/07rdhzs,"Whack, thwack"+463,/m/07pjjrj,"Smash, crash"+464,/m/07pc8lb,Breaking+465,/m/07pqn27,Bouncing+466,/m/07rbp7_,Whip+467,/m/07pyf11,Flap+468,/m/07qb_dv,Scratch+469,/m/07qv4k0,Scrape+470,/m/07pdjhy,Rub+471,/m/07s8j8t,Roll+472,/m/07plct2,Crushing+473,/t/dd00112,"Crumpling, crinkling"+474,/m/07qcx4z,Tearing+475,/m/02fs_r,"Beep, bleep"+476,/m/07qwdck,Ping+477,/m/07phxs1,Ding+478,/m/07rv4dm,Clang+479,/m/07s02z0,Squeal+480,/m/07qh7jl,Creak+481,/m/07qwyj0,Rustle+482,/m/07s34ls,Whir+483,/m/07qmpdm,Clatter+484,/m/07p9k1k,Sizzle+485,/m/07qc9xj,Clicking+486,/m/07rwm0c,Clickety-clack+487,/m/07phhsh,Rumble+488,/m/07qyrcz,Plop+489,/m/07qfgpx,"Jingle, tinkle"+490,/m/07rcgpl,Hum+491,/m/07p78v5,Zing+492,/t/dd00121,Boing+493,/m/07s12q4,Crunch+494,/m/028v0c,Silence+495,/m/01v_m0,Sine wave+496,/m/0b9m1,Harmonic+497,/m/0hdsk,Chirp tone+498,/m/0c1dj,Sound effect+499,/m/07pt_g0,Pulse+500,/t/dd00125,"Inside, small room"+501,/t/dd00126,"Inside, large room or hall"+502,/t/dd00127,"Inside, public space"+503,/t/dd00128,"Outside, urban or manmade"+504,/t/dd00129,"Outside, rural or natural"+505,/m/01b9nn,Reverberation+506,/m/01jnbd,Echo+507,/m/096m7z,Noise+508,/m/06_y0by,Environmental noise+509,/m/07rgkc5,Static+510,/m/06xkwv,Mains hum+511,/m/0g12c5,Distortion+512,/m/08p9q4,Sidetone+513,/m/07szfh9,Cacophony+514,/m/0chx_,White noise+515,/m/0cj0r,Pink noise+516,/m/07p_0gm,Throbbing+517,/m/01jwx6,Vibration+518,/m/07c52,Television+519,/m/06bz3,Radio+520,/m/07hvw1,Field recordingdiff --git a/backend/requirements-runtime.txt b/backend/requirements-runtime.txtindex 8112b0e..6e1b4f3 100644--- a/backend/requirements-runtime.txt+++ b/backend/requirements-runtime.txt@@ -4,6 +4,9 @@# via backend/requirements.txt.opencv-python-headless>=4.8.1.78numpy>=1.24.0+# Audio-event detection (YAMNet laughter/reaction channel) runs on ONNX Runtime —+# no torch/TF, so it stays on the hermetic native path.+onnxruntime>=1.16.0Pillow>=10.0.0questionary>=2.0.0python-dotenv>=1.2.2diff --git a/backend/requirements.txt b/backend/requirements.txtindex bb6ae7b..7532eb6 100644--- a/backend/requirements.txt+++ b/backend/requirements.txt@@ -12,6 +12,9 @@ openai-whisper>=20231117opencv-python-headless>=4.8.1.78numpy>=1.24.0+# Audio-event detection (laughter/reaction channel via YAMNet ONNX — no torch/TF)+onnxruntime>=1.16.0+# ThumbnailsPillow>=10.0.0diff --git a/backend/services/audio_events.py b/backend/services/audio_events.pynew file mode 100644index 0000000..ce1a265--- /dev/null+++ b/backend/services/audio_events.py@@ -0,0 +1,222 @@+"""+Audio-event detection (laughter, cheering, applause, screaming) for reaction-based+clip scoring.++Runs YAMNet (AudioSet, 521 classes) as a self-contained ONNX graph via onnxruntime,+which imports neither torch nor tensorflow — so this belongs on the native /+whisper.cpp runtime path. Audio is extracted at 16 kHz mono (the same format+whisper.cpp uses) and fed to the graph as a raw waveform; the log-mel front-end is+baked into the model, so there is no librosa dependency.++A laugh is the strongest language-independent "something funny happened" signal, and+it anti-correlates with speech at the frame level, which makes it a reliable anchor+for extending a clip backwards to the moment that caused the reaction.+"""++import csv+import subprocess+import tempfile+import wave+from pathlib import Path+from typing import Optional, Callable++import numpy as np++from utils.proc import run as proc_run++try:+ import onnxruntime as ort+ _ORT_AVAILABLE = True+except ImportError:+ _ORT_AVAILABLE = False++_MODELS_DIR = Path(__file__).resolve().parents[1] / "models"+_MODEL_PATH = _MODELS_DIR / "yamnet.onnx"+_CLASS_MAP_PATH = _MODELS_DIR / "yamnet_class_map.csv"++# AudioSet display names, grouped into the reaction channels we score on. Laughter is+# spread across fine-grained children in the ontology, so we collapse the whole family.+_LAUGHTER_KEYS = ("laugh", "giggle", "chuckle", "snicker", "chortle")+_CHEER_NAMES = ("Cheering", "Applause", "Whoop")+_SCREAM_NAMES = ("Screaming",)+_SPEECH_NAMES = ("Speech",)++_session = None+_class_index: dict[str, list[int]] = {}+++def _load_class_index() -> dict[str, list[int]]:+ names: dict[int, str] = {}+ with open(_CLASS_MAP_PATH, newline="") as f:+ reader = csv.reader(f)+ next(reader) # header: index,mid,display_name+ for row in reader:+ names[int(row[0])] = row[2]++ def by_substr(keys) -> list[int]:+ return [i for i, n in names.items() if any(k in n.lower() for k in keys)]++ def by_name(wanted) -> list[int]:+ return [i for i, n in names.items() if n in wanted]++ return {+ "laughter": by_substr(_LAUGHTER_KEYS),+ "cheering": by_name(_CHEER_NAMES),+ "screaming": by_name(_SCREAM_NAMES),+ "speech": by_name(_SPEECH_NAMES),+ }+++def _get_session():+ global _session, _class_index+ if not _ORT_AVAILABLE or not _MODEL_PATH.exists():+ return None+ if _session is None:+ _session = ort.InferenceSession(+ str(_MODEL_PATH), providers=["CPUExecutionProvider"]+ )+ _class_index = _load_class_index()+ return _session+++def is_available() -> bool:+ """True if onnxruntime and the YAMNet model are both present."""+ return _ORT_AVAILABLE and _MODEL_PATH.exists()+++def _read_waveform_16k_mono(video_path: str) -> Optional[np.ndarray]:+ """Extract audio as a float32 [-1, 1] mono waveform at 16 kHz via ffmpeg."""+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp:+ cmd = [+ "ffmpeg", "-y", "-i", video_path,+ "-vn", "-ar", "16000", "-ac", "1",+ "-f", "wav", tmp.name, "-loglevel", "error",+ ]+ result = proc_run(cmd, timeout=600, check=False)+ if result.returncode != 0:+ return None+ with wave.open(tmp.name) as w:+ frames = w.readframes(w.getnframes())+ if not frames:+ return None+ return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0+++def extract_audio_events(video_path: str) -> list[dict]:+ """+ Run YAMNet over a video's audio and return per-frame reaction probabilities.++ Returns a list of {time, laughter, cheering, screaming, speech} dicts, one per+ YAMNet frame (~0.48 s hop). Empty list if the model/runtime is unavailable or the+ audio can't be read, so callers degrade gracefully.+ """+ session = _get_session()+ if session is None:+ return []++ waveform = _read_waveform_16k_mono(video_path)+ if waveform is None or waveform.size == 0:+ return []++ input_name = session.get_inputs()[0].name+ scores = session.run(None, {input_name: waveform})[0] # [frames, 521]+ n_frames = scores.shape[0]+ if n_frames == 0:+ return []++ duration = waveform.size / 16000.0+ hop = duration / n_frames++ def channel_max(keys: list[int]) -> np.ndarray:+ return scores[:, keys].max(axis=1) if keys else np.zeros(n_frames)++ laughter = channel_max(_class_index["laughter"])+ cheering = channel_max(_class_index["cheering"])+ screaming = channel_max(_class_index["screaming"])+ speech = channel_max(_class_index["speech"])++ return [+ {+ "time": round(i * hop, 2),+ "laughter": round(float(laughter[i]), 3),+ "cheering": round(float(cheering[i]), 3),+ "screaming": round(float(screaming[i]), 3),+ "speech": round(float(speech[i]), 3),+ }+ for i in range(n_frames)+ ]+++def _reaction_level(event: dict) -> float:+ """Combined reaction strength for one frame: the loudest reaction channel."""+ return max(event["laughter"], event["cheering"], event["screaming"])+++def compute_event_scores(+ events_data: list[dict],+ segments: list[dict],+) -> list[float]:+ """+ Per-segment reaction score (0-10) from the peak reaction within each segment.++ Unlike audio energy (RMS in dB, only meaningful relative to a video's own+ baseline), YAMNet probabilities are absolute-calibrated in [0, 1], so the score is+ a direct scaling of the peak rather than a z-score. A brief chuckle registers+ ~0.25, a hearty laugh higher; peaks stand out sharply against a ~0 baseline.+ """+ if not events_data:+ return [0.0] * len(segments)++ scores = []+ for seg in segments:+ seg_start = seg.get("start", 0)+ seg_end = seg.get("end", 0)+ levels = [+ _reaction_level(e)+ for e in events_data+ if seg_start <= e["time"] <= seg_end+ ]+ peak = max(levels) if levels else 0.0+ scores.append(round(min(10.0, peak * 12.0), 2))++ return scores+++def get_event_profile(+ video_path: str,+ segments: list[dict],+ progress_callback: Optional[Callable] = None,+ reaction_threshold: float = 0.15,+) -> dict:+ """+ Full pipeline: detect audio events and score all segments.++ Returns {events_data, segment_scores, reaction_times} where reaction_times are the+ timestamps of frames whose reaction level clears reaction_threshold — the anchors a+ reaction-based detector expands backwards from.+ """+ if not is_available():+ return {"events_data": [], "segment_scores": [0.0] * len(segments), "reaction_times": []}++ if progress_callback:+ progress_callback(0, "Detecting laughter and reactions...")++ events_data = extract_audio_events(video_path)++ if progress_callback:+ progress_callback(70, "Scoring segments by reaction...")++ segment_scores = compute_event_scores(events_data, segments)++ reaction_times = [+ e["time"] for e in events_data if _reaction_level(e) >= reaction_threshold+ ]++ if progress_callback:+ progress_callback(100, "Reaction analysis complete")++ return {+ "events_data": events_data,+ "segment_scores": segment_scores,+ "reaction_times": reaction_times,+ }diff --git a/backend/services/profiles.py b/backend/services/profiles.pynew file mode 100644index 0000000..c5450da--- /dev/null+++ b/backend/services/profiles.py@@ -0,0 +1,80 @@+"""Content profiles — which signals drive moment detection, per content type.++A ContentProfile is orthogonal to FormatSpec: FormatSpec is a render-time aspect-ratio+decision, a ContentProfile is a detection-time decision about which signal channels+matter and how candidate moments are generated. A podcast is selected transcript-first+(the LLM reads dialogue); a party video has no useful transcript and is driven by+laughter/energy peaks. Both are the same fusion engine with different channel weights.++Weights are starting points, tuned with real footage. `candidate_source` picks how+moments are generated: "llm" keeps the existing transcript-first selector (podcast),+"saliency" peak-picks a fused signal curve (party/action, works with no transcript).+"""++from dataclasses import dataclass+++@dataclass(frozen=True)+class ContentProfile:+ name: str+ candidate_source: str # "llm" | "saliency"+ channel_weights: dict[str, float]+ # Reaction-based expansion: anchor on a laugh/cheer onset and extend backwards.+ reaction_lookback_sec: float+ reaction_payoff_sec: float+ # Peak-pick tuning for the saliency candidate source (per-video normalized units).+ peak_min_gap_sec: float+ peak_top_percentile: float+++_CHANNELS = ("transcript_semantic", "prosody", "audio_event", "energy", "motion", "face_reaction")+++def _weights(**kw) -> dict[str, float]:+ return {ch: float(kw.get(ch, 0.0)) for ch in _CHANNELS}+++PROFILES = {+ "podcast": ContentProfile(+ name="podcast",+ candidate_source="llm",+ channel_weights=_weights(+ transcript_semantic=0.5, prosody=0.2, audio_event=0.15, energy=0.1, face_reaction=0.05+ ),+ reaction_lookback_sec=8.0,+ reaction_payoff_sec=2.0,+ peak_min_gap_sec=15.0,+ peak_top_percentile=0.15,+ ),+ "party": ContentProfile(+ name="party",+ candidate_source="saliency",+ channel_weights=_weights(+ audio_event=0.4, prosody=0.2, energy=0.2, motion=0.1, face_reaction=0.1+ ),+ reaction_lookback_sec=8.0,+ reaction_payoff_sec=2.0,+ peak_min_gap_sec=8.0,+ peak_top_percentile=0.15,+ ),+ "action": ContentProfile(+ name="action",+ candidate_source="saliency",+ channel_weights=_weights(+ audio_event=0.3, motion=0.2, prosody=0.2, energy=0.2, transcript_semantic=0.1+ ),+ reaction_lookback_sec=6.0,+ reaction_payoff_sec=2.0,+ peak_min_gap_sec=8.0,+ peak_top_percentile=0.15,+ ),+}++DEFAULT_PROFILE = "podcast"+++def get_profile(name: str | None) -> ContentProfile:+ import sys+ if name is not None and name not in PROFILES:+ print(f"[profiles] unknown profile {name!r}; using {DEFAULT_PROFILE}", file=sys.stderr)+ return PROFILES.get(name or DEFAULT_PROFILE, PROFILES[DEFAULT_PROFILE])diff --git a/backend/services/transcript_packer.py b/backend/services/transcript_packer.pyindex f311496..cd88f48 100644--- a/backend/services/transcript_packer.py+++ b/backend/services/transcript_packer.py@@ -38,6 +38,8 @@ SILENCE_GAP_REPORT_SEC = 0.6 # list gaps >= this in the silence sectionPHRASE_MAX_SEC = 12.0PHRASE_MAX_CHARS = 160ENERGY_PEAKS_TO_REPORT = 20+REACTIONS_TO_REPORT = 20+REACTION_REPORT_THRESHOLD = 0.15def compute_cache_hash(video_path: str) -> str:@@ -272,6 +274,7 @@ def pack_transcript(transcript: dict,source_label: str,energy_data: Optional[list[dict]] = None,+ events_data: Optional[list[dict]] = None,) -> str:"""Render a packed markdown view of a transcription JSON."""duration = float(transcript.get("duration", 0.0))@@ -348,6 +351,31 @@ def pack_transcript(lines.append(f"[{_fmt_ts(t)}] {rms:.1f}dB{snippet}")lines.append("")+ # Laughter & reactions (optional) — the moment BEFORE a laugh is usually the+ # clippable one, so these timestamps flag where to look for the setup.+ if events_data:+ def level(e):+ return max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0))+ reactions = [e for e in events_data if level(e) >= REACTION_REPORT_THRESHOLD]+ reactions.sort(key=level, reverse=True)+ reactions = reactions[:REACTIONS_TO_REPORT]+ reactions.sort(key=lambda e: e.get("time", 0))+ if reactions:+ lines.append(f"## Laughter & reactions (top {len(reactions)})")+ for e in reactions:+ t = float(e.get("time", 0))+ kind = max(+ ("laughter", "cheering", "screaming"),+ key=lambda k: e.get(k, 0),+ )+ near = _phrase_for_time(phrases, t)+ snippet = ""+ if near:+ txt = near["text"]+ snippet = f' near: "{txt[:70]}{"…" if len(txt) > 70 else ""}"'+ lines.append(f"[{_fmt_ts(t)}] {kind} {e.get(kind, 0):.2f}{snippet}")+ lines.append("")+return "\n".join(lines)@@ -372,10 +400,11 @@ def write_packed(cache_hash: str,source_label: Optional[str] = None,energy_data: Optional[list[dict]] = None,+ events_data: Optional[list[dict]] = None,) -> tuple[str, str]:"""Pack a transcript dict and write to .podcli/packed/<hash>.md."""label = source_label or cache_hash- md = pack_transcript(transcript, label, energy_data=energy_data)+ md = pack_transcript(transcript, label, energy_data=energy_data, events_data=events_data)os.makedirs(_packed_dir(), exist_ok=True)out_path = packed_path_for(cache_hash)with open(out_path, "w", encoding="utf-8") as f:diff --git a/plans/moment-detection.md b/plans/moment-detection.mdnew file mode 100644index 0000000..4665bd6--- /dev/null+++ b/plans/moment-detection.md@@ -0,0 +1,110 @@+# podcli → Generalized moment detection (multi-signal saliency engine)++> Goal: one detector that works across content types. The podcast clip selector and a new "auto-cut the funny/interesting moments" highlighter (party videos, vlogs, action, streams) become **two profiles of one engine**, not two features. Anchor use case that surfaced the need: a folder of party clips, "cut me the funny bits" — any language, no labels. Podcast selection is preserved as the default profile.++## North star++```text+one long video (any content, any language)+ → extract N signal channels (transcript, energy, laughter, prosody, motion, face)+ → per-video-normalize each into a 0..1 saliency curve on a common time grid+ → per-PROFILE weighted fusion → one interestingness curve+ → peak-pick (NMS + prominence) → reaction-expand → snap to natural boundaries+ → candidate moments → (render to formats via the existing fan-out)+```++## The one decision everything hangs on++**Content profile is orthogonal to output format.** `FormatSpec` (9:16 / 16:9 / 1:1) is a *render-time* aspect-ratio decision. A **`ContentProfile`** (podcast / party / action / auto) is a *detection-time* decision about **which signals matter and how candidates are generated**. A moment is detected under a profile, then rendered to one or more formats. Do not overload `FormatSpec.score_key` for this — mirror its pattern in a separate `ContentProfile` spec.++The existing transcript-LLM selector is **one channel** in this engine (the `transcript_semantic` channel), not the whole thing. That is the reframe: today the LLM *is* selection; tomorrow it is the dominant channel for the podcast profile and near-zero for party.++## Why this generalizes (research-backed)++Multi-signal highlight detection is a solved shape: compute a per-modality saliency curve, normalize **per-video** (never global), fuse (weighted sum with soft-OR bias, audio weighted heaviest for human-centric video), peak-pick, and for reaction-driven content extend **backwards from the reaction** to the cause. FunnyNet-W: a funny moment is "an n-second clip *followed by* laughter," audio carries >50% of the decision, optimal look-back ≈ 8s. Cai/Lu/Cai built laughter+applause+cheer detection *for home videos* at ~93% recall. All of it is unsupervised — no labeled data, only pretrained off-the-shelf models. See the audio-model and fusion research threads (this session) for citations.++## Profiles as concrete weight vectors++Channels each emit a per-video-normalized 0..1 curve; a profile is a weight vector + candidate-source + peak params.++| Channel | podcast | party | action | Source status |+|---|---|---|---|---|+| `transcript_semantic` (LLM dialogue/quotability) | **0.5** | 0.0 | 0.1 | REUSE `claude_suggest.py` |+| `prosody` (pitch/energy/rate spikes) | 0.2 | 0.2 | 0.2 | NEW (numpy, no librosa) |+| `audio_event` (laughter/cheer/applause/scream) | 0.15 | **0.4** | **0.3** | NEW (YAMNet-ONNX, no torch) |+| `energy` (RMS, peak-weighted) | 0.1 | 0.2 | 0.2 | REUSE `audio_analyzer.py` |+| `motion` (optical flow + scene-cut density) | 0.0 | 0.1 | **0.2** | PARTIAL (cuts exist; flow NEW) |+| `face_reaction` (mouth activity / smile proxy) | 0.05 | 0.1 | 0.0 | REUSE `face_analysis.py` (crude) |+| **candidate source** | `llm` | `saliency` | `saliency` | — |++Weights are starting points, tuned later. `auto` = a cheap first pass picks the profile: clean speech + 1-2 stable faces → podcast; sparse speech + high motion + crowd/laughter audio → party; sustained high motion → action.++## Two candidate-generation modes (the key to not regressing podcast)++- **`llm` (podcast):** the existing `claude_suggest` flow generates candidates over the whole transcript, exactly as today. The fused signal score blends in at the ranking seam. **When party channels have weight 0, output is byte-for-byte the current behavior.** This is the safety property.+- **`saliency` (party / action):** the fused curve drives candidate generation directly — `find_peaks` → reaction-expand → boundary-snap → candidate windows. Works with **no usable transcript**. The LLM is optional here, used only to title/caption the already-chosen windows.++## Architecture — new modules++1. **`backend/services/audio_events.py`** (NEW, no-torch)+ - YAMNet exported to a **self-contained ONNX graph** (raw 16 kHz waveform in, log-mel baked into the graph → **no librosa needed**). ~15 MB, Apache-2.0. AudioSet classes `Laughter`, `Cheering`, `Applause`, `Screaming` are native outputs (+ Giggle/Belly-laugh/Whoop children).+ - Reuse `transcription_whispercpp._extract_wav` (`-ar 16000 -ac 1`) for audio; **do not import `speaker_detection`** (drags in torch).+ - Returns per-window (≈1 Hz, matching energy) class probabilities → one 0..1 curve per trigger class.+ - Runtime: `onnxruntime`. **First test whether `cv2.dnn` (already loads YuNet `.onnx`) can run yamnet.onnx** — if yes, zero new deps.+ - Ship `backend/models/yamnet.onnx` + a **dev-only** `scripts/export_yamnet.py` (uses tf2onnx; never shipped or imported at runtime).++2. **`backend/services/saliency.py`** (NEW)+ - `normalize_per_video(curve)` — robust z-score via numpy median/MAD, or percentile-rank. **Never global.**+ - `fuse(channels, profile)` — weighted sum with soft-OR bias.+ - `pick_peaks(curve, profile)` — **numpy reimplementation of `find_peaks`** (~40 lines: height floor `mean + kσ`, min-distance NMS by descending height, prominence filter, min-width). Avoids adding scipy. Keep peaks above the top ~10-15th percentile of the video's **own** prominence distribution (adaptive count).+ - `reaction_expand(peak, events)` — if the peak's driver is an `audio_event` class, move anchor to reaction onset, extend ~8 s back, keep 1-3 s of payoff.+ - `snap_boundaries(window)` — snap start/end to nearest {Whisper sentence-end, silence gap >300 ms, scene cut, motion-quiet local min} within ±500 ms.++3. **`backend/services/profiles.py`** (NEW) — `ContentProfile` dataclass (name, `channel_weights`, `candidate_source`, peak params, reaction-expand params, boundary sources) mirroring `FormatSpec`; `get_profile(name)`; `auto_detect(signals)`.++4. **`motion` channel** (Phase 3) — `cv2.calcOpticalFlowFarneback` on sampled frames (**OpenCV already present**) + reuse `local_reframe.count_scene_cuts` / `clip_generator._detect_scene_cuts`.++5. **`prosody` channel** (Phase 2/3) — numpy F0 (autocorrelation) + short-term energy + speech-rate vs speaker baseline. No librosa.++## Dependency deltas (no-torch native path)++- **ADD** `onnxruntime` to `requirements-runtime.txt` — the only new hard dep. **Settled by spike:** OpenCV-DNN *cannot* run the YAMNet graph (rejects the dynamic-shape mel front-end: `dynamic 'zero' shapes are not supported`), so the zero-new-dep route is out. `onnxruntime` still pulls in no torch/TF.+- **NO** librosa (self-contained waveform-in graph, confirmed), **NO** torch/TF at runtime.+- `scipy` is present in the dev venv (so `find_peaks` works there) but is **not** in `requirements-runtime.txt` — either add it or keep the numpy peak-pick reimpl for the hermetic runtime. Decide at Phase 2.+- Optical flow + face already covered by `opencv-python-headless`.+- `yamnet.onnx` weight (~16 MB) committed to `backend/models/`; no export step needed — a self-contained waveform-in export already exists at HF `zeropointnine/yamnet-onnx` (input `waveform` [dynamic], outputs `[frames,521]` scores + `[frames,1024]` embeddings + `[frames,64]` mel). Verify its license before committing; keep a dev-only `scripts/export_yamnet.py` (tf2onnx) as the reproducible fallback.++## Spike results (validated this session)++Ran the self-contained YAMNet ONNX on real podcli audio (a 38 s Deeptech Decoded clip, extracted with podcli's exact `-ar 16000 -ac 1`):+- **No-torch confirmed:** `torch`/`tensorflow` absent from the process before and after inference (onnxruntime only).+- **Correct classification:** Speech 0.774 (dominant, correct for a talking clip), Silence 0.186, and Laughter/Cheering/Applause/Screaming all exactly 0.000 — correct negative control (serious monologue, no laughter). Mel front-end baked in the graph → featurization provably correct, no code to own.+- **Negligible cost:** 98 ms for 38 s of audio (~380× realtime); a 3 h video ≈ 28 s of audio-event analysis.+- **Positive reaction detection confirmed on podcast content.** Swept 32 real clips: the detector ranked the one comedic show + two chuckle moments above the dry technical majority (which scored exactly 0.00). Zooming in, laughter fires as a **sharp 1-2 frame spike anti-correlated with Speech** (e.g. laugh=0.23 while speech drops 1.00→0.11 at 13.5s, then back to speech) — the exact reaction signature the 8s-backward-expansion relies on. Absolute values are modest (0.23-0.25 for brief chuckles in pre-cut clips) but that is a non-issue: a spike on a 0.00 baseline is trivially caught by **prominence-based peak-picking**, which is why per-video normalization (not absolute thresholds) is mandated above.+- **Still worth a check (needs real party footage):** amplitude/robustness on noisy handheld party audio with music and overlapping speech — YAMNet's known soft spot. If weak, AST ONNX (`onnx-community`, pre-exported, mAP 0.485) is the drop-in fallback; needs a 128-bin log-mel numpy front-end.++## Integration seams (from the code map)++1. **Detect-once hub** — `backend/cli.py` ~658-720 (and MCP twin `backend/main.py:handle_suggest_clips` ~358). Energy is *already* computed once here (`get_energy_profile`) and `face_map` extracted. All new channels compute here; fusion produces the saliency curve.+2. **Scoring merge** — `claude_suggest.suggest_with_claude` normalization (~999) + `_select_top_by_score` (~687). Blend the LLM score in as the `transcript_semantic` channel; key the weights off the active `ContentProfile`.+3. **Transcript-packer co-location** — `transcript_packer.pack_transcript` (~274) already flags top-RMS moments for the in-chat MCP agent. Add laughter/event/motion flags so the conversational path also sees them.++## Contract-change tax++A new `profile` param threads the **same ~12 hops the `format` field did**: `suggest-clips.handler.ts` inputSchema, `src/models/index.ts` types, `src/server.ts` Zod (dual-declaration), `src/ui/web-server.ts` `styledClips` whitelist, `python-executor` stdin, `main.py` params, and `cli.py`'s `generate_clip` call sites — or it silently reverts to the default. Follow the `format` precedent exactly.++## Phasing++- **Phase 0 — profile scaffolding, zero behavior change.** Add `ContentProfile` abstraction; thread the `profile` param through the ~12 hops; `default = podcast` reproduces current selection exactly. **Gate: existing test suite green; same clips out for a fixed transcript.**+- **Phase 1 — audio-event channel (the isolated valuable core).** YAMNet-ONNX laughter/cheer/applause/scream computed in the detect-once hub. Feeds podcast ranking as a labeled signal (laughs already spike energy; now they're *named*) and lays the party foundation. **Gate: laughter timestamps validated on a sample clip; podcast output unchanged unless the channel is given weight.**+- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** `saliency.py` fusion + numpy peak-pick + reaction-expand (8 s) + boundary-snap. Party profile = energy + audio_event + prosody, no transcript, no motion. **Party videos auto-clip end to end. Gate: demo on real party footage.**+- **Phase 3 — visual channels + action profile + multi-file pooling.** Optical flow (OpenCV) + face-reaction channels; action profile; pool peaks across a *folder* of clips and rank globally ("best 15 bits from tonight" across 80 phone videos). **Gate: catches a silent visual gag; folder-level ranking works.**+- **Phase 4 (optional) — highlight reel renderer.** Ordering, pacing, optional music-bed ducking, transitions — a thin renderer atop the detected moments, reusing the clip-render stack.++## Guardrails++- **Per-video normalization is mandatory** — party clips vary wildly in level/room/camera; global normalization is wrong.+- **Podcast profile must reproduce current behavior** when party channels are weight 0. Ship behind the profile param; default is safe. This is the non-regression contract.+- **`find_peaks` `distance` param is the NMS / min-gap** — it deletes lower peaks within the gap automatically. Adaptive clip count = top percentile of the video's own prominence distribution, not an absolute N.+- **YAMNet ONNX must never import torch/TF at runtime.** The export step is dev/CI only; the native runtime installs `onnxruntime` (or reuses OpenCV-DNN) and feeds raw 16 kHz samples.+- **Reaction-expansion only fires for saliency candidates** whose trigger is an audio-event class — never rewrite LLM-chosen podcast windows.diff --git a/tests/test_audio_events.py b/tests/test_audio_events.pynew file mode 100644index 0000000..cea4e63--- /dev/null+++ b/tests/test_audio_events.py@@ -0,0 +1,80 @@+"""Tests for backend.services.audio_events scoring and profile scaffolding."""++import os+import sys+import unittest++ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+BACKEND_ROOT = os.path.join(ROOT, "backend")+if BACKEND_ROOT not in sys.path:+ sys.path.insert(0, BACKEND_ROOT)++from services import audio_events as ae+from services import profiles+++def _event(t, laughter=0.0, cheering=0.0, screaming=0.0, speech=0.0):+ return {+ "time": t,+ "laughter": laughter,+ "cheering": cheering,+ "screaming": screaming,+ "speech": speech,+ }+++class ComputeEventScoresTests(unittest.TestCase):+ def test_empty_events_returns_zeros(self):+ segments = [{"start": 0, "end": 10}, {"start": 10, "end": 20}]+ self.assertEqual(ae.compute_event_scores([], segments), [0.0, 0.0])++ def test_empty_segments_returns_empty(self):+ self.assertEqual(ae.compute_event_scores([_event(1, laughter=0.5)], []), [])++ def test_segment_with_laugh_scores_higher_than_silent(self):+ events = [_event(2, laughter=0.5, speech=0.1), _event(30, speech=1.0)]+ scores = ae.compute_event_scores(+ events, [{"start": 0, "end": 10}, {"start": 20, "end": 40}]+ )+ self.assertGreater(scores[0], scores[1])+ self.assertEqual(scores[1], 0.0)++ def test_score_uses_loudest_reaction_channel(self):+ # cheering should count even with zero laughter+ events = [_event(1, cheering=0.5)]+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])+ self.assertGreater(scores[0], 0.0)++ def test_score_is_clamped_to_ten(self):+ events = [_event(1, laughter=1.0)]+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])+ self.assertLessEqual(scores[0], 10.0)++ def test_speech_alone_is_not_a_reaction(self):+ events = [_event(1, speech=1.0)]+ scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}])+ self.assertEqual(scores[0], 0.0)+++class ProfileTests(unittest.TestCase):+ def test_default_is_podcast_and_llm_sourced(self):+ p = profiles.get_profile(None)+ self.assertEqual(p.name, "podcast")+ self.assertEqual(p.candidate_source, "llm")++ def test_unknown_profile_falls_back_to_default(self):+ self.assertEqual(profiles.get_profile("nonsense").name, profiles.DEFAULT_PROFILE)++ def test_party_is_saliency_sourced_and_ignores_transcript(self):+ p = profiles.get_profile("party")+ self.assertEqual(p.candidate_source, "saliency")+ self.assertEqual(p.channel_weights["transcript_semantic"], 0.0)+ self.assertGreater(p.channel_weights["audio_event"], 0.0)++ def test_podcast_weights_are_transcript_dominant(self):+ w = profiles.get_profile("podcast").channel_weights+ self.assertEqual(max(w, key=w.get), "transcript_semantic")+++if __name__ == "__main__":+ unittest.main()