A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(music): a real SoundFont comes out as it sounds - zones it plays together are one kept, an instrument set whole octaves from its recordings comes out at their pitch and says so, and the pitch heard in a recording is right for a high note at a low rate and counts its tuning
yhh4678df850bd2
diff --git a/desktop b/desktopindex 5984d99ba..f60bc1230 160000--- a/desktop+++ b/desktop@@ -1 +1 @@-Subproject commit 5984d99ba081186a0b717e48b11a06dfca798acf+Subproject commit f60bc12300a662c52016b603db2d4936bfd0dae3diff --git a/pipeline/src/music/musicRender.ts b/pipeline/src/music/musicRender.tsindex 5de9d1f99..43d977ae8 100644--- a/pipeline/src/music/musicRender.ts+++ b/pipeline/src/music/musicRender.ts@@ -273,7 +273,30 @@ export function samplePitch(data: SampleData, near: number): number | null {if (!(top > 0.6)) return null;let at = scores.findIndex((score) => score[1] >= top * 0.92);while (at + 1 < scores.length && scores[at + 1]![1] > scores[at]![1]) at++;- return data.rate / scores[at]![0];+ // A high note in a recording made at a low rate has a period of a dozen samples and a half: no whole slide fits+ // it, and twice the period, which is whole, fits better. So a half and a third of the slide found are tried too,+ // between the samples, and the shortest that fits nearly as well is the pitch.+ const fits = (lag: number): number => {+ let sum = 0, a = 0, b = 0;+ const whole = Math.floor(lag), part = lag - whole;+ for (let n = 0; n < size; n += 2) {+ const x = samples[start + n]!, y = samples[start + n + whole]! * (1 - part) + samples[start + n + whole + 1]! * part;+ sum += x * y; a += x * x; b += y * y;+ }+ return sum / Math.sqrt(a * b + 1e-12);+ };+ let lag = scores[at]![0];+ // Just off the whole number, a little either side, where the fit is best.+ const best = (centre: number): [number, number] => { let found: [number, number] = [centre, -1]; for (let d = -0.5; d <= 0.5; d += 0.05) { const f = fits(centre + d); if (f > found[1]) found = [centre + d, f]; } return found;+ };+ let [fine, fit] = best(lag);+ for (const part of [3, 2]) {+ if (lag / part < 4 || data.rate / (lag / part) > near * 4.2) continue;+ const [shorter, shorterFit] = best(lag / part);+ if (shorterFit >= fit * 0.9) { [fine, fit] = [shorter, shorterFit]; break; }+ }+ lag = fine;+ return data.rate / lag;}const TAIL_SECONDS = 4;@@ -426,11 +449,15 @@ export function renderMusic(source: MusicSource, given: Readonly<Record<string,for (const name of names) {const wrong: { file: string; said: number; is: number }[] = [];const zones = source.instruments[name]!.samples ?? [];+ const heard = new Set<string>();for (const zone of zones) {const data = given[zone.file];- if (!data) continue;+ // A recording used for several ranges is listened to once.+ if (!data || heard.has(`${zone.file}|${zone.note}`)) continue;+ heard.add(`${zone.file}|${zone.note}`);const said = midiOf(zone.note)!;- const hz = 440 * Math.pow(2, (said - 69) / 12);+ // Where it is said to be sharp or flat of its note, that is where it is listened for.+ const hz = 440 * Math.pow(2, (said + (zone.tune ?? 0) / 100 - 69) / 12);const found = samplePitch(data, hz);const off = found === null ? 0 : 12 * Math.log2(found / hz);if (Math.abs(off) >= 0.7) wrong.push({ file: zone.file, said, is: Math.round(said + off) });@@ -439,8 +466,8 @@ export function renderMusic(source: MusicSource, given: Readonly<Record<string,const by = (w: { said: number; is: number }): number => w.is - w.said;const semitones = (n: number): string => `${Math.abs(n)} semitone${Math.abs(n) === 1 ? '' : 's'} ${n > 0 ? 'higher' : 'lower'}`;// A whole library out by the same amount is one thing to say, and nearly always an octave: it counts its octaves from another C.- if (wrong.length === zones.length && wrong.length > 1 && wrong.every((w) => by(w) === by(wrong[0]!))) {- notes.push(`"${name}": every one of its ${zones.length} recordings sounds ${semitones(by(wrong[0]!))} than the note it is said to be (${wrong[0]!.file} is said to be ${noteName(wrong[0]!.said)} and sounds as ${noteName(wrong[0]!.is)})${Math.abs(by(wrong[0]!)) === 12 ? ', as a library does that counts its octaves from another C' : ''}: the instrument plays that much out. Move each \`note\` by that much`);+ if (wrong.length === heard.size && wrong.length > 1 && wrong.every((w) => by(w) === by(wrong[0]!))) {+ notes.push(`"${name}": every one of its ${heard.size} recordings sounds ${semitones(by(wrong[0]!))} than the note it is said to be (${wrong[0]!.file} is said to be ${noteName(wrong[0]!.said)} and sounds as ${noteName(wrong[0]!.is)})${Math.abs(by(wrong[0]!)) === 12 ? ', as a library does that counts its octaves from another C' : ''}: the instrument plays that much out. Move each \`note\` by that much`);} else {for (const w of wrong) notes.push(`"${name}": ${w.file} is said to be ${noteName(w.said)} and sounds as ${noteName(w.is)}: every note played from it is ${semitones(by(w))} than written. Set its note to "${noteName(w.is)}"`);}diff --git a/pipeline/src/music/soundFont.ts b/pipeline/src/music/soundFont.tsindex 76cf47822..b1ecdd85c 100644--- a/pipeline/src/music/soundFont.ts+++ b/pipeline/src/music/soundFont.ts@@ -10,7 +10,12 @@* `.wav` files, after which the SoundFont is not needed: the project holds what it uses and no more.** What a SoundFont can say and this does not carry over: its filters and envelopes beyond attack and release,- * its modulators, and the second side of a recording made in stereo.+ * its modulators, and the second side of a recording made in stereo. Two things it does are undone on the way+ * out, and said. Where it plays several recordings at once for one note (a second rank of pipes, a string+ * under a hammer) one is kept, since a sample here is one recording to a note. And where it sets an+ * instrument whole octaves from its recordings, the recordings come out at the pitch they are: an instrument+ * here goes down an octave by its `octave`, and one whose middle C sounded the C below would be out of tune+ * with everything written beside it.*/import type { MusicSample } from './musicSource';import type { SampleData } from './musicRender';@@ -28,7 +33,7 @@ export interface SoundFontPreset {high: number;}-interface Zone { sample: number; keyLow: number; keyHigh: number; velLow: number; velHigh: number; root: number; cents: number; loop: boolean; offsets: [number, number, number, number]; attack: number | null; release: number | null }+interface Zone { sample: number; keyLow: number; keyHigh: number; velLow: number; velHigh: number; root: number; cents: number; octaves: number; loop: boolean; offsets: [number, number, number, number]; attack: number | null; release: number | null }interface SampleHeader { name: string; start: number; end: number; loopStart: number; loopEnd: number; rate: number; pitch: number; correction: number; type: number }export interface SoundFont {@@ -122,10 +127,13 @@ export function readSoundFont(bytes: Uint8Array): SoundFont {const [keyLow, keyHigh, velLow, velHigh] = [Math.max(ik[0], pk[0]), Math.min(ik[1], pk[1]), Math.max(iv[0], pv[0]), Math.min(iv[1], pv[1])];if (keyLow > keyHigh || velLow > velHigh) continue;const rootKey = get(GEN.rootKey);+ const coarse = signed(get(GEN.coarseTune) ?? 0) + add(GEN.coarseTune);mine.push({sample: iz.get(GEN.sampleId)!, keyLow, keyHigh, velLow, velHigh,root: rootKey !== undefined && signed(rootKey) >= 0 ? rootKey : header.pitch <= 127 ? header.pitch : 60,- cents: (signed(get(GEN.coarseTune) ?? 0) + add(GEN.coarseTune)) * 100 + signed(get(GEN.fineTune) ?? 0) + add(GEN.fineTune) + header.correction,+ // Whole octaves are kept apart from the rest: they are where the instrument is set, not how the recording is tuned.+ octaves: coarse % 12 === 0 ? coarse / 12 : 0,+ cents: (coarse % 12 === 0 ? 0 : coarse) * 100 + signed(get(GEN.fineTune) ?? 0) + add(GEN.fineTune) + header.correction,loop: ((get(GEN.sampleModes) ?? 0) & 1) === 1,offsets: [signed(get(GEN.startOffset) ?? 0) + signed(get(GEN.startCoarse) ?? 0) * 32768, signed(get(GEN.endOffset) ?? 0) + signed(get(GEN.endCoarse) ?? 0) * 32768,@@ -163,8 +171,15 @@ export interface LiftedInstrument {export function liftSoundFontPreset(font: SoundFont, index: number, dir: string): LiftedInstrument {const preset = font.presets[index];if (!preset) throw new Error(`there is no instrument ${index} in this SoundFont; it has ${font.presets.length}, numbered from 0`);- const zones = font.zones[index]!;- if (zones.length === 0) throw new Error(`"${preset.name}" has no recordings that can be read`);+ const all = font.zones[index]!;+ if (all.length === 0) throw new Error(`"${preset.name}" has no recordings that can be read`);+ // Zones that share a note and a hardness are played together by a SoundFont. One is kept: the one set nearest+ // the pitch of its recording, and of those the first.+ const zones: typeof all = [];+ for (const zone of [...all].sort((a, b) => Math.abs(a.octaves) - Math.abs(b.octaves))) {+ if (!zones.some((k) => k.keyLow <= zone.keyHigh && zone.keyLow <= k.keyHigh && k.velLow <= zone.velHigh && zone.velLow <= k.velHigh)) zones.push(zone);+ }+ zones.sort((a, b) => a.keyLow - b.keyLow || a.velLow - b.velLow);if (zones.length > 256) throw new Error(`"${preset.name}" is ${zones.length} recordings, and an instrument takes 256`);const notes: string[] = [];const files: Record<string, SampleData> = {};@@ -211,6 +226,9 @@ export function liftSoundFontPreset(font: SoundFont, index: number, dir: string)const middle = (list: number[]): number => [...list].sort((a, b) => a - b)[Math.floor(list.length / 2)]!;const attack = attacks.length ? Math.min(2, middle(attacks)) : 0;const release = releases.length ? Math.max(0.05, Math.min(4, middle(releases))) : 0;+ if (zones.length < all.length) notes.push(`this SoundFont plays ${all.length - zones.length} of its ${all.length} zones on top of others, several recordings to one note; one is kept for each note, so it is thinner here than there`);+ const set = zones.filter((z) => z.octaves !== 0);+ if (set.length) notes.push(`${set.length === zones.length ? 'it is' : `${set.length} of its zones are`} set ${Math.abs(set[0]!.octaves)} octave${Math.abs(set[0]!.octaves) === 1 ? '' : 's'} ${set[0]!.octaves < 0 ? 'down' : 'up'} in the SoundFont; the recordings are taken out at the pitch they are, so give the instrument octave: ${set[0]!.octaves} to have it where the SoundFont had it`);if (preset.bank === 128) notes.push(`"${preset.name}" is a drum kit: each key is another drum, so a step plays the drum on that note and not a pitch`);return {instrument: { samples, ...(attack > 0.01 ? { attack: Math.round(attack * 1000) / 1000 } : {}), ...(release > 0 ? { release: Math.round(release * 1000) / 1000 } : {}) },diff --git a/pipeline/tests/music.test.ts b/pipeline/tests/music.test.tsindex ac58f7c3d..dda4649e9 100644--- a/pipeline/tests/music.test.ts+++ b/pipeline/tests/music.test.ts@@ -302,6 +302,9 @@ describe('an instrument made of recordings', () => {expect(samplePitch(rich(220), 110)).toBeCloseTo(220, -0.5);expect(samplePitch(rich(220), 440)).toBeCloseTo(220, -0.5);let seed = 12345;+ // A high note recorded at a low rate: its period is twelve samples and a half, and twice that is not its pitch.+ const coarse = { rate: 11025, channels: [Float32Array.from({ length: 2874 }, (_, n) => { const t = (2 * Math.PI * 883 * n) / 11025; return 0.4 * Math.sin(t) + 0.1 * Math.sin(3 * t); })] };+ expect(samplePitch(coarse, 880)).toBeCloseTo(883, -1);const noise = { rate: 44100, channels: [Float32Array.from({ length: 44100 }, () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 2147483648 - 1; })] };expect(samplePitch(noise, 220)).toBeNull();diff --git a/pipeline/tests/soundfont.test.ts b/pipeline/tests/soundfont.test.tsindex 6803c97f5..28d26e557 100644--- a/pipeline/tests/soundfont.test.ts+++ b/pipeline/tests/soundfont.test.ts@@ -139,3 +139,34 @@ describe('a SoundFont read for its instruments', () => {expect(() => liftSoundFontPreset(readSoundFont(FONT), 7, 'k')).toThrow(/there is no instrument 7 in this SoundFont; it has 2, numbered from 0/);});});++describe('what a SoundFont does that an instrument of samples does not', () => {+ const TWO: Recording[] = [{ name: 'Pipe', hz: 220, rate: 44100, frames: 8820, pitch: 57 }, { name: 'Pipe Low', hz: 110, rate: 44100, frames: 8820, pitch: 45 }];++ it('keeps one of the recordings it plays together for a note, the one nearest its own pitch', () => {+ // Two ranks over the same keys: the second an octave down. And a third zone above them both, on its own.+ const font = soundFont(TWO, [+ [[GEN.keyRange, range(0, 80)], [51, -12], [GEN.sampleId, 0]],+ [[GEN.keyRange, range(0, 80)], [GEN.sampleId, 1]],+ [[GEN.keyRange, range(81, 127)], [GEN.sampleId, 0]],+ ], [{ name: 'Two Ranks', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);+ const lifted = liftSoundFontPreset(readSoundFont(font), 0, 'o');+ expect(lifted.instrument.samples).toEqual([+ { file: 'o/Pipe_Low.wav', note: 'A2', low: 'C-1', high: 'G#5' },+ { file: 'o/Pipe.wav', note: 'A3', low: 'A5', high: 'G9' },+ ]);+ expect(lifted.notes.join()).toMatch(/plays 1 of its 3 zones on top of others, several recordings to one note; one is kept/);+ expect(Object.keys(lifted.files)).toEqual(['o/Pipe_Low.wav', 'o/Pipe.wav']);+ });++ it('takes an instrument set an octave down out at the pitch its recordings are, and says so', () => {+ const font = soundFont(TWO, [[[51, -12], [GEN.sampleId, 0]]], [{ name: 'Sub', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);+ const lifted = liftSoundFontPreset(readSoundFont(font), 0, 'o');+ // The recording is of the A below middle C, and that is the note it is given: not the A above, as an octave of tuning would have it.+ expect(lifted.instrument.samples).toEqual([{ file: 'o/Pipe.wav', note: 'A3' }]);+ expect(lifted.notes).toEqual(['it is set 1 octave down in the SoundFont; the recordings are taken out at the pitch they are, so give the instrument octave: -1 to have it where the SoundFont had it']);+ // Tuning that is not whole octaves is the recording being another note, and moves it.+ const fifth = soundFont(TWO, [[[51, 7], [GEN.sampleId, 0]]], [{ name: 'Up', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);+ expect(liftSoundFontPreset(readSoundFont(fifth), 0, 'o').instrument.samples).toEqual([{ file: 'o/Pipe.wav', note: 'D3' }]);+ });+});diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjsindex bfdf6a885..c237db50c 100644--- a/tools/releaseNotes.mjs+++ b/tools/releaseNotes.mjs@@ -4626,6 +4626,10 @@ export const NOTED = {{"note": "An AI can make background music."},"feat(music): soundfont_read lists a SoundFont and takes an instrument out of it, and a sound's sample layers read their recordings - the tool text says dynamics, turns and where recordings come from":{"note": "An AI can make background music."},+ "fix(music): a real SoundFont comes out as it sounds - zones it plays together are one kept, an instrument set whole octaves from its recordings comes out at their pitch and says so, and the pitch heard in a recording is right for a high note at a low rate and counts its tuning":+ {"note": "An AI can make background music."},+ "docs(music): soundfont_read says what a SoundFont does that is undone on the way out":+ {"note": "An AI can make background music."},"feat(sky): a sky of your own shades the ground under its clouds - defineSkyEffect takes groundShadow and a second function, skyShadow(world, sun), what is left of the light at a world position, and the sky pass multiplies every surface it finds by it, as Sky Atmosphere does with its own clouds":{"note": "A sky of your own shades the ground under its clouds."},"fix(terrain): a colour map and an occlusion map lie the way the heightmap does - Terrain.colorMap, Terrain.occlusionMap and Foliage.colorMap were sampled with their rows turned over, so a picture painted to match a heightmap, a splatmap or a density map came out mirrored north to south under them; an image over an area now has its first row along the least z whoever reads it":