Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(music): a real SoundFont comes out as it sounds - zones it plays together are one kept, an instrument set whole octaves from its recordings comes out at their pitch and says so, and the pitch heard in a recording is right for a high note at a low rate and counts its tuning

yhh4678df850bd2
diff --git a/desktop b/desktop
index 5984d99ba..f60bc1230 160000
--- a/desktop
+++ b/desktop
@@ -1 +1 @@
-Subproject commit 5984d99ba081186a0b717e48b11a06dfca798acf
+Subproject commit f60bc12300a662c52016b603db2d4936bfd0dae3
diff --git a/pipeline/src/music/musicRender.ts b/pipeline/src/music/musicRender.ts
index 5de9d1f99..43d977ae8 100644
--- a/pipeline/src/music/musicRender.ts
+++ b/pipeline/src/music/musicRender.ts
@@ -273,7 +273,30 @@ export function samplePitch(data: SampleData, near: number): number | null {
if (!(top > 0.6)) return null;
let at = scores.findIndex((score) => score[1] >= top * 0.92);
while (at + 1 < scores.length && scores[at + 1]![1] > scores[at]![1]) at++;
- return data.rate / scores[at]![0];
+ // A high note in a recording made at a low rate has a period of a dozen samples and a half: no whole slide fits
+ // it, and twice the period, which is whole, fits better. So a half and a third of the slide found are tried too,
+ // between the samples, and the shortest that fits nearly as well is the pitch.
+ const fits = (lag: number): number => {
+ let sum = 0, a = 0, b = 0;
+ const whole = Math.floor(lag), part = lag - whole;
+ for (let n = 0; n < size; n += 2) {
+ const x = samples[start + n]!, y = samples[start + n + whole]! * (1 - part) + samples[start + n + whole + 1]! * part;
+ sum += x * y; a += x * x; b += y * y;
+ }
+ return sum / Math.sqrt(a * b + 1e-12);
+ };
+ let lag = scores[at]![0];
+ // Just off the whole number, a little either side, where the fit is best.
+ const best = (centre: number): [number, number] => { let found: [number, number] = [centre, -1]; for (let d = -0.5; d <= 0.5; d += 0.05) { const f = fits(centre + d); if (f > found[1]) found = [centre + d, f]; } return found;
+ };
+ let [fine, fit] = best(lag);
+ for (const part of [3, 2]) {
+ if (lag / part < 4 || data.rate / (lag / part) > near * 4.2) continue;
+ const [shorter, shorterFit] = best(lag / part);
+ if (shorterFit >= fit * 0.9) { [fine, fit] = [shorter, shorterFit]; break; }
+ }
+ lag = fine;
+ return data.rate / lag;
}
const TAIL_SECONDS = 4;
@@ -426,11 +449,15 @@ export function renderMusic(source: MusicSource, given: Readonly<Record<string,
for (const name of names) {
const wrong: { file: string; said: number; is: number }[] = [];
const zones = source.instruments[name]!.samples ?? [];
+ const heard = new Set<string>();
for (const zone of zones) {
const data = given[zone.file];
- if (!data) continue;
+ // A recording used for several ranges is listened to once.
+ if (!data || heard.has(`${zone.file}|${zone.note}`)) continue;
+ heard.add(`${zone.file}|${zone.note}`);
const said = midiOf(zone.note)!;
- const hz = 440 * Math.pow(2, (said - 69) / 12);
+ // Where it is said to be sharp or flat of its note, that is where it is listened for.
+ const hz = 440 * Math.pow(2, (said + (zone.tune ?? 0) / 100 - 69) / 12);
const found = samplePitch(data, hz);
const off = found === null ? 0 : 12 * Math.log2(found / hz);
if (Math.abs(off) >= 0.7) wrong.push({ file: zone.file, said, is: Math.round(said + off) });
@@ -439,8 +466,8 @@ export function renderMusic(source: MusicSource, given: Readonly<Record<string,
const by = (w: { said: number; is: number }): number => w.is - w.said;
const semitones = (n: number): string => `${Math.abs(n)} semitone${Math.abs(n) === 1 ? '' : 's'} ${n > 0 ? 'higher' : 'lower'}`;
// A whole library out by the same amount is one thing to say, and nearly always an octave: it counts its octaves from another C.
- if (wrong.length === zones.length && wrong.length > 1 && wrong.every((w) => by(w) === by(wrong[0]!))) {
- notes.push(`"${name}": every one of its ${zones.length} recordings sounds ${semitones(by(wrong[0]!))} than the note it is said to be (${wrong[0]!.file} is said to be ${noteName(wrong[0]!.said)} and sounds as ${noteName(wrong[0]!.is)})${Math.abs(by(wrong[0]!)) === 12 ? ', as a library does that counts its octaves from another C' : ''}: the instrument plays that much out. Move each \`note\` by that much`);
+ if (wrong.length === heard.size && wrong.length > 1 && wrong.every((w) => by(w) === by(wrong[0]!))) {
+ notes.push(`"${name}": every one of its ${heard.size} recordings sounds ${semitones(by(wrong[0]!))} than the note it is said to be (${wrong[0]!.file} is said to be ${noteName(wrong[0]!.said)} and sounds as ${noteName(wrong[0]!.is)})${Math.abs(by(wrong[0]!)) === 12 ? ', as a library does that counts its octaves from another C' : ''}: the instrument plays that much out. Move each \`note\` by that much`);
} else {
for (const w of wrong) notes.push(`"${name}": ${w.file} is said to be ${noteName(w.said)} and sounds as ${noteName(w.is)}: every note played from it is ${semitones(by(w))} than written. Set its note to "${noteName(w.is)}"`);
}
diff --git a/pipeline/src/music/soundFont.ts b/pipeline/src/music/soundFont.ts
index 76cf47822..b1ecdd85c 100644
--- a/pipeline/src/music/soundFont.ts
+++ b/pipeline/src/music/soundFont.ts
@@ -10,7 +10,12 @@
* `.wav` files, after which the SoundFont is not needed: the project holds what it uses and no more.
*
* What a SoundFont can say and this does not carry over: its filters and envelopes beyond attack and release,
- * its modulators, and the second side of a recording made in stereo.
+ * its modulators, and the second side of a recording made in stereo. Two things it does are undone on the way
+ * out, and said. Where it plays several recordings at once for one note (a second rank of pipes, a string
+ * under a hammer) one is kept, since a sample here is one recording to a note. And where it sets an
+ * instrument whole octaves from its recordings, the recordings come out at the pitch they are: an instrument
+ * here goes down an octave by its `octave`, and one whose middle C sounded the C below would be out of tune
+ * with everything written beside it.
*/
import type { MusicSample } from './musicSource';
import type { SampleData } from './musicRender';
@@ -28,7 +33,7 @@ export interface SoundFontPreset {
high: number;
}
-interface Zone { sample: number; keyLow: number; keyHigh: number; velLow: number; velHigh: number; root: number; cents: number; loop: boolean; offsets: [number, number, number, number]; attack: number | null; release: number | null }
+interface Zone { sample: number; keyLow: number; keyHigh: number; velLow: number; velHigh: number; root: number; cents: number; octaves: number; loop: boolean; offsets: [number, number, number, number]; attack: number | null; release: number | null }
interface SampleHeader { name: string; start: number; end: number; loopStart: number; loopEnd: number; rate: number; pitch: number; correction: number; type: number }
export interface SoundFont {
@@ -122,10 +127,13 @@ export function readSoundFont(bytes: Uint8Array): SoundFont {
const [keyLow, keyHigh, velLow, velHigh] = [Math.max(ik[0], pk[0]), Math.min(ik[1], pk[1]), Math.max(iv[0], pv[0]), Math.min(iv[1], pv[1])];
if (keyLow > keyHigh || velLow > velHigh) continue;
const rootKey = get(GEN.rootKey);
+ const coarse = signed(get(GEN.coarseTune) ?? 0) + add(GEN.coarseTune);
mine.push({
sample: iz.get(GEN.sampleId)!, keyLow, keyHigh, velLow, velHigh,
root: rootKey !== undefined && signed(rootKey) >= 0 ? rootKey : header.pitch <= 127 ? header.pitch : 60,
- cents: (signed(get(GEN.coarseTune) ?? 0) + add(GEN.coarseTune)) * 100 + signed(get(GEN.fineTune) ?? 0) + add(GEN.fineTune) + header.correction,
+ // Whole octaves are kept apart from the rest: they are where the instrument is set, not how the recording is tuned.
+ octaves: coarse % 12 === 0 ? coarse / 12 : 0,
+ cents: (coarse % 12 === 0 ? 0 : coarse) * 100 + signed(get(GEN.fineTune) ?? 0) + add(GEN.fineTune) + header.correction,
loop: ((get(GEN.sampleModes) ?? 0) & 1) === 1,
offsets: [
signed(get(GEN.startOffset) ?? 0) + signed(get(GEN.startCoarse) ?? 0) * 32768, signed(get(GEN.endOffset) ?? 0) + signed(get(GEN.endCoarse) ?? 0) * 32768,
@@ -163,8 +171,15 @@ export interface LiftedInstrument {
export function liftSoundFontPreset(font: SoundFont, index: number, dir: string): LiftedInstrument {
const preset = font.presets[index];
if (!preset) throw new Error(`there is no instrument ${index} in this SoundFont; it has ${font.presets.length}, numbered from 0`);
- const zones = font.zones[index]!;
- if (zones.length === 0) throw new Error(`"${preset.name}" has no recordings that can be read`);
+ const all = font.zones[index]!;
+ if (all.length === 0) throw new Error(`"${preset.name}" has no recordings that can be read`);
+ // Zones that share a note and a hardness are played together by a SoundFont. One is kept: the one set nearest
+ // the pitch of its recording, and of those the first.
+ const zones: typeof all = [];
+ for (const zone of [...all].sort((a, b) => Math.abs(a.octaves) - Math.abs(b.octaves))) {
+ if (!zones.some((k) => k.keyLow <= zone.keyHigh && zone.keyLow <= k.keyHigh && k.velLow <= zone.velHigh && zone.velLow <= k.velHigh)) zones.push(zone);
+ }
+ zones.sort((a, b) => a.keyLow - b.keyLow || a.velLow - b.velLow);
if (zones.length > 256) throw new Error(`"${preset.name}" is ${zones.length} recordings, and an instrument takes 256`);
const notes: string[] = [];
const files: Record<string, SampleData> = {};
@@ -211,6 +226,9 @@ export function liftSoundFontPreset(font: SoundFont, index: number, dir: string)
const middle = (list: number[]): number => [...list].sort((a, b) => a - b)[Math.floor(list.length / 2)]!;
const attack = attacks.length ? Math.min(2, middle(attacks)) : 0;
const release = releases.length ? Math.max(0.05, Math.min(4, middle(releases))) : 0;
+ if (zones.length < all.length) notes.push(`this SoundFont plays ${all.length - zones.length} of its ${all.length} zones on top of others, several recordings to one note; one is kept for each note, so it is thinner here than there`);
+ const set = zones.filter((z) => z.octaves !== 0);
+ if (set.length) notes.push(`${set.length === zones.length ? 'it is' : `${set.length} of its zones are`} set ${Math.abs(set[0]!.octaves)} octave${Math.abs(set[0]!.octaves) === 1 ? '' : 's'} ${set[0]!.octaves < 0 ? 'down' : 'up'} in the SoundFont; the recordings are taken out at the pitch they are, so give the instrument octave: ${set[0]!.octaves} to have it where the SoundFont had it`);
if (preset.bank === 128) notes.push(`"${preset.name}" is a drum kit: each key is another drum, so a step plays the drum on that note and not a pitch`);
return {
instrument: { samples, ...(attack > 0.01 ? { attack: Math.round(attack * 1000) / 1000 } : {}), ...(release > 0 ? { release: Math.round(release * 1000) / 1000 } : {}) },
diff --git a/pipeline/tests/music.test.ts b/pipeline/tests/music.test.ts
index ac58f7c3d..dda4649e9 100644
--- a/pipeline/tests/music.test.ts
+++ b/pipeline/tests/music.test.ts
@@ -302,6 +302,9 @@ describe('an instrument made of recordings', () => {
expect(samplePitch(rich(220), 110)).toBeCloseTo(220, -0.5);
expect(samplePitch(rich(220), 440)).toBeCloseTo(220, -0.5);
let seed = 12345;
+ // A high note recorded at a low rate: its period is twelve samples and a half, and twice that is not its pitch.
+ const coarse = { rate: 11025, channels: [Float32Array.from({ length: 2874 }, (_, n) => { const t = (2 * Math.PI * 883 * n) / 11025; return 0.4 * Math.sin(t) + 0.1 * Math.sin(3 * t); })] };
+ expect(samplePitch(coarse, 880)).toBeCloseTo(883, -1);
const noise = { rate: 44100, channels: [Float32Array.from({ length: 44100 }, () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 2147483648 - 1; })] };
expect(samplePitch(noise, 220)).toBeNull();
diff --git a/pipeline/tests/soundfont.test.ts b/pipeline/tests/soundfont.test.ts
index 6803c97f5..28d26e557 100644
--- a/pipeline/tests/soundfont.test.ts
+++ b/pipeline/tests/soundfont.test.ts
@@ -139,3 +139,34 @@ describe('a SoundFont read for its instruments', () => {
expect(() => liftSoundFontPreset(readSoundFont(FONT), 7, 'k')).toThrow(/there is no instrument 7 in this SoundFont; it has 2, numbered from 0/);
});
});
+
+describe('what a SoundFont does that an instrument of samples does not', () => {
+ const TWO: Recording[] = [{ name: 'Pipe', hz: 220, rate: 44100, frames: 8820, pitch: 57 }, { name: 'Pipe Low', hz: 110, rate: 44100, frames: 8820, pitch: 45 }];
+
+ it('keeps one of the recordings it plays together for a note, the one nearest its own pitch', () => {
+ // Two ranks over the same keys: the second an octave down. And a third zone above them both, on its own.
+ const font = soundFont(TWO, [
+ [[GEN.keyRange, range(0, 80)], [51, -12], [GEN.sampleId, 0]],
+ [[GEN.keyRange, range(0, 80)], [GEN.sampleId, 1]],
+ [[GEN.keyRange, range(81, 127)], [GEN.sampleId, 0]],
+ ], [{ name: 'Two Ranks', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);
+ const lifted = liftSoundFontPreset(readSoundFont(font), 0, 'o');
+ expect(lifted.instrument.samples).toEqual([
+ { file: 'o/Pipe_Low.wav', note: 'A2', low: 'C-1', high: 'G#5' },
+ { file: 'o/Pipe.wav', note: 'A3', low: 'A5', high: 'G9' },
+ ]);
+ expect(lifted.notes.join()).toMatch(/plays 1 of its 3 zones on top of others, several recordings to one note; one is kept/);
+ expect(Object.keys(lifted.files)).toEqual(['o/Pipe_Low.wav', 'o/Pipe.wav']);
+ });
+
+ it('takes an instrument set an octave down out at the pitch its recordings are, and says so', () => {
+ const font = soundFont(TWO, [[[51, -12], [GEN.sampleId, 0]]], [{ name: 'Sub', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);
+ const lifted = liftSoundFontPreset(readSoundFont(font), 0, 'o');
+ // The recording is of the A below middle C, and that is the note it is given: not the A above, as an octave of tuning would have it.
+ expect(lifted.instrument.samples).toEqual([{ file: 'o/Pipe.wav', note: 'A3' }]);
+ expect(lifted.notes).toEqual(['it is set 1 octave down in the SoundFont; the recordings are taken out at the pitch they are, so give the instrument octave: -1 to have it where the SoundFont had it']);
+ // Tuning that is not whole octaves is the recording being another note, and moves it.
+ const fifth = soundFont(TWO, [[[51, 7], [GEN.sampleId, 0]]], [{ name: 'Up', bank: 0, program: 19, zone: [[GEN.instrument, 0]] }]);
+ expect(liftSoundFontPreset(readSoundFont(fifth), 0, 'o').instrument.samples).toEqual([{ file: 'o/Pipe.wav', note: 'D3' }]);
+ });
+});
diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjs
index bfdf6a885..c237db50c 100644
--- a/tools/releaseNotes.mjs
+++ b/tools/releaseNotes.mjs
@@ -4626,6 +4626,10 @@ export const NOTED = {
{"note": "An AI can make background music."},
"feat(music): soundfont_read lists a SoundFont and takes an instrument out of it, and a sound's sample layers read their recordings - the tool text says dynamics, turns and where recordings come from":
{"note": "An AI can make background music."},
+ "fix(music): a real SoundFont comes out as it sounds - zones it plays together are one kept, an instrument set whole octaves from its recordings comes out at their pitch and says so, and the pitch heard in a recording is right for a high note at a low rate and counts its tuning":
+ {"note": "An AI can make background music."},
+ "docs(music): soundfont_read says what a SoundFont does that is undone on the way out":
+ {"note": "An AI can make background music."},
"feat(sky): a sky of your own shades the ground under its clouds - defineSkyEffect takes groundShadow and a second function, skyShadow(world, sun), what is left of the light at a world position, and the sky pass multiplies every surface it finds by it, as Sky Atmosphere does with its own clouds":
{"note": "A sky of your own shades the ground under its clouds."},
"fix(terrain): a colour map and an occlusion map lie the way the heightmap does - Terrain.colorMap, Terrain.occlusionMap and Foliage.colorMap were sampled with their rows turned over, so a picture painted to match a heightmap, a splatmap or a density map came out mirrored north to south under them; an image over an area now has its first row along the least z whoever reads it":