Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

Initial code and docs commit

Rachel Bittnereea661fb01af
diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
new file mode 100644
index 0000000..36caba3
--- /dev/null
+++ b/.github/workflows/tox.yml
@@ -0,0 +1,45 @@
+name: tox
+on:
+ push:
+ pull_request:
+
+jobs:
+ test:
+ name: test ${{ matrix.py }} - ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ os:
+ - Ubuntu
+ - Windows
+ - MacOs
+ py:
+ - "3.9"
+ - "3.8"
+ - "3.7"
+ steps:
+ - name: Setup python for test ${{ matrix.py }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.py }}
+ - uses: actions/checkout@v2
+ - name: Install soundlibs Ubuntu
+ run: sudo apt-get update && sudo apt-get install --no-install-recommends -y --fix-missing pkg-config libsndfile1
+ if: matrix.os == 'Ubuntu'
+ - name: Install soundlibs MacOs
+ run: brew install libsndfile
+ if: matrix.os == 'MacOs'
+ - name: Install soundlibs Windows
+ run: choco install libsndfile
+ if: matrix.os == 'Windows'
+ - name: Install tox
+ run: python -m pip install tox
+ # We will only check this on the minimum python version
+ - name: Check formatting, lint and mypy
+ run: tox -c tox.ini -e check-formatting,lint,mypy
+ if: matrix.py == '3.8'
+ - name: Run test suite
+ run: tox -c tox.ini -e py$(echo ${{ matrix.py }} | tr -d '.'),manifest
+ - name: Check that basic-pitch can be run as a commandline
+ run: pip3 install -e . && basic-pitch --help
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2ce9b95
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,147 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# For macbooks...
+**/.DS_Store
+
+# pyenv
+.python-version
+
+# junit
+junit.xml
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..69f6457
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,70 @@
+# Code of Conduct
+
+This code of conduct outlines our expectations for participants within the **Spotify FOSS** community, as well as steps to reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community.
+
+Our open source community strives to:
+
+* **Be friendly and patient.**
+* **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability.
+* **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language.
+* **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one.
+* **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable.
+* **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes.
+
+## Definitions
+
+Harassment includes, but is not limited to:
+
+- Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation
+- Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment
+- Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle
+- Physical contact and simulated physical contact (eg, textual descriptions like “*hug*” or “*backrub*”) without consent or after a request to stop
+- Threats of violence, both physical and psychological
+- Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm
+- Deliberate intimidation
+- Stalking or following
+- Harassing photography or recording, including logging online activity for harassment purposes
+- Sustained disruption of discussion
+- Unwelcome sexual attention, including gratuitous or off-topic sexual images or behaviour
+- Pattern of inappropriate social contact, such as requesting/assuming inappropriate levels of intimacy with others
+- Continued one-on-one communication after requests to cease
+- Deliberate “outing” of any aspect of a person’s identity without their consent except as necessary to protect others from intentional abuse
+- Publication of non-harassing private communication
+
+Our open source community prioritizes marginalized people’s safety over privileged people’s comfort. We will not act on complaints regarding:
+
+- ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’
+- Reasonable communication of boundaries, such as “leave me alone,” “go away,” or “I’m not discussing this with you”
+- Refusal to explain or debate social justice concepts
+- Communicating in a ‘tone’ you don’t find congenial
+- Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions
+
+
+### Diversity Statement
+
+We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong.
+
+Although this list cannot be exhaustive, we explicitly honor diversity in age, gender, gender identity or expression, culture, ethnicity, language, national origin, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, and technical ability. We will not tolerate discrimination based on any of the protected
+characteristics above, including participants with disabilities.
+
+### Reporting Issues
+
+If you experience or witness unacceptable behavior—or have any other concerns—please report it by contacting us via **[email protected]**. All reports will be handled with discretion. In your report please include:
+
+- Your contact information.
+- Names (real, nicknames, or pseudonyms) of any individuals involved. If there are additional witnesses, please
+include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link.
+- Any additional information that may be helpful.
+
+After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse.
+
+### Attribution & Acknowledgements
+
+We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration:
+
+* [Django](https://www.djangoproject.com/conduct/reporting/)
+* [Python](https://www.python.org/community/diversity/)
+* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct)
+* [Contributor Covenant](http://contributor-covenant.org/)
+* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/)
+* [Citizen Code of Conduct](http://citizencodeofconduct.org/)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..31d1a60
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,90 @@
+# How to Contribute
+
+Basic Pitch welcomes your contributions!
+
+## Getting Started
+
+To get your environment set up to build `basic-pitch`, you'll need Python and Pip installed on your machine.
+
+We recommend first installing the following non-python dependencies:
+
+- [libsndfile](http://libsndfile.github.io/libsndfile/) is a C library for reading and writing files containing sampled sound through one standard library interface.
+ - To install on MacOs, run `brew install libsndfile` using [Homebrew](https://brew.sh/)
+ - To install on Windows, run `choco install libsndfile` using [Chocolatey](https://chocolatey.org/)
+ - To install on Ubuntu, run `sudo apt-get update && sudo apt-get install --no-install-recommends -y --fix-missing pkg-config libsndfile1`
+- [ffmpeg](https://ffmpeg.org/) is a complete, cross-platform solution to record, convert and stream audio in all `basic-pitch` supported formats
+
+To compile a debug build of `basic-pitch` that allows using a debugger (like gdb or lldb), use the following command to build the package locally and install a symbolic link for debugging:
+```shell
+python3 setup.py build develop
+```
+
+Then, you can `import basic_pitch` from Python (or run the tests with `tox`) to test out your local changes.
+
+## Workflow
+
+We follow the [GitHub Flow Workflow](https://guides.github.com/introduction/flow/):
+
+1. Fork the project
+1. Check out the `main` branch
+1. Create a feature branch
+1. Write code and tests for your change
+1. From your branch, make a pull request against `https://github.com/spotify/basic-pitch`
+1. Work with repo maintainers to get your change reviewed
+1. Wait for your change to be pulled into `https://github.com/spotify/basic-pitch/main`
+1. Delete your feature branch
+
+## Testing
+
+We use `tox` for testing - running tests from end-to-end should be as simple as:
+
+```
+tox
+```
+
+## Style
+
+Use `black` with defaults for Python code.
+
+## Issues
+
+When creating an issue please try to ahere to the following format:
+
+ module-name: One line summary of the issue (less than 72 characters)
+
+ ### Expected behaviour
+
+ As concisely as possible, describe the expected behaviour.
+
+ ### Actual behaviour
+
+ As concisely as possible, describe the observed behaviour.
+
+ ### Steps to reproduce the behaviour
+
+ List all relevant steps to reproduce the observed behaviour.
+
+## Documentation
+
+We also welcome improvements to the project documentation or to the existing
+docs. Please file an [issue](https://github.com/spotify/basic-pitch/issues/new).
+
+## First Contributions
+
+If you are a first time contributor to `basic-pitch`, familiarize yourself with the:
+* [Code of Conduct](CODE_OF_CONDUCT.md)
+* [GitHub Flow Workflow](https://guides.github.com/introduction/flow/)
+<!-- * Issue and pull request style guides -->
+
+When you're ready, navigate to [issues](https://github.com/spotify/basic-pitch/issues/new). Some issues have been identified by community members as [good first issues](https://github.com/spotify/basic-pitch/labels/good%20first%20issue).
+
+There is a lot to learn when making your first contribution. As you gain experience, you will be able to make contributions faster. You can submit an issue using the [question](https://github.com/spotify/basic-pitch/labels/question) label if you encounter challenges.
+
+# License
+
+By contributing your code, you agree to license your contribution under the
+terms of the [LICENSE](https://github.com/spotify/basic-pitch/blob/main/LICENSE).
+
+# Code of Conduct
+
+Read our [Code of Conduct](CODE_OF_CONDUCT.md) for the project.
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
index 261eeb9..dc22a73 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,3 +1,5 @@
+Copyright 2022 Spotify AB
+
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..16788a7
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,6 @@
+include *.txt tox.ini *.rst *.md LICENSE
+include catalog-info.yaml
+recursive-include tests *.py *.wav
+recursive-include basic_pitch *.py
+recursive-include basic_pitch/saved_models/*
+recursive-include basic_pitch *.index *.pb variables.data*
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..7b08b10
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,31 @@
+Basic Pitch
+Copyright 2022 Spotify AB
+
+This product includes software developed at
+Spotify AB (http://www.spotify.com/).
+
+This product includes software from Librosa (ISC).
+* Copyright (C) 2013--2017, librosa development team.
+
+This product includes software from mir_eval (MIT)
+* Copyright (C) 2014 Colin Raffel
+
+This product includes software from numpy (BSD)
+* Copyright (C) 2005-2022, NumPy Developers.
+
+This product includes software from pretty-midi (MIT)
+* Copyright (C) 2014 Colin Raffel
+
+This product includes software from resampy (ISC)
+* Copyright (C) 2016, Brian McFee
+
+This product includes software from scipy (BSD)
+* Copyright (C) 2001-2002 Enthought, Inc. 2003-2022, SciPy Developers
+
+This product includes software from tensorflow (Apache 2.0)
+* Copyright (C) 2019 Google, LLC <[email protected]>
+
+The tests for `basic-pitch` include audio files from the
+Vocadito dataset liscened under Creative Commons
+Attribution 4.0 International. The dataset can be found at:
+https://zenodo.org/record/5578807#.YnRm5vPMKDU
diff --git a/OWNERS.md b/OWNERS.md
new file mode 100644
index 0000000..ccbb6b1
--- /dev/null
+++ b/OWNERS.md
@@ -0,0 +1,35 @@
+# Owners
+
+- See [CONTRIBUTING.md](CONTRIBUTING.md) for general contribution guidelines.
+
+## Core Developers
+
+- David Rubinstein (@drubinstein, Spotify)
+- Rachel Bittner (@rabitt, Spotify)
+
+## Triagers
+
+- David Rubinstein (@drubinstein, Spotify)
+- Rachel Bittner (@rabitt, Spotify)
+- Charae Tongg (@charaizlabs, Spotify)
+- Peter Sobot (@psobot, Spotify)
+- Juanjo Bosch (@juanjobosch, Spotify)
+- Vincent Degroote (@vinzentt, Spotify)
+
+## Contributors
+
+- Rachel Bittner (@rabitt, Spotify)
+- Juanjo Bosch (@juanjobosch, Spotify)
+- Vincent Degroote (@vinzentt, Spotify)
+- Simon Durand (@simondurand, Spotify)
+- David Rubinstein (@drubinstein, Spotify)
+- Gabriel Meseguer-Brocal (@gabolsgabs, Deezer)
+- Nicola Montecchio (@nicolamontecchio, Spotify)
+- Scott Sheffield (@scottsheffield, Spotify)
+- Peter Sobot (@psobot, Spotify)
+- Daniel Stoller (@f90, Spotify)
+- Ching Sung (@chingsungx, Spotify)
+- Charae Tongg (@charaizlabs, Spotify)
+- Jan Van Balen (@jvbalen, Spotify)
+
+The research and development of this project is led by [Spotify's Audio Intelligence Lab](https://research.atspotify.com/audio-intelligence/).
diff --git a/README.md b/README.md
index 260ab0a..f8d7c07 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,185 @@
-# basic-pitch
-basic-pitch take 2
+![Basic Pitch Logo](https://user-images.githubusercontent.com/213293/167478083-de988de2-9137-4325-8a5f-ceeb51233753.png)
+
+
+
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/basic-pitch)
+![Supported Platforms](https://img.shields.io/badge/platforms-macOS%20%7C%20Windows%20%7C%20Linux-green)
+
+
+Basic Pitch is a Python library for Automatic Music Transcription (AMT), using lightweight neural network developed by [Spotify's Audio Intelligence Lab](https://research.atspotify.com/audio-intelligence/). It's small, easy-to-use, and `pip install`-able.
+
+Basic Pitch may be simple, but it's is far from "basic"! `basic-pitch` is efficient and easy to use, and its multipitch support, its ability to generalize across instruments, and its note accuracy competes with much larger and more resource-hungry AMT systems.
+
+Provide a compatible audio file and basic-pitch will generate a MIDI file, complete with pitch bends. Basic pitch is instrument-agnostic and supports polyphonic instruments, so you can freely enjoy transcription of all your favorite music, no matter what instrument is used. Basic pitch works best on one instrument at a time.
+
+### Research Paper
+This library was released in conjunction with Spotify's publication at [ICASSP 2022](https://2022.ieeeicassp.org/). You can read more about this research in the paper, [A Lightweight Instrument-Agnostic Model for Polyphonic Note Transcription and Multipitch Estimation](https://arxiv.org/abs/2203.09893).
+
+If you use this library in academic research, consider citing it:
+```bibtex
+@inproceedings{2022_BittnerBRME_LightweightNoteTranscription_ICASSP,
+ author= {Bittner, Rachel M. and Bosch, Juan Jos\'e and Rubinstein, David and Meseguer-Brocal, Gabriel and Ewert, Sebastian},
+ title= {A Lightweight Instrument-Agnostic Model for Polyphonic Note Transcription and Multipitch Estimation},
+ booktitle= {Proceedings of the IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},
+ address= {Singapore},
+ year= 2022,
+}
+```
+
+### Demo
+If, for whatever reason, you're not yet completely inspired, or you're just like so totally over the general vibe and stuff, checkout our snappy demo website, [basicpitch.spotify.com](https://basicpitch.spotify.com), to experiment with our model on whatever music audio you provide!
+
+
+## Installation
+
+`basic-pitch` is available via PyPI. To install the current release:
+
+ pip install basic-pitch
+
+To update Basic Pitch to the latest version, add `--upgrade` to the above command.
+
+#### Compatible Environments:
+- MacOS, Windows and Ubuntu operating systems
+- Python versions 3.7, 3.8, 3.9
+
+
+## Usage
+
+### Model Prediction
+
+#### Command Line Tool
+
+This library offers a command line tool interface. A basic prediction command will generate and save a MIDI file transcription of audio at the `<input-audio-path>` to the `<output-directory>`:
+
+```bash
+basic-pitch <output-directory> <input-audio-path>
+```
+
+To process more than one audio file at a time:
+
+```bash
+basic-pitch <output-directory> <input-audio-path-1> <input-audio-path-2> <input-audio-path-3>
+```
+
+Optionally, you may append any of the following flags to your prediction command to save additional formats of the prediction output to the `<output-directory>`:
+
+- `--sonify-midi` to additionally save a `.wav` audio rendering of the MIDI file
+- `--save-model-outputs` to additionally save raw model outputs as an NPZ file
+- `--save-note-events` to additionally save the predicted note events as a CSV file
+
+To discover more parameter control, run:
+```bash
+basic-pitch --help
+```
+
+#### Programmatic
+
+**predict()**
+
+Import `basic-pitch` into your own Python code and run the [`predict`](basic_pitch/inference.py) functions directly, providing an `<input-audio-path>` and returning the model's prediction results:
+
+```python
+from basic_pitch.inference import predict
+from basic_pitch import ICASSP_2022_MODEL_PATH
+
+model_output, midi_data, note_activations = predict(<input-audio-path>)
+```
+
+- `<minimum-frequency>` & `<maximum-frequency>` (*float*s) set the maximum and minimum allowed note frequency, in Hz, returned by the model. Pitch events with frequencies outside of this range will be excluded from the prediction results.
+- `model_output` is the raw model inference output
+- `midi_data` is the transcribed MIDI data derived from the `model_output`
+- `note_events` is a list of note events derived from the `model_output`
+
+**predict() in a loop**
+
+To run prediction within a loop, you'll want to load the model yourself and provide `predict()` with the loaded model object itself to be used for repeated prediction calls, in order to avoid redundant and sluggish model loading.
+
+```python
+import tensorflow as tf
+
+from basic_pitch.inference import predict
+from basic_pitch import ICASSP_2022_MODEL_PATH
+
+basic_pitch_model = tf.saved_model.load(str(ICASSP_2022_MODEL_PATH))
+
+for x in range():
+ ...
+ model_output, midi_data, note_activations = predict(
+ <loop-x-input-audio-path>,
+ basic_pitch_model,
+ )
+ ...
+```
+
+**predict_and_save()**
+
+If you would like `basic-pitch` orchestrate the generation and saving of our various supported output file types, you may use [`predict_and_save`](basic_pitch/inference.py) instead of using [`predict`](basic_pitch/inference.py) directly:
+
+```python
+from basic_pitch.inference import predict_and_save
+
+predict_and_save(
+ <input-audio-path-list>,
+ <output-directory>,
+ <save-midi>,
+ <sonify-midi>,
+ <save-model-outputs>,
+ <save-note-events>,
+)
+```
+
+where:
+ - `<input-audio-path-list>` & `<output-directory>`
+ - directory paths for `basic-pitch` to read from/write to.
+ - `<save-midi>`
+ - *bool* to control generating and saving a MIDI file to the `<output-directory>`
+ - `<sonify-midi>`
+ - *bool* to control saving a WAV audio rendering of the MIDI file to the `<output-directory>`
+ - `<save-model-outputs>`
+ - *bool* to control saving the raw model output as a NPZ file to the `<output-directory>`
+ - `<save-note-events>`
+ - *bool* to control saving predicted note events as a CSV file `<output-directory>`
+
+
+
+### Model Input
+
+**Supported Audio Codecs**
+
+ `basic-pitch` accepts all sound files that are compatible with its version of [`librosa`](https://librosa.org/doc/latest/index.html), including:
+
+- `.mp3`
+- `.ogg`
+- `.wav`
+- `.flac`
+- `.m4a`
+
+**Mono Channel Audio Only**
+
+While you may use stereo audio as an input to our model, at prediction time, the channels of the input will be down-mixed to mono, and then analyzed and transcribed.
+
+**File Size/Audio Length**
+
+This model can process any size or length of audio, but processing of larger/longer audio files could be limited by your machine's available disk space. To process these files, we recommend streaming the audio of the file, processing windows of audio at a time.
+
+**Sample Rate**
+
+Input audio maybe be of any sample rate, however, all audio will be resampled to 22050 Hz before processing.
+
+
+## Contributing
+
+Contributions to `basic-pitch` are welcomed! See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
+
+## Copyright and License
+`basic-pitch` is Copyright 2022 Spotify AB.
+
+This software is licensed under the Apache License, Version 2.0 (the "Apache License"). You may choose either license to govern your use of this software only upon the condition that you accept all of the terms of either the Apache License.
+
+You may obtain a copy of the Apache License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the Apache License or the GPL License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache License for the specific language governing permissions and limitations under the Apache License.
+
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..89155ad
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,16 @@
+# Security Policy
+
+We're big believers in protecting your privacy and security. As a company, we not only have a vested interest, but also a deep desire to see the Internet remain as safe as possible for us all.
+
+So, needless to say, we take security issues very seriously.
+
+In our opinion, the practice of 'responsible disclosure' is the best way to safeguard the Internet. It allows individuals to notify companies like Spotify of any security threats before going public with the information. This gives us a fighting chance to resolve the problem before the criminally-minded become aware of it.
+
+Responsible disclosure is the industry best practice, and we recommend it as a procedure to anyone researching security vulnerabilities.
+
+## Reporting a Vulnerability
+
+If you have discovered a vulnerability in this open source project or another serious security issue,
+please submit it to the Spotify bounty program hosted by HackerOne.
+
+https://hackerone.com/spotify
diff --git a/basic_pitch/__init__.py b/basic_pitch/__init__.py
new file mode 100644
index 0000000..29138f9
--- /dev/null
+++ b/basic_pitch/__init__.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pathlib
+
+__author__ = "Spotify"
+__version__ = "0.0.1"
+__email__ = "[email protected]"
+__demowebsite__ = "https://basicpitch.io"
+__description__ = "Basic Pitch"
+__url__ = "https://github.com/spotify/basic-pitch"
+
+ICASSP_2022_MODEL_PATH = pathlib.Path(__file__).parent / "saved_models/icassp_2022/nmp"
diff --git a/basic_pitch/commandline_printing.py b/basic_pitch/commandline_printing.py
new file mode 100644
index 0000000..f51d1e5
--- /dev/null
+++ b/basic_pitch/commandline_printing.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+import pathlib
+import sys
+import threading
+import time
+from contextlib import contextmanager
+from typing import Iterator, Optional, Union
+
+TF_LOG_LEVEL_KEY = "TF_CPP_MIN_LOG_LEVEL"
+TF_LOG_LEVEL_NO_WARNINGS_VALUE = "3"
+s_print_lock = threading.Lock()
+OUTPUT_EMOJIS = {
+ "MIDI": "💅",
+ "MODEL_OUTPUT_NPZ": "💁‍♀️",
+ "MIDI_SONIFICATION": "🎧",
+ "NOTE_EVENTS": "🌸",
+}
+
+
+def generating_file_message(output_type: str) -> None:
+ """Print a message that a file is being generated
+
+ Args:
+ output_type: string indicating which kind of file is being generated
+
+ """
+ print(f"\n\n Creating {output_type.replace('_', ' ').lower()}...")
+
+
+def file_saved_confirmation(output_type: str, save_path: Union[pathlib.Path, str]) -> None:
+ """Print a confirmation that the file was saved succesfully
+
+ Args:
+ output_type: The kind of file that is being generated.
+ save_path: The path to output file.
+
+ """
+ print(f" {OUTPUT_EMOJIS[output_type]} Saved to {save_path}")
+
+
+def failed_to_save(output_type: str, save_path: Union[pathlib.Path, str]) -> None:
+ """Print a failure to save message
+
+ Args:
+ output_type: The kind of file that is being generated.
+ save_path: The path to output file.
+
+ """
+ print(f"\n🚨 Failed to save {output_type.replace('_', ' ').lower()} to {save_path} \n")
+
+
+@contextmanager
+def no_tf_warnings() -> Iterator[None]:
+ """
+ Supress tensorflow warnings in this context
+ """
+ tf_logging_level = os.environ.get(TF_LOG_LEVEL_KEY, TF_LOG_LEVEL_NO_WARNINGS_VALUE)
+ os.environ[TF_LOG_LEVEL_KEY] = TF_LOG_LEVEL_NO_WARNINGS_VALUE
+ yield
+ os.environ[TF_LOG_LEVEL_KEY] = tf_logging_level
+
+
+@contextmanager
+def entertaining_waiting(operation_name: Optional[str] = None) -> Iterator[None]:
+ """
+ Display a jazzy loading animation in the commandline for the duration of this context
+
+ Args:
+ operation_name: the name of the operation contained in this context to be displayed in
+ the commandline
+ """
+ if operation_name is None:
+ operation_name = f"Running Command: '{inspect.stack()[2].function}'"
+
+ loading = True
+ loading_states = ["🥁", "🎷", "🎸", "🎻", "🎹", "🪘", "🪗", "🎺", "🪕"]
+ blank_message = " " * (len(loading_states) * 2 + len(operation_name))
+
+ def animate() -> None:
+ emoji_index = 0
+ while loading:
+ if emoji_index >= len(loading_states):
+ emoji_index = 0
+ loading_message = blank_message
+ else:
+ loading_message = "".join(loading_states[0:emoji_index])
+
+ s_flush()
+ s_print(f"\r{operation_name}... " + loading_message)
+ emoji_index += 1
+ time.sleep(0.3)
+
+ t = threading.Thread(target=animate)
+ t.start()
+
+ yield
+ loading = False
+
+
+def s_print(msg: str) -> None:
+ """
+ Thread safe print function
+
+ Args:
+ msg: the text to be printed to stdout
+ """
+ with s_print_lock:
+ sys.stdout.write(msg)
+
+
+def s_flush() -> None:
+ """
+ Thread safe flush function
+ """
+ with s_print_lock:
+ sys.stdout.flush()
diff --git a/basic_pitch/constants.py b/basic_pitch/constants.py
new file mode 100644
index 0000000..a78a487
--- /dev/null
+++ b/basic_pitch/constants.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+FFT_HOP = 256
+N_FFT = 8 * FFT_HOP
+
+NOTES_BINS_PER_SEMITONE = 1
+CONTOURS_BINS_PER_SEMITONE = 3
+# base frequency of the CENTRAL bin of the first semitone (i.e., the
+# second bin if annotations_bins_per_semitone is 3)
+ANNOTATIONS_BASE_FREQUENCY = 27.5 # lowest key on a piano
+ANNOTATIONS_N_SEMITONES = 88 # number of piano keys
+AUDIO_SAMPLE_RATE = 22050
+AUDIO_N_CHANNELS = 1
+N_FREQ_BINS_NOTES = ANNOTATIONS_N_SEMITONES * NOTES_BINS_PER_SEMITONE
+N_FREQ_BINS_CONTOURS = ANNOTATIONS_N_SEMITONES * CONTOURS_BINS_PER_SEMITONE
+
+AUDIO_WINDOW_LENGTH = 2 # duration in seconds of training examples - original 1
+
+ANNOTATIONS_FPS = AUDIO_SAMPLE_RATE // FFT_HOP
+ANNOTATION_HOP = 1.0 / ANNOTATIONS_FPS
+
+# ANNOT_N_TIME_FRAMES is the number of frames in the time-frequency representations we compute
+ANNOT_N_FRAMES = ANNOTATIONS_FPS * AUDIO_WINDOW_LENGTH
+
+# AUDIO_N_SAMPLES is the number of samples in the (clipped) audio that we use as input to the models
+AUDIO_N_SAMPLES = AUDIO_SAMPLE_RATE * AUDIO_WINDOW_LENGTH - FFT_HOP
+
+DATASET_SAMPLING_FREQUENCY = {
+ "MAESTRO": 5,
+ "GuitarSet": 2,
+ "MedleyDB-Pitch": 2,
+ "iKala": 2,
+ "slakh": 2,
+}
+
+
+def _freq_bins(bins_per_semitone: int, base_frequency: float, n_semitones: int) -> np.array:
+ d = 2.0 ** (1.0 / (12 * bins_per_semitone))
+ bin_freqs = base_frequency * d ** np.arange(bins_per_semitone * n_semitones)
+ return bin_freqs
+
+
+FREQ_BINS_NOTES = _freq_bins(NOTES_BINS_PER_SEMITONE, ANNOTATIONS_BASE_FREQUENCY, ANNOTATIONS_N_SEMITONES)
+FREQ_BINS_CONTOURS = _freq_bins(CONTOURS_BINS_PER_SEMITONE, ANNOTATIONS_BASE_FREQUENCY, ANNOTATIONS_N_SEMITONES)
diff --git a/basic_pitch/inference.py b/basic_pitch/inference.py
new file mode 100644
index 0000000..ef3df49
--- /dev/null
+++ b/basic_pitch/inference.py
@@ -0,0 +1,415 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import enum
+import json
+import os
+import pathlib
+from typing import Dict, List, Optional, Tuple, Union
+
+from tensorflow import Tensor, signal, keras, saved_model
+import numpy as np
+import librosa
+import pretty_midi
+
+from basic_pitch.constants import (
+ AUDIO_SAMPLE_RATE,
+ AUDIO_N_SAMPLES,
+ ANNOTATIONS_FPS,
+ FFT_HOP,
+)
+from basic_pitch import ICASSP_2022_MODEL_PATH, note_creation as infer
+from basic_pitch.commandline_printing import (
+ entertaining_waiting,
+ generating_file_message,
+ no_tf_warnings,
+ file_saved_confirmation,
+ failed_to_save,
+)
+
+
+def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, List[Dict[str, int]]]:
+ """
+ Pad appropriately an audio file, and return as
+ windowed signal, with window length = AUDIO_N_SAMPLES
+
+ Returns:
+ audio_windowed: tensor with shape (n_windows, AUDIO_N_SAMPLES, 1)
+ audio windowed into fixed length chunks
+ window_times: list of {'start':.., 'end':...} objects (times in seconds)
+
+ """
+ from tensorflow import expand_dims # imporing this here so the module loads faster
+
+ audio_windowed = expand_dims(
+ signal.frame(audio_original, AUDIO_N_SAMPLES, hop_size, pad_end=True, pad_value=0),
+ axis=-1,
+ )
+ window_times = [
+ {
+ "start": t_start,
+ "end": t_start + (AUDIO_N_SAMPLES / AUDIO_SAMPLE_RATE),
+ }
+ for t_start in np.arange(audio_windowed.shape[0]) * hop_size / AUDIO_SAMPLE_RATE
+ ]
+ return audio_windowed, window_times
+
+
+def get_audio_input(
+ audio_path: Union[pathlib.Path, str], overlap_len: int, hop_size: int
+) -> Tuple[Tensor, List[Dict[str, int]], int]:
+ """
+ Read wave file (as mono), pad appropriately, and return as
+ windowed signal, with window length = AUDIO_N_SAMPLES
+
+ Returns:
+ audio_windowed: tensor with shape (n_windows, AUDIO_N_SAMPLES, 1)
+ audio windowed into fixed length chunks
+ window_times: list of {'start':.., 'end':...} objects (times in seconds)
+ audio_original_length: int
+ length of original audio file, in frames, BEFORE padding.
+
+ """
+ assert overlap_len % 2 == 0, "overlap_length must be even, got {}".format(overlap_len)
+ audio_original, _ = librosa.load(audio_path, sr=AUDIO_SAMPLE_RATE, mono=True)
+ original_length = audio_original.shape[0]
+ audio_original = np.concatenate([np.zeros((int(overlap_len / 2),), dtype=np.float32), audio_original])
+ audio_windowed, window_times = window_audio_file(audio_original, hop_size)
+ return audio_windowed, window_times, original_length
+
+
+def unwrap_output(output: Tensor, audio_original_length: int, n_overlapping_frames: int) -> np.array:
+ """Unwrap batched model predictions to a single matrix.
+
+ Args:
+ output: array (n_batches, n_times_short, n_freqs)
+ audio_original_length: length of original audio signal (in samples)
+ n_overlapping_frames: number of overlapping frames in the output
+
+ Returns:
+ array (n_times, n_freqs)
+ """
+ raw_output = output.numpy()
+ if len(raw_output.shape) != 3:
+ return None
+
+ n_olap = int(0.5 * n_overlapping_frames)
+ if n_olap > 0:
+ # remove half of the overlapping frames from beginning and end
+ raw_output = raw_output[:, n_olap:-n_olap, :]
+
+ output_shape = raw_output.shape
+ n_output_frames_original = int(np.floor(audio_original_length * (ANNOTATIONS_FPS / AUDIO_SAMPLE_RATE)))
+ unwrapped_output = raw_output.reshape(output_shape[0] * output_shape[1], output_shape[2])
+ return unwrapped_output[:n_output_frames_original, :] # trim to original audio length
+
+
+def run_inference(
+ audio_path: Union[pathlib.Path, str], model: keras.Model, debug_file: Optional[pathlib.Path] = None
+) -> Dict[str, np.array]:
+ """Run the model on the input audio path.
+
+ Args:
+ audio_path: The audio to run inference on.
+ model: A loaded keras model to run inference with.
+ debug_file: An optional path to output debug data to. Useful for testing/verification.
+
+ Returns:
+ A dictionary with the notes, onsets and contours from model inference.
+ """
+ # overlap 30 frames
+ n_overlapping_frames = 30
+ overlap_len = n_overlapping_frames * FFT_HOP
+ hop_size = AUDIO_N_SAMPLES - overlap_len
+
+ audio_windowed, _, audio_original_length = get_audio_input(audio_path, overlap_len, hop_size)
+
+ output = model(audio_windowed)
+ unwrapped_output = {k: unwrap_output(output[k], audio_original_length, n_overlapping_frames) for k in output}
+
+ if debug_file:
+ with open(debug_file, "w") as f:
+ json.dump(
+ {
+ "audio_windowed": audio_windowed.numpy().tolist(),
+ "audio_original_length": audio_original_length,
+ "hop_size_samples": hop_size,
+ "overlap_length_samples": overlap_len,
+ "unwrapped_output": {k: v.tolist() for k, v in unwrapped_output.items()},
+ },
+ f,
+ )
+
+ return unwrapped_output
+
+
+class OutputExtensions(enum.Enum):
+ MIDI = "mid"
+ MODEL_OUTPUT_NPZ = "npz"
+ MIDI_SONIFICATION = "wav"
+ NOTE_EVENTS = "csv"
+
+
+def verify_input_path(audio_path: Union[pathlib.Path, str]) -> None:
+ """Verify that an input path is valid and can be processed
+
+ Args:
+ audio_path: Path to an audio file.
+
+ Raises:
+ ValueError: If the audio file is invalid.
+ """
+ if not os.path.isfile(audio_path):
+ raise ValueError(f"🚨 {audio_path} is not a file path.")
+
+ if not os.path.exists(audio_path):
+ raise ValueError(f"🚨 {audio_path} does not exist.")
+
+
+def verify_output_dir(output_dir: Union[pathlib.Path, str]) -> None:
+ """Verify that an output directory is valid and can be processed
+
+ Args:
+ output_dir: Path to an output directory.
+
+ Raises:
+ ValueError: If the output directory is invalid.
+ """
+ if not os.path.isdir(output_dir):
+ raise ValueError(f"🚨 {output_dir} is not a directory.")
+
+ if not os.path.exists(output_dir):
+ raise ValueError(f"🚨 {output_dir} does not exist.")
+
+
+def build_output_path(
+ audio_path: Union[pathlib.Path, str], output_directory: Union[pathlib.Path, str], output_type: OutputExtensions
+) -> pathlib.Path:
+ """Create an output path and make sure it doesn't already exist.
+
+ Args:
+ audio_path: The original file path.
+ output_directory: The directory we will output to.
+ output_type: The type of output file we are creating.
+
+ Raises:
+ IOError: If the generated path already exists.
+
+ Returns:
+ A new path in the output_directory with the stem audio_path and an extension
+ based on output_type.
+ """
+ audio_path = str(audio_path)
+ if not isinstance(output_directory, pathlib.Path):
+ output_directory = pathlib.Path(output_directory)
+
+ basename, _ = os.path.splitext(os.path.basename(audio_path))
+
+ output_path = output_directory / f"{basename}_basic_pitch.{output_type.value}"
+
+ generating_file_message(output_type.name)
+
+ if output_path.exists():
+ raise IOError(
+ f" 🚨 {str(output_path)} already exists and would be overwritten. Skipping output files for {audio_path}."
+ )
+
+ return output_path
+
+
+def save_note_events(
+ note_events: List[Tuple[float, float, int, float, Optional[List[int]]]], save_path: Union[pathlib.Path, str]
+) -> None:
+ """Save note events to file
+
+ Args:
+ note_events: A list of note event tuples to save. Tuples have the format
+ ("start_time_s", "end_time_s", "pitch_midi", "velocity", "list of pitch bend values")
+ save_path: The location we're saving it
+ """
+
+ with open(save_path, "w") as fhandle:
+ writer = csv.writer(fhandle, delimiter=",")
+ writer.writerow(["start_time_s", "end_time_s", "pitch_midi", "velocity", "pitch_bend"])
+ for start_time, end_time, note_number, amplitude, pitch_bend in note_events:
+ row = [start_time, end_time, note_number, int(np.round(127 * amplitude))]
+ if pitch_bend:
+ row.extend(pitch_bend)
+ writer.writerow(row)
+
+
+def predict(
+ audio_path: Union[pathlib.Path, str],
+ model_or_model_path: Union[keras.Model, pathlib.Path, str] = ICASSP_2022_MODEL_PATH,
+ onset_threshold: float = 0.5,
+ frame_threshold: float = 0.3,
+ minimum_note_length: float = 58,
+ minimum_frequency: Optional[float] = None,
+ maximum_frequency: Optional[float] = None,
+ melodia_trick: bool = True,
+ debug_file: Optional[pathlib.Path] = None,
+) -> Tuple[Dict[str, np.array], pretty_midi.PrettyMIDI, List[Tuple[float, float, int, float, Optional[List[int]]]]]:
+ """Run a single prediction.
+
+ Args:
+ audio_path: File path for the audio to run inference on.
+ model_or_model_path: Path to load the Keras saved model from. Can be local or on GCS.
+ onset_threshold: Minimum energy required for an onset to be considered present.
+ frame_threshold: Minimum energy requirement for a frame to be considered present.
+ minimum_note_length: The minimum allowed note length in frames.
+ minimum_freq: Minimum allowed output frequency, in Hz. If None, all frequencies are used.
+ maximum_freq: Maximum allowed output frequency, in Hz. If None, all frequencies are used.
+ melodia_trick: Use the melodia post-processing step.
+ debug_file: An optional path to output debug data to. Useful for testing/verification.
+ Returns:
+ The model output, midi data and note events from a single prediction
+ """
+
+ with entertaining_waiting(f"Predicting MIDI for {audio_path}..."), no_tf_warnings():
+ # It's convenient to be able to pass in a keras saved model so if
+ # someone wants to place this function in a loop,
+ # the model doesn't have to be reloaded every function call
+ if isinstance(model_or_model_path, (pathlib.Path, str)):
+ model = saved_model.load(str(model_or_model_path))
+ else:
+ model = model_or_model_path
+
+ model_output = run_inference(audio_path, model, debug_file)
+
+ min_note_len = int(np.round(minimum_note_length / 1000 * (AUDIO_SAMPLE_RATE / FFT_HOP)))
+ midi_data, note_events = infer.model_output_to_notes(
+ model_output,
+ onset_thresh=onset_threshold,
+ frame_thresh=frame_threshold,
+ min_note_len=min_note_len, # convert to frames
+ min_freq=minimum_frequency,
+ max_freq=maximum_frequency,
+ melodia_trick=melodia_trick,
+ )
+
+ if debug_file:
+ with open(debug_file) as f:
+ debug_data = json.load(f)
+ with open(debug_file, "w") as f:
+ json.dump(
+ {
+ **debug_data,
+ "min_note_length": min_note_len,
+ "onset_thresh": onset_threshold,
+ "frame_thresh": frame_threshold,
+ "estimated_notes": [
+ (
+ float(start_time),
+ float(end_time),
+ int(pitch),
+ float(amplitude),
+ [int(b) for b in pitch_bends] if pitch_bends else None,
+ )
+ for start_time, end_time, pitch, amplitude, pitch_bends in note_events
+ ],
+ },
+ f,
+ )
+
+ return model_output, midi_data, note_events
+
+
+def predict_and_save(
+ audio_path_list: List[Union[pathlib.Path, str]],
+ output_directory: Union[pathlib.Path, str],
+ save_midi: bool,
+ sonify_midi: bool,
+ save_model_outputs: bool,
+ save_notes: bool,
+ model_path: Union[pathlib.Path, str] = ICASSP_2022_MODEL_PATH,
+ onset_threshold: float = 0.5,
+ frame_threshold: float = 0.3,
+ minimum_note_length: float = 58,
+ minimum_frequency: Optional[float] = None,
+ maximum_frequency: Optional[float] = None,
+ melodia_trick: bool = True,
+ debug_file: Optional[pathlib.Path] = None,
+) -> None:
+ """Make a prediction and save the results to file.
+
+ Args:
+ audio_path_list: List of file paths for the audio to run inference on.
+ output_directory: Directory to output MIDI and all other outputs derived from the model to.
+ save_midi: True to save midi.
+ sonify_midi: Whether or not to render audio from the MIDI and output it to a file.
+ save_model_outputs: True to save contours, onsets and notes from the model prediction.
+ save_notes: True to save note events.
+ model_path: Path to load the Keras saved model from. Can be local or on GCS.
+ onset_threshold: Minimum energy required for an onset to be considered present.
+ frame_threshold: Minimum energy requirement for a frame to be considered present.
+ minimum_note_length: The minimum allowed note length in frames.
+ minimum_freq: Minimum allowed output frequency, in Hz. If None, all frequencies are used.
+ maximum_freq: Maximum allowed output frequency, in Hz. If None, all frequencies are used.
+ melodia_trick: Use the melodia post-processing step.
+ debug_file: An optional path to output debug data to. Useful for testing/verification.
+ """
+ model = saved_model.load(str(model_path))
+
+ for audio_path in audio_path_list:
+ print("")
+ try:
+ model_output, midi_data, note_events = predict(
+ pathlib.Path(audio_path),
+ model,
+ onset_threshold,
+ frame_threshold,
+ minimum_note_length,
+ minimum_frequency,
+ maximum_frequency,
+ melodia_trick,
+ debug_file,
+ )
+
+ if save_model_outputs:
+ model_output_path = build_output_path(audio_path, output_directory, OutputExtensions.MODEL_OUTPUT_NPZ)
+ try:
+ np.savez(model_output_path, basic_pitch_model_output=model_output)
+ file_saved_confirmation(OutputExtensions.MODEL_OUTPUT_NPZ.name, model_output_path)
+ except Exception:
+ failed_to_save(OutputExtensions.MODEL_OUTPUT_NPZ.name, model_output_path)
+
+ if save_midi:
+ midi_path = build_output_path(audio_path, output_directory, OutputExtensions.MIDI)
+ try:
+ midi_data.write(str(midi_path))
+ file_saved_confirmation(OutputExtensions.MIDI.name, midi_path)
+ except Exception:
+ failed_to_save(OutputExtensions.MIDI.name, midi_path)
+
+ if sonify_midi:
+ midi_sonify_path = build_output_path(audio_path, output_directory, OutputExtensions.MIDI_SONIFICATION)
+ try:
+ infer.sonify_midi(midi_data, midi_sonify_path)
+ file_saved_confirmation(OutputExtensions.MIDI_SONIFICATION.name, midi_sonify_path)
+ except Exception:
+ failed_to_save(OutputExtensions.MIDI_SONIFICATION.name, midi_sonify_path)
+
+ if save_notes:
+ note_events_path = build_output_path(audio_path, output_directory, OutputExtensions.NOTE_EVENTS)
+ try:
+ save_note_events(note_events, note_events_path)
+ file_saved_confirmation(OutputExtensions.NOTE_EVENTS.name, note_events_path)
+ except Exception:
+ failed_to_save(OutputExtensions.NOTE_EVENTS.name, note_events_path)
+ except Exception as e:
+ print(e)
diff --git a/basic_pitch/layers/__init__.py b/basic_pitch/layers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/basic_pitch/layers/math.py b/basic_pitch/layers/math.py
new file mode 100644
index 0000000..be043a1
--- /dev/null
+++ b/basic_pitch/layers/math.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+
+
+def log_base_b(x: tf.Tensor, base: int) -> tf.Tensor:
+ """
+ Compute log_b(x)
+ Args:
+ x : input
+ base : log base. E.g. for log10 base=10
+ Returns:
+ log_base(x)
+ """
+ numerator = tf.math.log(x)
+ denominator = tf.math.log(tf.constant(base, dtype=numerator.dtype))
+ return numerator / denominator
diff --git a/basic_pitch/layers/nnaudio.py b/basic_pitch/layers/nnaudio.py
new file mode 100644
index 0000000..45edb65
--- /dev/null
+++ b/basic_pitch/layers/nnaudio.py
@@ -0,0 +1,648 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This module is comprised of PyTorch layers from NNAudio and ported to TensorFlow:
+# https://github.com/KinWaiCheuk/nnAudio
+# The above code is released under an MIT license.
+
+import warnings
+import tensorflow as tf
+import numpy as np
+from typing import Any, List, Optional, Tuple, Union
+
+import scipy.signal
+
+
+def create_lowpass_filter(
+ band_center: float = 0.5,
+ kernel_length: int = 256,
+ transition_bandwidth: float = 0.03,
+ dtype: tf.dtypes.DType = tf.float32,
+) -> np.ndarray:
+ """
+ Calculate the highest frequency we need to preserve and the lowest frequency we allow
+ to pass through. Note that frequency is on a scale from 0 to 1 where 0 is 0 and 1 is
+ the Nyquist frequency of the signal BEFORE downsampling.
+ """
+
+ passband_max = band_center / (1 + transition_bandwidth)
+ stopband_min = band_center * (1 + transition_bandwidth)
+
+ # We specify a list of key frequencies for which we will require
+ # that the filter match a specific output gain.
+ # From [0.0 to passband_max] is the frequency range we want to keep
+ # untouched and [stopband_min, 1.0] is the range we want to remove
+ key_frequencies = [0.0, passband_max, stopband_min, 1.0]
+
+ # We specify a list of output gains to correspond to the key
+ # frequencies listed above.
+ # The first two gains are 1.0 because they correspond to the first
+ # two key frequencies. the second two are 0.0 because they
+ # correspond to the stopband frequencies
+ gain_at_key_frequencies = [1.0, 1.0, 0.0, 0.0]
+
+ # This command produces the filter kernel coefficients
+ filter_kernel = scipy.signal.firwin2(kernel_length, key_frequencies, gain_at_key_frequencies)
+
+ return tf.constant(filter_kernel, dtype=dtype)
+
+
+def next_power_of_2(A: int) -> int:
+ """A helper function to calculate the next nearest number to the power of 2."""
+ return int(np.ceil(np.log2(A)))
+
+
+def early_downsample(
+ sr: Union[float, int],
+ hop_length: int,
+ n_octaves: int,
+ nyquist_hz: float,
+ filter_cutoff_hz: float,
+) -> Tuple[Union[float, int], int, int]:
+ """Return new sampling rate and hop length after early downsampling"""
+ downsample_count = early_downsample_count(nyquist_hz, filter_cutoff_hz, hop_length, n_octaves)
+ downsample_factor = 2 ** (downsample_count)
+
+ hop_length //= downsample_factor # Getting new hop_length
+ new_sr = sr / float(downsample_factor) # Getting new sampling rate
+
+ return new_sr, hop_length, downsample_factor
+
+
+# The following two downsampling count functions are obtained from librosa CQT
+# They are used to determine the number of pre resamplings if the starting and ending frequency
+# are both in low frequency regions.
+def early_downsample_count(nyquist_hz: float, filter_cutoff_hz: float, hop_length: int, n_octaves: int) -> int:
+ """Compute the number of early downsampling operations"""
+
+ downsample_count1 = max(0, int(np.ceil(np.log2(0.85 * nyquist_hz / filter_cutoff_hz)) - 1) - 1)
+ num_twos = next_power_of_2(hop_length)
+ downsample_count2 = max(0, num_twos - n_octaves + 1)
+
+ return min(downsample_count1, downsample_count2)
+
+
+def get_early_downsample_params(
+ sr: Union[float, int], hop_length: int, fmax_t: float, Q: float, n_octaves: int, dtype: tf.dtypes.DType
+) -> Tuple[Union[float, int], int, float, np.array, bool]:
+ """Compute downsampling parameters used for early downsampling"""
+
+ window_bandwidth = 1.5 # for hann window
+ filter_cutoff = fmax_t * (1 + 0.5 * window_bandwidth / Q)
+ sr, hop_length, downsample_factor = early_downsample(sr, hop_length, n_octaves, sr // 2, filter_cutoff)
+ if downsample_factor != 1:
+ earlydownsample = True
+ early_downsample_filter = create_lowpass_filter(
+ band_center=1 / downsample_factor,
+ kernel_length=256,
+ transition_bandwidth=0.03,
+ dtype=dtype,
+ )
+ else:
+ early_downsample_filter = None
+ earlydownsample = False
+
+ return sr, hop_length, downsample_factor, early_downsample_filter, earlydownsample
+
+
+def get_window_dispatch(window: Union[str, Tuple[str, float]], N: int, fftbins: bool = True) -> np.array:
+ if isinstance(window, str):
+ return scipy.signal.get_window(window, N, fftbins=fftbins)
+ elif isinstance(window, tuple):
+ if window[0] == "gaussian":
+ assert window[1] >= 0
+ sigma = np.floor(-N / 2 / np.sqrt(-2 * np.log(10 ** (-window[1] / 20))))
+ return scipy.signal.get_window(("gaussian", sigma), N, fftbins=fftbins)
+ else:
+ Warning("Tuple windows may have undesired behaviour regarding Q factor")
+ elif isinstance(window, float):
+ Warning("You are using Kaiser window with beta factor " + str(window) + ". Correct behaviour not checked.")
+ else:
+ raise Exception("The function get_window from scipy only supports strings, tuples and floats.")
+
+
+def create_cqt_kernels(
+ Q: float,
+ fs: float,
+ fmin: float,
+ n_bins: int = 84,
+ bins_per_octave: int = 12,
+ norm: int = 1,
+ window: str = "hann",
+ fmax: Optional[float] = None,
+ topbin_check: bool = True,
+) -> Tuple[np.array, int, np.array, np.array]:
+ """
+ Automatically create CQT kernels in time domain
+ """
+
+ fftLen = 2 ** next_power_of_2(np.ceil(Q * fs / fmin))
+
+ if (fmax is not None) and (n_bins is None):
+ n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin)) # Calculate the number of bins
+ freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))
+
+ elif (fmax is None) and (n_bins is not None):
+ freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))
+
+ else:
+ warnings.warn("If fmax is given, n_bins will be ignored", SyntaxWarning)
+ n_bins = np.ceil(bins_per_octave * np.log2(fmax / fmin)) # Calculate the number of bins
+ freqs = fmin * 2.0 ** (np.r_[0:n_bins] / np.float(bins_per_octave))
+
+ if np.max(freqs) > fs / 2 and topbin_check is True:
+ raise ValueError(
+ "The top bin {}Hz has exceeded the Nyquist frequency, please reduce the n_bins".format(np.max(freqs))
+ )
+
+ tempKernel = np.zeros((int(n_bins), int(fftLen)), dtype=np.complex64)
+
+ lengths = np.ceil(Q * fs / freqs)
+ for k in range(0, int(n_bins)):
+ freq = freqs[k]
+ _l = np.ceil(Q * fs / freq)
+
+ # Centering the kernels, pad more zeros on RHS
+ start = int(np.ceil(fftLen / 2.0 - _l / 2.0)) - int(_l % 2)
+
+ sig = (
+ get_window_dispatch(window, int(_l), fftbins=True)
+ * np.exp(np.r_[-_l // 2 : _l // 2] * 1j * 2 * np.pi * freq / fs)
+ / _l
+ )
+
+ if norm: # Normalizing the filter # Trying to normalize like librosa
+ tempKernel[k, start : start + int(_l)] = sig / np.linalg.norm(sig, norm)
+ else:
+ tempKernel[k, start : start + int(_l)] = sig
+
+ return tempKernel, fftLen, lengths, freqs
+
+
+def get_cqt_complex(
+ x: tf.Tensor,
+ cqt_kernels_real: tf.Tensor,
+ cqt_kernels_imag: tf.Tensor,
+ hop_length: int,
+ padding: tf.keras.layers.Layer,
+) -> tf.Tensor:
+ """Multiplying the STFT result with the cqt_kernel, check out the 1992 CQT paper [1]
+ for how to multiple the STFT result with the CQT kernel
+ [2] Brown, Judith C.C. and Miller Puckette. “An efficient algorithm for the calculation of
+ a constant Q transform.” (1992)."""
+
+ try:
+ x = padding(x) # When center is True, we need padding at the beginning and ending
+ except Exception:
+ warnings.warn(
+ f"\ninput size = {x.shape}\tkernel size = {cqt_kernels_real.shape[-1]}\n"
+ "padding with reflection mode might not be the best choice, try using constant padding",
+ UserWarning,
+ )
+ x = tf.pad(x, (cqt_kernels_real.shape[-1] // 2, cqt_kernels_real.shape[-1] // 2))
+ CQT_real = tf.transpose(
+ tf.nn.conv1d(
+ tf.transpose(x, [0, 2, 1]),
+ tf.transpose(cqt_kernels_real, [2, 1, 0]),
+ padding="VALID",
+ stride=hop_length,
+ ),
+ [0, 2, 1],
+ )
+ CQT_imag = -tf.transpose(
+ tf.nn.conv1d(
+ tf.transpose(x, [0, 2, 1]),
+ tf.transpose(cqt_kernels_imag, [2, 1, 0]),
+ padding="VALID",
+ stride=hop_length,
+ ),
+ [0, 2, 1],
+ )
+
+ return tf.stack((CQT_real, CQT_imag), axis=-1)
+
+
+def downsampling_by_n(x: tf.Tensor, filter_kernel: tf.Tensor, n: float, match_torch_exactly: bool = True) -> tf.Tensor:
+ """
+ Downsample the given tensor using the given filter kernel.
+ The input tensor is expected to have shape `(n_batches, channels, width)`,
+ and the filter kernel is expected to have shape `(num_output_channels,)` (i.e.: 1D)
+
+ If match_torch_exactly is passed, we manually pad the input rather than having TensorFlow do so with "SAME".
+ The result is subtly different than Torch's output, but it is compatible with TensorFlow Lite (as of v2.4.1).
+ """
+
+ if match_torch_exactly:
+ paddings = [
+ [0, 0],
+ [0, 0],
+ [(filter_kernel.shape[-1] - 1) // 2, (filter_kernel.shape[-1] - 1) // 2],
+ ]
+ padded = tf.pad(x, paddings)
+
+ # Store this tensor in the shape `(n_batches, width, channels)`
+ padded_nwc = tf.transpose(padded, [0, 2, 1])
+ result_nwc = tf.nn.conv1d(padded_nwc, filter_kernel[:, None, None], padding="VALID", stride=n)
+ else:
+ x_nwc = tf.transpose(x, [0, 2, 1])
+ result_nwc = tf.nn.conv1d(x_nwc, filter_kernel[:, None, None], padding="SAME", stride=n)
+ result_ncw = tf.transpose(result_nwc, [0, 2, 1])
+ return result_ncw
+
+
+class ReflectionPad1D(tf.keras.layers.Layer):
+ """
+ Replica of Torch's nn.ReflectionPad1D in TF.
+ """
+
+ def __init__(self, padding: Union[int, Tuple[int]] = 1, **kwargs: Any):
+ self.padding = padding
+ self.input_spec = [tf.keras.layers.InputSpec(ndim=3)]
+ super(ReflectionPad1D, self).__init__(**kwargs)
+
+ def compute_output_shape(self, s: List[int]) -> Tuple[int, int, int]:
+ return (s[0], s[1], s[2] + 2 * self.padding if isinstance(self.padding, int) else self.padding[0])
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ return tf.pad(x, [[0, 0], [0, 0], [self.padding, self.padding]], "REFLECT")
+
+
+class ConstantPad1D(tf.keras.layers.Layer):
+ """
+ Replica of Torch's nn.ConstantPad1D in TF.
+ """
+
+ def __init__(self, padding: Union[int, Tuple[int]] = 1, value: int = 0, **kwargs: Any):
+ self.padding = padding
+ self.value = value
+ self.input_spec = [tf.keras.layers.InputSpec(ndim=3)]
+ super(ConstantPad1D, self).__init__(**kwargs)
+
+ def compute_output_shape(self, s: List[int]) -> Tuple[int, int, int]:
+ return (s[0], s[1], s[2] + 2 * self.padding if isinstance(self.padding, int) else self.padding[0])
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ return tf.pad(x, [[0, 0], [0, 0], [self.padding, self.padding]], "CONSTANT", self.value)
+
+
+def pad_center(data: np.ndarray, size: int, axis: int = -1, **kwargs: Any) -> np.ndarray:
+ """Wrapper for np.pad to automatically center an array prior to padding.
+ This is analogous to `str.center()`
+
+ Examples
+ --------
+ >>> # Generate a vector
+ >>> data = np.ones(5)
+ >>> librosa.util.pad_center(data, 10, mode='constant')
+ array([ 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.])
+ >>> # Pad a matrix along its first dimension
+ >>> data = np.ones((3, 5))
+ >>> librosa.util.pad_center(data, 7, axis=0)
+ array([[ 0., 0., 0., 0., 0.],
+ [ 0., 0., 0., 0., 0.],
+ [ 1., 1., 1., 1., 1.],
+ [ 1., 1., 1., 1., 1.],
+ [ 1., 1., 1., 1., 1.],
+ [ 0., 0., 0., 0., 0.],
+ [ 0., 0., 0., 0., 0.]])
+ >>> # Or its second dimension
+ >>> librosa.util.pad_center(data, 7, axis=1)
+ array([[ 0., 1., 1., 1., 1., 1., 0.],
+ [ 0., 1., 1., 1., 1., 1., 0.],
+ [ 0., 1., 1., 1., 1., 1., 0.]])
+
+ Parameters
+ ----------
+ data : np.ndarray
+ Vector to be padded and centered
+ size : int >= len(data) [scalar]
+ Length to pad `data`
+ axis : int
+ Axis along which to pad and center the data
+ kwargs : additional keyword arguments
+ arguments passed to `np.pad()`
+
+ Returns
+ -------
+ data_padded : np.ndarray
+ `data` centered and padded to length `size` along the
+ specified axis
+
+ Raises
+ ------
+ ValueError
+ If `size < data.shape[axis]`
+
+ See Also
+ --------
+ numpy.pad
+ """
+
+ kwargs.setdefault("mode", "constant")
+
+ n = data.shape[axis]
+
+ lpad = int((size - n) // 2)
+
+ lengths = [(0, 0)] * data.ndim
+ lengths[axis] = (lpad, int(size - n - lpad))
+
+ if lpad < 0:
+ raise ValueError(("Target size ({:d}) must be at least input size ({:d})").format(size, n))
+
+ return np.pad(data, lengths, **kwargs)
+
+
+class CQT2010v2(tf.keras.layers.Layer):
+ """This layer calculates the CQT of the input signal.
+ Input signal should be in either of the following shapes.
+ 1. (len_audio)
+ 2. (num_audio, len_audio)
+ 3. (num_audio, 1, len_audio)
+ The correct shape will be inferred autommatically if the input follows these 3 shapes.
+ Most of the arguments follow the convention from librosa.
+
+ This layer uses about 1MB of memory per second of input audio with its default arguments.
+
+ This alogrithm uses the resampling method proposed in [1].
+ Instead of convoluting the STFT results with a gigantic CQT kernel covering the full frequency
+ spectrum, we make a small CQT kernel covering only the top octave. Then we keep downsampling the
+ input audio by a factor of 2 to convoluting it with the small CQT kernel.
+ Everytime the input audio is downsampled, the CQT relative to the downsampled input is equivalent
+ to the next lower octave.
+ The kernel creation process is still same as the 1992 algorithm. Therefore, we can reuse the
+ code from the 1992 alogrithm [2]
+ [1] Schörkhuber, Christian. “CONSTANT-Q TRANSFORM TOOLBOX FOR MUSIC PROCESSING.” (2010).
+ [2] Brown, Judith C.C. and Miller Puckette. “An efficient algorithm for the calculation of a
+ constant Q transform.” (1992).
+ Early downsampling factor is to downsample the input audio to reduce the CQT kernel size.
+ The result with and without early downsampling are more or less the same except in the very low
+ frequency region where freq < 40Hz.
+ Parameters
+ ----------
+ sr : int
+ The sampling rate for the input audio. It is used to calucate the correct ``fmin`` and ``fmax``.
+ Setting the correct sampling rate is very important for calculating the correct frequency.
+ hop_length : int
+ The hop (or stride) size. Default value is 512.
+ fmin : float
+ The frequency for the lowest CQT bin. Default is 32.70Hz, which coresponds to the note C0.
+ fmax : float
+ The frequency for the highest CQT bin. Default is ``None``, therefore the higest CQT bin is
+ inferred from the ``n_bins`` and ``bins_per_octave``. If ``fmax`` is not ``None``, then the
+ argument ``n_bins`` will be ignored and ``n_bins`` will be calculated automatically.
+ Default is ``None``
+ n_bins : int
+ The total numbers of CQT bins. Default is 84. Will be ignored if ``fmax`` is not ``None``.
+ bins_per_octave : int
+ Number of bins per octave. Default is 12.
+ norm : bool
+ Normalization for the CQT result.
+ basis_norm : int
+ Normalization for the CQT kernels. ``1`` means L1 normalization, and ``2`` means L2 normalization.
+ Default is ``1``, which is same as the normalization used in librosa.
+ window : str
+ The windowing function for CQT. It uses ``scipy.signal.get_window``, please refer to
+ scipy documentation for possible windowing functions. The default value is 'hann'
+ pad_mode : str
+ The padding method. Default value is 'reflect'.
+ trainable : bool
+ Determine if the CQT kernels are trainable or not. If ``True``, the gradients for CQT kernels
+ will also be caluclated and the CQT kernels will be updated during model training.
+ Default value is ``False``
+ output_format : str
+ Determine the return type.
+ 'Magnitude' will return the magnitude of the STFT result, shape = ``(num_samples, freq_bins, time_steps)``;
+ 'Complex' will return the STFT result in complex number, shape = ``(num_samples, freq_bins, time_steps, 2)``;
+ 'Phase' will return the phase of the STFT reuslt, shape = ``(num_samples, freq_bins,time_steps, 2)``.
+ The complex number is stored as ``(real, imag)`` in the last axis. Default value is 'Magnitude'.
+ verbose : bool
+ If ``True``, it shows layer information. If ``False``, it suppresses all prints.
+ device : str
+ Choose which device to initialize this layer. Default value is 'cpu'.
+ Returns
+ -------
+ spectrogram : tf.Tensor
+
+ It returns a tensor of spectrograms.
+ shape = ``(num_samples, freq_bins,time_steps)`` if ``output_format='Magnitude'``;
+ shape = ``(num_samples, freq_bins,time_steps, 2)`` if ``output_format='Complex' or 'Phase'``;
+ Examples
+ --------
+ >>> spec_layer = Spectrogram.CQT2010v2()
+ >>> specs = spec_layer(x)
+ """
+
+ def __init__(
+ self,
+ sr: int = 22050,
+ hop_length: int = 512,
+ fmin: float = 32.70,
+ fmax: Optional[float] = None,
+ n_bins: int = 84,
+ filter_scale: int = 1,
+ bins_per_octave: int = 12,
+ norm: bool = True,
+ basis_norm: int = 1,
+ window: str = "hann",
+ pad_mode: str = "reflect",
+ earlydownsample: bool = True,
+ trainable: bool = False,
+ output_format: str = "Magnitude",
+ match_torch_exactly: bool = True,
+ ):
+ super().__init__()
+
+ self.sample_rate: Union[float, int] = sr
+ self.hop_length = hop_length
+ self.fmin = fmin
+ self.fmax = fmax
+ self.n_bins = n_bins
+ self.filter_scale = filter_scale
+ self.bins_per_octave = bins_per_octave
+ self.norm = norm
+ self.basis_norm = basis_norm
+ self.window = window
+ self.pad_mode = pad_mode
+ self.earlydownsample = earlydownsample
+ self.trainable = trainable
+ self.output_format = output_format
+ self.match_torch_exactly = match_torch_exactly
+ self.normalization_type = "librosa"
+
+ def get_config(self) -> Any:
+ config = super().get_config().copy()
+ config.update(
+ {
+ "sample_rate": self.sample_rate,
+ "hop_length": self.hop_length,
+ "fmin": self.fmin,
+ "fmax": self.fmax,
+ "n_bins": self.n_bins,
+ "filter_scale": self.filter_scale,
+ "bins_per_octave": self.bins_per_octave,
+ "norm": self.norm,
+ "basis_norm": self.basis_norm,
+ "window": self.window,
+ "pad_mode": self.pad_mode,
+ "output_format": self.output_format,
+ "earlydownsample": self.earlydownsample,
+ "trainable": self.trainable,
+ "match_torch_exactly": self.match_torch_exactly,
+ }
+ )
+ return config
+
+ def build(self, input_shape: tf.TensorShape) -> None:
+ # This will be used to calculate filter_cutoff and creating CQT kernels
+ Q = float(self.filter_scale) / (2 ** (1 / self.bins_per_octave) - 1)
+
+ self.lowpass_filter = create_lowpass_filter(band_center=0.5, kernel_length=256, transition_bandwidth=0.001)
+
+ # Calculate num of filter requires for the kernel
+ # n_octaves determines how many resampling requires for the CQT
+ n_filters = min(self.bins_per_octave, self.n_bins)
+ self.n_octaves = int(np.ceil(float(self.n_bins) / self.bins_per_octave))
+
+ # Calculate the lowest frequency bin for the top octave kernel
+ self.fmin_t = self.fmin * 2 ** (self.n_octaves - 1)
+ remainder = self.n_bins % self.bins_per_octave
+
+ if remainder == 0:
+ # Calculate the top bin frequency
+ fmax_t = self.fmin_t * 2 ** ((self.bins_per_octave - 1) / self.bins_per_octave)
+ else:
+ # Calculate the top bin frequency
+ fmax_t = self.fmin_t * 2 ** ((remainder - 1) / self.bins_per_octave)
+
+ self.fmin_t = fmax_t / 2 ** (1 - 1 / self.bins_per_octave) # Adjusting the top minium bins
+ if fmax_t > self.sample_rate / 2:
+ raise ValueError(
+ "The top bin {}Hz has exceeded the Nyquist frequency, please reduce the n_bins".format(fmax_t)
+ )
+
+ if self.earlydownsample is True: # Do early downsampling if this argument is True
+ (
+ self.sample_rate,
+ self.hop_length,
+ self.downsample_factor,
+ early_downsample_filter,
+ self.earlydownsample,
+ ) = get_early_downsample_params(self.sample_rate, self.hop_length, fmax_t, Q, self.n_octaves, self.dtype)
+
+ self.early_downsample_filter = early_downsample_filter
+ else:
+ self.downsample_factor = 1.0
+
+ # Preparing CQT kernels
+ basis, self.n_fft, _, _ = create_cqt_kernels(
+ Q,
+ self.sample_rate,
+ self.fmin_t,
+ n_filters,
+ self.bins_per_octave,
+ norm=self.basis_norm,
+ topbin_check=False,
+ )
+
+ # For the normalization in the end
+ # The freqs returned by create_cqt_kernels cannot be used
+ # Since that returns only the top octave bins
+ # We need the information for all freq bin
+ freqs = self.fmin * 2.0 ** (np.r_[0 : self.n_bins] / np.float(self.bins_per_octave))
+ self.frequencies = freqs
+
+ self.lengths = np.ceil(Q * self.sample_rate / freqs)
+
+ self.basis = basis
+ # NOTE(psobot): this is where the implementation here starts to differ from CQT2010.
+
+ # These cqt_kernel is already in the frequency domain
+ self.cqt_kernels_real = tf.expand_dims(basis.real.astype(self.dtype), 1)
+ self.cqt_kernels_imag = tf.expand_dims(basis.imag.astype(self.dtype), 1)
+
+ if self.trainable:
+ self.cqt_kernels_real = tf.Variable(initial_value=self.cqt_kernels_real, trainable=True)
+ self.cqt_kernels_imag = tf.Variable(initial_value=self.cqt_kernels_imag, trainable=True)
+
+ # If center==True, the STFT window will be put in the middle, and paddings at the beginning
+ # and ending are required.
+ if self.pad_mode == "constant":
+ self.padding = ConstantPad1D(self.n_fft // 2, 0)
+ elif self.pad_mode == "reflect":
+ self.padding = ReflectionPad1D(self.n_fft // 2)
+
+ rank = len(input_shape)
+ if rank == 2:
+ self.reshape_input = lambda x: x[:, None, :]
+ elif rank == 1:
+ self.reshape_input = lambda x: x[None, None, :]
+ elif rank == 3:
+ self.reshape_input = lambda x: x
+ else:
+ raise ValueError(f"Input shape must be rank <= 3, found shape {input_shape}")
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ x = self.reshape_input(x) # type: ignore
+
+ if self.earlydownsample is True:
+ x = downsampling_by_n(x, self.early_downsample_filter, self.downsample_factor, self.match_torch_exactly)
+
+ hop = self.hop_length
+
+ # Getting the top octave CQT
+ CQT = get_cqt_complex(x, self.cqt_kernels_real, self.cqt_kernels_imag, hop, self.padding)
+
+ x_down = x # Preparing a new variable for downsampling
+
+ for _ in range(self.n_octaves - 1):
+ hop = hop // 2
+ x_down = downsampling_by_n(x_down, self.lowpass_filter, 2, self.match_torch_exactly)
+ CQT1 = get_cqt_complex(x_down, self.cqt_kernels_real, self.cqt_kernels_imag, hop, self.padding)
+ CQT = tf.concat((CQT1, CQT), axis=1)
+
+ CQT = CQT[:, -self.n_bins :, :] # Removing unwanted bottom bins
+
+ # Normalizing the output with the downsampling factor, 2**(self.n_octaves-1) is make it
+ # same mag as 1992
+ CQT = CQT * self.downsample_factor
+
+ # Normalize again to get same result as librosa
+ if self.normalization_type == "librosa":
+ CQT *= tf.math.sqrt(tf.cast(self.lengths.reshape((-1, 1, 1)), self.dtype))
+ elif self.normalization_type == "convolutional":
+ pass
+ elif self.normalization_type == "wrap":
+ CQT *= 2
+ else:
+ raise ValueError("The normalization_type %r is not part of our current options." % self.normalization_type)
+
+ # Transpose the output to match the output of the other spectrogram layers.
+ if self.output_format.lower() == "magnitude":
+ # Getting CQT Amplitude
+ return tf.transpose(tf.math.sqrt(tf.math.reduce_sum(tf.math.pow(CQT, 2), axis=-1)), [0, 2, 1])
+
+ elif self.output_format.lower() == "complex":
+ return CQT
+
+ elif self.output_format.lower() == "phase":
+ phase_real = tf.math.cos(tf.math.atan2(CQT[:, :, :, 1], CQT[:, :, :, 0]))
+ phase_imag = tf.math.sin(tf.math.atan2(CQT[:, :, :, 1], CQT[:, :, :, 0]))
+ return tf.stack((phase_real, phase_imag), axis=-1)
+
+
+CQT = CQT2010v2
diff --git a/basic_pitch/layers/signal.py b/basic_pitch/layers/signal.py
new file mode 100644
index 0000000..f5f629f
--- /dev/null
+++ b/basic_pitch/layers/signal.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Optional
+import tensorflow as tf
+from basic_pitch.layers.math import log_base_b
+
+
+class Stft(tf.keras.layers.Layer):
+ def __init__(
+ self,
+ fft_length: int = 2048,
+ hop_length: Optional[int] = None,
+ window_length: Optional[int] = None,
+ window_fn: Callable[[int, tf.dtypes.DType], tf.Tensor] = tf.signal.hann_window,
+ pad_end: bool = False,
+ center: bool = True,
+ pad_mode: str = "REFLECT",
+ name: Optional[str] = None,
+ dtype: tf.dtypes.DType = tf.float32,
+ ):
+ """
+ A Tensorflow Keras layer that calculates an STFT.
+ The input is real-valued with shape (num_batches, num_samples).
+ The output is complex-valued with shape (num_batches, time, fft_length // 2 + 1)
+
+ Args:
+ hop_length: The "stride" or number of samples to iterate before the start of the next frame.
+ fft_length: FFT length.
+ window_length: Window length. If None, then fft_length is used.
+ window_fn: A callable that takes a window length and a dtype and returns a window.
+ pad_end: Whether to pad the end of signals with zeros when the provided frame length and step produces
+ a frame that lies partially past its end.
+ center:
+ If True, the signal y is padded so that frame D[:, t] is centered at y[t * hop_length].
+ If False, then D[:, t] begins at y[t * hop_length].
+ pad_mode: Padding to use if center is True. One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive).
+ name: Name of the layer.
+ dtype: Type used in calcuation.
+ """
+ super().__init__(trainable=False, name=name, dtype=dtype, dynamic=False)
+ self.fft_length = fft_length
+ self.window_length = window_length if window_length else self.fft_length
+ self.hop_length = hop_length if hop_length else self.window_length // 4
+ self.window_fn = window_fn
+ self.final_window_fn = window_fn
+ self.pad_end = pad_end
+ self.center = center
+ self.pad_mode = pad_mode
+
+ def build(self, input_shape: tf.TensorShape) -> None:
+ if self.window_length < self.fft_length:
+ lpad = (self.fft_length - self.window_length) // 2
+ rpad = self.fft_length - self.window_length - lpad
+
+ def padded_window(window_length: int, dtype: tf.dtypes.DType = tf.float32) -> tf.Tensor:
+ # This is a trick to match librosa's way of handling window lengths < their fft_lengths
+ # In that case the window is 0 padded such that the window is centered around 0s
+ # In the Tensorflow case, the window is computed, multiplied against the frame and then
+ # Right padded with 0's.
+ return tf.pad(self.window_fn(self.window_length, dtype=dtype), [[lpad, rpad]]) # type: ignore
+
+ self.final_window_fn = padded_window
+
+ if self.center:
+ self.spec = tf.keras.layers.Lambda(
+ lambda x: tf.pad(
+ x,
+ [[0, 0] for _ in range(input_shape.rank - 1)] + [[self.fft_length // 2, self.fft_length // 2]],
+ mode=self.pad_mode,
+ )
+ )
+ else:
+ self.spec = tf.keras.layers.Lambda(lambda x: x)
+
+ def call(self, inputs: tf.Tensor) -> tf.Tensor:
+ return tf.signal.stft(
+ signals=self.spec(inputs),
+ frame_length=self.fft_length,
+ frame_step=self.hop_length,
+ fft_length=self.fft_length,
+ window_fn=self.final_window_fn,
+ pad_end=self.pad_end,
+ )
+
+ def get_config(self) -> Any:
+ config = super().get_config().copy()
+ config.update(
+ {
+ "fft_length": self.fft_length,
+ "window_length": self.window_length,
+ "hop_length": self.hop_length,
+ "window_fn": self.window_fn,
+ "pad_end": self.pad_end,
+ "center": self.center,
+ "pad_mode": self.pad_mode,
+ }
+ )
+ return config
+
+
+class Spectrogram(Stft):
+ def __init__(
+ self,
+ power: int = 2,
+ *args: Any,
+ **kwargs: Any,
+ ):
+ """
+ A Tensorflow Keras layer that calculates the magnitude spectrogram.
+ The input is real-valued with shape (num_batches, num_samples).
+ The output is real-valued with shape (num_batches, time, fft_length // 2 + 1)
+
+ Args:
+ power: Exponent to raise abs(stft) to.
+ **kwargs: Any arguments that you'd pass to Stft
+ """
+ super().__init__(
+ *args,
+ **kwargs,
+ )
+ self.power = power
+
+ def call(self, inputs: tf.Tensor) -> tf.Tensor:
+ return tf.math.pow(
+ tf.math.abs(super().call(inputs)),
+ self.power,
+ )
+
+ def get_config(self) -> Any:
+ config = super().get_config().copy()
+ config.update(
+ {
+ "power": self.power,
+ }
+ )
+ return config
+
+
+class NormalizedLog(tf.keras.layers.Layer):
+ """
+ Takes an input with a shape of either (batch, x, y, z) or (batch, y, z)
+ and rescales each (y, z) to dB, scaled 0 - 1.
+ Only x=1 is supported.
+ This layer adds 1e-10 to all values as a way to avoid NaN math.
+ """
+
+ def build(self, input_shape: tf.Tensor) -> None:
+ self.squeeze_batch = lambda batch: batch
+ rank = input_shape.rank
+ if rank == 4:
+ assert input_shape[1] == 1, "If the rank is 4, the second dimension must be length 1"
+ self.squeeze_batch = lambda batch: tf.squeeze(batch, axis=1)
+ else:
+ assert rank == 3, f"Only ranks 3 and 4 are supported!. Received rank {rank} for {input_shape}."
+
+ def call(self, inputs: tf.Tensor) -> tf.Tensor:
+ inputs = self.squeeze_batch(inputs) # type: ignore
+ # convert magnitude to power
+ power = tf.math.square(inputs)
+ log_power = 10 * log_base_b(power + 1e-10, 10)
+
+ log_power_min = tf.reshape(tf.math.reduce_min(log_power, axis=[1, 2]), [tf.shape(inputs)[0], 1, 1])
+ log_power_offset = log_power - log_power_min
+ log_power_offset_max = tf.reshape(
+ tf.math.reduce_max(log_power_offset, axis=[1, 2]),
+ [tf.shape(inputs)[0], 1, 1],
+ )
+ log_power_normalized = tf.math.divide_no_nan(log_power_offset, log_power_offset_max)
+
+ return tf.reshape(log_power_normalized, tf.shape(inputs))
diff --git a/basic_pitch/models.py b/basic_pitch/models.py
new file mode 100644
index 0000000..0e72651
--- /dev/null
+++ b/basic_pitch/models.py
@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Dict
+import numpy as np
+import tensorflow as tf
+
+from basic_pitch import nn
+from basic_pitch.constants import (
+ ANNOTATIONS_BASE_FREQUENCY,
+ ANNOTATIONS_N_SEMITONES,
+ AUDIO_N_SAMPLES,
+ AUDIO_SAMPLE_RATE,
+ CONTOURS_BINS_PER_SEMITONE,
+ FFT_HOP,
+ N_FREQ_BINS_CONTOURS,
+)
+from basic_pitch.layers import signal, nnaudio
+
+tfkl = tf.keras.layers
+
+MAX_N_SEMITONES = int(np.floor(12.0 * np.log2(0.5 * AUDIO_SAMPLE_RATE / ANNOTATIONS_BASE_FREQUENCY)))
+
+
+def transcription_loss(y_true: tf.Tensor, y_pred: tf.Tensor, label_smoothing: float) -> tf.Tensor:
+ """Really a binary cross entropy loss. Used to calculate the loss between the predicted
+ posteriorgrams and the ground truth matrices.
+
+ Args:
+ y_true: The true labels.
+ y_pred: The predicted labels.
+ label_smoothing: Squeeze labels towards 0.5.
+
+ Returns:
+ The transcription loss.
+ """
+ bce = tf.keras.losses.binary_crossentropy(y_true, y_pred, label_smoothing=label_smoothing)
+ return bce
+
+
+def weighted_transcription_loss(
+ y_true: tf.Tensor, y_pred: tf.Tensor, label_smoothing: float, positive_weight: float = 0.5
+) -> tf.Tensor:
+ """The transcription loss where the positive and negative true labels are balanced by a weighting factor.
+
+ Args:
+ y_true: The true labels.
+ y_pred: The predicted labels.
+ label_smoothing: Smoothing factor. Squeezes labels towards 0.5.
+ positive_weight: Weighting factor for the positive labels.
+
+ Returns:
+ The weighted transcription loss.
+ """
+ negative_mask = tf.equal(y_true, 0)
+ nonnegative_mask = tf.logical_not(negative_mask)
+ bce_negative = tf.keras.losses.binary_crossentropy(
+ tf.boolean_mask(y_true, negative_mask),
+ tf.boolean_mask(y_pred, negative_mask),
+ label_smoothing=label_smoothing,
+ )
+ bce_nonnegative = tf.keras.losses.binary_crossentropy(
+ tf.boolean_mask(y_true, nonnegative_mask),
+ tf.boolean_mask(y_pred, nonnegative_mask),
+ label_smoothing=label_smoothing,
+ )
+ return ((1 - positive_weight) * bce_negative) + (positive_weight * bce_nonnegative)
+
+
+def onset_loss(
+ weighted: bool, label_smoothing: float, positive_weight: float
+) -> Callable[[tf.Tensor, tf.Tensor], tf.Tensor]:
+ """
+
+ Args:
+ weighted: Whether or not to use a weighted cross entropy loss.
+ label_smoothing: Smoothing factor. Squeezes labels towards 0.5.
+ positive_weight: Weighting factor for the positive labels.
+
+ Returns:
+ A function that calculates the transcription loss. The function will
+ return weighted_transcription_loss if weighted is true else it will return
+ transcription_loss.
+ """
+ if weighted:
+ return lambda x, y: weighted_transcription_loss(
+ x, y, label_smoothing=label_smoothing, positive_weight=positive_weight
+ )
+ return lambda x, y: transcription_loss(x, y, label_smoothing=label_smoothing)
+
+
+def loss(label_smoothing: float = 0.2, weighted: bool = False, positive_weight: float = 0.5) -> Dict[str, Any]:
+ """Creates a keras-compatible dictionary of loss functions to calculate
+ the loss for the contour, note and onset posteriorgrams.
+
+ Args:
+ label_smoothing: Smoothing factor. Squeezes labels towards 0.5.
+ weighted: Whether or not to use a weighted cross entropy loss.
+ positive_weight: Weighting factor for the positive labels.
+
+ Returns:
+ A dictionary with keys "contour," "note," and "onset" with functions as values to be used to calculate
+ transcription losses.
+
+ """
+ loss_fn = lambda x, y: transcription_loss(x, y, label_smoothing=label_smoothing)
+ loss_onset = onset_loss(weighted, label_smoothing, positive_weight)
+ return {
+ "contour": loss_fn,
+ "note": loss_fn,
+ "onset": loss_onset,
+ }
+
+
+def _initializer() -> tf.keras.initializers.VarianceScaling:
+ return tf.keras.initializers.VarianceScaling(scale=2.0, mode="fan_avg", distribution="uniform", seed=None)
+
+
+def _kernel_constraint() -> tf.keras.constraints.UnitNorm:
+ return tf.keras.constraints.UnitNorm(axis=[0, 1, 2])
+
+
+def get_cqt(inputs: tf.Tensor, n_harmonics: int, use_batchnorm: bool) -> tf.Tensor:
+ """Calculate the CQT of the input audio.
+
+ Input shape: (batch, number of audio samples, 1)
+ Output shape: (batch, number of frequency bins, number of time frames)
+
+ Args:
+ inputs: The audio input.
+ n_harmonics: The number of harmonics to capture above the maximum output frequency.
+ Used to calculate the number of semitones for the CQT.
+ use_batchnorm: If True, applies batch normalization after computing the CQT
+
+ Returns:
+ The log-normalized CQT of the input audio.
+ """
+ n_semitones = np.min(
+ [
+ int(np.ceil(12.0 * np.log2(n_harmonics)) + ANNOTATIONS_N_SEMITONES),
+ MAX_N_SEMITONES,
+ ]
+ )
+ x = nn.FlattenAudioCh()(inputs)
+ x = nnaudio.CQT(
+ sr=AUDIO_SAMPLE_RATE,
+ hop_length=FFT_HOP,
+ fmin=ANNOTATIONS_BASE_FREQUENCY,
+ n_bins=n_semitones * CONTOURS_BINS_PER_SEMITONE,
+ bins_per_octave=12 * CONTOURS_BINS_PER_SEMITONE,
+ )(x)
+ x = signal.NormalizedLog()(x)
+ x = tf.expand_dims(x, -1)
+ if use_batchnorm:
+ x = tfkl.BatchNormalization()(x)
+ return x
+
+
+def model(
+ n_harmonics: int = 8,
+ n_filters_contour: int = 32,
+ n_filters_onsets: int = 32,
+ n_filters_notes: int = 32,
+ no_contours: bool = False,
+) -> tf.keras.Model:
+ """Basic Pitch's model implementation.
+
+ Args:
+ n_harmonics: The number of harmonics to use in the harmonic stacking layer.
+ n_filters_contour: Number of filters for the contour convolutional layer.
+ n_filters_onsets: Number of filters for the onsets convolutional layer.
+ n_filters_notes: Number of filters for the notes convolutional layer.
+ no_contours: Whether or not to include contours in the output.
+ """
+ # input representation
+ inputs = tf.keras.Input(shape=(AUDIO_N_SAMPLES, 1)) # (batch, time, ch)
+ x = get_cqt(inputs, n_harmonics, True)
+
+ if n_harmonics > 1:
+ x = nn.HarmonicStacking(
+ CONTOURS_BINS_PER_SEMITONE,
+ [0.5] + list(range(1, n_harmonics)),
+ N_FREQ_BINS_CONTOURS,
+ )(x)
+ else:
+ x = nn.HarmonicStacking(
+ CONTOURS_BINS_PER_SEMITONE,
+ [1],
+ N_FREQ_BINS_CONTOURS,
+ )(x)
+
+ # contour layers - fully convolutional
+ x_contours = tfkl.Conv2D(
+ n_filters_contour,
+ (5, 5),
+ padding="same",
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ )(x)
+
+ x_contours = tfkl.BatchNormalization()(x_contours)
+ x_contours = tfkl.ReLU()(x_contours)
+
+ x_contours = tfkl.Conv2D(
+ 8,
+ (3, 3 * 13),
+ padding="same",
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ )(x)
+
+ x_contours = tfkl.BatchNormalization()(x_contours)
+ x_contours = tfkl.ReLU()(x_contours)
+
+ if not no_contours:
+ contour_name = "contour"
+ x_contours = tfkl.Conv2D(
+ 1,
+ (5, 5),
+ padding="same",
+ activation="sigmoid",
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ name="contours-reduced",
+ )(x_contours)
+ x_contours = nn.FlattenFreqCh(name=contour_name)(x_contours) # contour output
+
+ # reduced contour output as input to notes
+ x_contours_reduced = tf.expand_dims(x_contours, -1)
+ else:
+ x_contours_reduced = x_contours
+
+ x_contours_reduced = tfkl.Conv2D(
+ n_filters_notes,
+ (7, 7),
+ padding="same",
+ strides=(1, 3),
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ )(x_contours_reduced)
+ x_contours_reduced = tfkl.ReLU()(x_contours_reduced)
+
+ # note output layer
+ note_name = "note"
+ x_notes_pre = tfkl.Conv2D(
+ 1,
+ (7, 3),
+ padding="same",
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ activation="sigmoid",
+ )(x_contours_reduced)
+ x_notes = nn.FlattenFreqCh(name=note_name)(x_notes_pre)
+
+ # onset output layer
+
+ # onsets - fully convolutional
+ x_onset = tfkl.Conv2D(
+ n_filters_onsets,
+ (5, 5),
+ padding="same",
+ strides=(1, 3),
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ )(x)
+ x_onset = tfkl.BatchNormalization()(x_onset)
+ x_onset = tfkl.ReLU()(x_onset)
+ x_onset = tfkl.Concatenate(axis=3, name="concat")([x_notes_pre, x_onset])
+ x_onset = tfkl.Conv2D(
+ 1,
+ (3, 3),
+ padding="same",
+ activation="sigmoid",
+ kernel_initializer=_initializer(),
+ kernel_constraint=_kernel_constraint(),
+ )(x_onset)
+
+ onset_name = "onset"
+ x_onset = nn.FlattenFreqCh(
+ name=onset_name,
+ )(x_onset)
+
+ outputs = {"onset": x_onset, "contour": x_contours, "note": x_notes}
+
+ return tf.keras.Model(inputs=inputs, outputs=outputs)
diff --git a/basic_pitch/nn.py b/basic_pitch/nn.py
new file mode 100644
index 0000000..2a85648
--- /dev/null
+++ b/basic_pitch/nn.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, List
+
+import tensorflow as tf
+import tensorflow.keras.backend as K
+
+from basic_pitch.layers.math import log_base_b
+
+
+class HarmonicStacking(tf.keras.layers.Layer):
+ """Harmonic stacking layer
+
+ Input shape: (n_batch, n_times, n_freqs, 1)
+ Output shape: (n_batch, n_times, n_output_freqs, len(harmonics))
+
+ n_freqs should be much larger than n_output_freqs so that information from the upper
+ harmonics is captured.
+
+ Attributes:
+ bins_per_semitone: The number of bins per semitone of the input CQT
+ harmonics: List of harmonics to use. Should be positive numbers.
+ shifts: A list containing the number of bins to shift in frequency for each harmonic
+ n_output_freqs: The number of frequency bins in each harmonic layer.
+ """
+
+ def __init__(
+ self, bins_per_semitone: int, harmonics: List[float], n_output_freqs: int, name: str = "harmonic_stacking"
+ ):
+ """Downsample frequency by stride, upsample channels by 4."""
+ super().__init__(trainable=False, name=name)
+ self.bins_per_semitone = bins_per_semitone
+ self.harmonics = harmonics
+ self.shifts = [
+ int(tf.math.round(12.0 * self.bins_per_semitone * log_base_b(float(h), 2))) for h in self.harmonics
+ ]
+ self.n_output_freqs = n_output_freqs
+
+ def get_config(self) -> Any:
+ config = super().get_config().copy()
+ config.update(
+ {
+ "bins_per_semitone": self.bins_per_semitone,
+ "harmonics": self.harmonics,
+ "n_output_freqs": self.n_output_freqs,
+ "name": self.name,
+ }
+ )
+ return config
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ # (n_batch, n_times, n_freqs, 1)
+ tf.debugging.assert_equal(tf.shape(x).shape, 4)
+ channels = []
+ for shift in self.shifts:
+ if shift == 0:
+ padded = x
+ elif shift > 0:
+ paddings = tf.constant([[0, 0], [0, 0], [0, shift], [0, 0]])
+ padded = tf.pad(x[:, :, shift:, :], paddings)
+ elif shift < 0:
+ paddings = tf.constant([[0, 0], [0, 0], [-shift, 0], [0, 0]])
+ padded = tf.pad(x[:, :, :shift, :], paddings)
+ else:
+ raise ValueError
+
+ channels.append(padded)
+ x = tf.concat(channels, axis=-1)
+ x = x[:, :, : self.n_output_freqs, :] # return only the first n_output_freqs frequency channels
+ return x
+
+
+class FlattenAudioCh(tf.keras.layers.Layer):
+ """Layer which removes a "channels" dimension of size 1.
+
+ Input shape: (batch, time, 1)
+ Output shape: (batch, time)
+ """
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ """x: (batch, time, ch)"""
+ shapes = K.int_shape(x)
+ tf.assert_equal(shapes[2], 1)
+ return tf.keras.layers.Reshape([shapes[1]])(x) # ignore batch size
+
+
+class FlattenFreqCh(tf.keras.layers.Layer):
+ """Layer to flatten the frequency channel and make each channel
+ part of the frequency dimension.
+
+ Input shape: (batch, time, freq, ch)
+ Output shape: (batch, time, freq*ch)
+ """
+
+ def call(self, x: tf.Tensor) -> tf.Tensor:
+ shapes = K.int_shape(x)
+ return tf.keras.layers.Reshape([shapes[1], shapes[2] * shapes[3]])(x) # ignore batch size
diff --git a/basic_pitch/note_creation.py b/basic_pitch/note_creation.py
new file mode 100644
index 0000000..b66d233
--- /dev/null
+++ b/basic_pitch/note_creation.py
@@ -0,0 +1,479 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pathlib
+from typing import Dict, List, Optional, Tuple, Union
+import mir_eval
+import librosa
+import resampy
+import numpy as np
+import pretty_midi
+import scipy
+from scipy.io import wavfile
+
+from basic_pitch.constants import (
+ AUDIO_SAMPLE_RATE,
+ ANNOTATIONS_N_SEMITONES,
+ ANNOTATIONS_BASE_FREQUENCY,
+ AUDIO_N_SAMPLES,
+ ANNOT_N_FRAMES,
+ CONTOURS_BINS_PER_SEMITONE,
+ FFT_HOP,
+ N_FREQ_BINS_CONTOURS,
+)
+
+MIDI_OFFSET = 21
+SONIFY_FS = 3000
+N_PITCH_BEND_TICKS = 8192
+MAX_FREQ_IDX = 87
+
+
+def model_output_to_notes(
+ output: Dict[str, np.array],
+ onset_thresh: float,
+ frame_thresh: float,
+ infer_onsets: bool = True,
+ min_note_len: int = 5,
+ min_freq: Optional[float] = None,
+ max_freq: Optional[float] = None,
+ include_pitch_bends: bool = True,
+ melodia_trick: bool = True,
+) -> Tuple[pretty_midi.PrettyMIDI, List[Tuple[float, float, int, float, Optional[List[int]]]]]:
+ """Convert model output to MIDI
+
+ Args:
+ output: A dictionary with shape
+ {
+ 'frame': array of shape (n_times, n_freqs),
+ 'onset': array of shape (n_times, n_freqs),
+ 'contour': array of shape (n_times, 3*n_freqs)
+ }
+ representing the output of the basic pitch model.
+ onset_thresh: Minimum amplitude of an onset activation to be considered an onset.
+ infer_onsets: If True, add additional onsets when there are large differences in frame amplitudes.
+ min_note_len: The minimum allowed note length in frames.
+ min_freq: Minimum allowed output frequency, in Hz. If None, all frequencies are used.
+ max_freq: Maximum allowed output frequency, in Hz. If None, all frequencies are used.
+ melodia_trick: Use the melodia post-processing step.
+
+ Returns:
+ midi : pretty_midi.PrettyMIDI object
+ note_events: A list of note event tuples (start_time_s, end_time_s, pitch_midi, amplitude)
+ """
+ frames = output["note"]
+ onsets = output["onset"]
+ contours = output["contour"]
+
+ estimated_notes = output_to_notes_polyphonic(
+ frames,
+ onsets,
+ onset_thresh=onset_thresh,
+ frame_thresh=frame_thresh,
+ infer_onsets=infer_onsets,
+ min_note_len=min_note_len,
+ min_freq=min_freq,
+ max_freq=max_freq,
+ melodia_trick=melodia_trick,
+ )
+ if include_pitch_bends:
+ estimated_notes_with_pitch_bend = get_pitch_bends(contours, estimated_notes)
+ else:
+ estimated_notes_with_pitch_bend = [(note[0], note[1], note[2], note[3], None) for note in estimated_notes]
+
+ times_s = model_frames_to_time(contours.shape[0])
+ estimated_notes_time_seconds = [
+ (times_s[note[0]], times_s[note[1]], note[2], note[3], note[4]) for note in estimated_notes_with_pitch_bend
+ ]
+
+ return note_events_to_midi(estimated_notes_time_seconds), estimated_notes_time_seconds
+
+
+def sonify_midi(midi: pretty_midi.PrettyMIDI, save_path: Union[pathlib.Path, str]) -> None:
+ """Sonify a pretty_midi midi object and save to a file.
+
+ Args:
+ midi: A pretty_midi.PrettyMIDI object that will be sonified.
+ save_path: Where to save the sonified midi.
+ """
+ y = midi.synthesize(44100)
+ wavfile.write(save_path, 44100, y)
+
+
+def sonify_salience(
+ gram: np.array, semitone_resolution: float, save_path: Optional[str] = None, thresh: float = 0.2
+) -> Tuple[np.array, int]:
+ """Sonify a salience matrix.
+
+ Args:
+ gram: A matrix of pitch salience values with range 0-1, with shape (n_freqs, n_times).
+ The frequencies are logarithmically spaced.
+ semitone_resolution: The number of bins per semitone in gram.
+ save_path: Optional location to save the sonified salience.
+ thresh: Salience values below thresh will not be sonified. Used to increase the speed of this function.
+
+ Returns:
+ A tuple of the sonified salience as an audio signal and the associated sample rate.
+ """
+ freqs = librosa.core.cqt_frequencies(
+ n_bins=ANNOTATIONS_N_SEMITONES * semitone_resolution,
+ fmin=ANNOTATIONS_BASE_FREQUENCY,
+ bins_per_octave=12 * semitone_resolution,
+ )
+ # this function is slow - for speed, only sonify frequencies below
+ # sonify_fs/2 Hz (e.g. 1000 Hz)
+ max_freq_idx = np.where(freqs > SONIFY_FS / 2)[0][0]
+ times = librosa.core.frames_to_time(
+ np.arange(gram.shape[1]),
+ sr=AUDIO_SAMPLE_RATE,
+ hop_length=AUDIO_N_SAMPLES / ANNOT_N_FRAMES, # THIS IS THE CORRECT HOP!!
+ )
+ gram[gram < thresh] = 0
+ y = mir_eval.sonify.time_frequency(gram[:max_freq_idx, :], freqs[:max_freq_idx], times, fs=SONIFY_FS)
+ if save_path:
+ y_resamp = resampy.resample(y, SONIFY_FS, 44100)
+ wavfile.write(save_path, 44100, y_resamp)
+
+ return y, SONIFY_FS
+
+
+def midi_pitch_to_contour_bin(pitch_midi: int) -> np.array:
+ """Convert midi pitch to conrresponding index in contour matrix
+
+ Args:
+ pitch_midi: pitch in midi
+
+ Returns:
+ index in contour matrix
+
+ """
+ pitch_hz = librosa.midi_to_hz(pitch_midi)
+ return 12.0 * CONTOURS_BINS_PER_SEMITONE * np.log2(pitch_hz / ANNOTATIONS_BASE_FREQUENCY)
+
+
+def get_pitch_bends(
+ contours: np.ndarray, note_events: List[Tuple[int, int, int, float]], n_bins_tolerance: int = 25
+) -> List[Tuple[int, int, int, float, Optional[List[int]]]]:
+ """Given note events and contours, estimate pitch bends per note.
+ Pitch bends are represented as a sequence of evenly spaced midi pitch bend control units.
+ The time stamps of each pitch bend can be inferred by computing an evenly spaced grid between
+ the start and end times of each note event.
+
+ Args:
+ contours: Matrix of estimated pitch contours
+ note_events: note event tuple
+ n_bins_tolerance: Pitch bend estimation range. Defaults to 25.
+
+ Returns:
+ note events with pitch bends
+ """
+ window_length = n_bins_tolerance * 2 + 1
+ freq_gaussian = scipy.signal.gaussian(window_length, std=5)
+ note_events_with_pitch_bends = []
+ for start_idx, end_idx, pitch_midi, amplitude in note_events:
+ freq_idx = int(np.round(midi_pitch_to_contour_bin(pitch_midi)))
+ freq_start_idx = np.max([freq_idx - n_bins_tolerance, 0])
+ freq_end_idx = np.min([N_FREQ_BINS_CONTOURS, freq_idx + n_bins_tolerance + 1])
+
+ pitch_bend_submatrix = (
+ contours[start_idx:end_idx, freq_start_idx:freq_end_idx]
+ * freq_gaussian[
+ np.max([0, n_bins_tolerance - freq_idx]) : window_length
+ - np.max([0, freq_idx - (N_FREQ_BINS_CONTOURS - n_bins_tolerance - 1)])
+ ]
+ )
+ pb_shift = n_bins_tolerance - np.max([0, n_bins_tolerance - freq_idx])
+
+ bends: Optional[List[int]] = list(
+ np.argmax(pitch_bend_submatrix, axis=1) - pb_shift
+ ) # this is in units of 1/3 semitones
+ note_events_with_pitch_bends.append((start_idx, end_idx, pitch_midi, amplitude, bends))
+ return note_events_with_pitch_bends
+
+
+def note_events_to_midi(
+ note_events_with_pitch_bends: List[Tuple[float, float, int, float, Optional[List[int]]]]
+) -> pretty_midi.PrettyMIDI:
+ """Create a pretty_midi object from note events
+
+ Args:
+ note_events : list of tuples [(start_time_seconds, end_time_seconds, pitch_midi, amplitude)]
+ where amplitude is a number between 0 and 1
+ save_path : path to save midi file. If None, no midi file is saved
+
+ Returns:
+ pretty_midi.PrettyMIDI() object
+
+ """
+ mid = pretty_midi.PrettyMIDI()
+
+ piano_program = pretty_midi.instrument_name_to_program("Electric Piano 1")
+ instrument_per_piano_key = {i: pretty_midi.Instrument(program=piano_program) for i in range(21, 109)}
+ for start_time, end_time, note_number, amplitude, pitch_bend in note_events_with_pitch_bends:
+ note = pretty_midi.Note(
+ velocity=int(np.round(127 * amplitude)),
+ pitch=note_number,
+ start=start_time,
+ end=end_time,
+ )
+
+ instrument_per_piano_key[note_number].notes.append(note)
+ if not pitch_bend:
+ continue
+ pitch_bend_times = np.linspace(start_time, end_time, len(pitch_bend))
+ pitch_bend_midi_ticks = np.round(np.array(pitch_bend) * 4096 / CONTOURS_BINS_PER_SEMITONE).astype(int)
+ # This supports pitch bends up to 2 semitones
+ # If we estimate pitch bends above/below 2 semitones, crop them here when adding them to the midi file
+ pitch_bend_midi_ticks[pitch_bend_midi_ticks > N_PITCH_BEND_TICKS - 1] = N_PITCH_BEND_TICKS - 1
+ pitch_bend_midi_ticks[pitch_bend_midi_ticks < -N_PITCH_BEND_TICKS] = -N_PITCH_BEND_TICKS
+ for pb_time, pb_midi in zip(pitch_bend_times, pitch_bend_midi_ticks):
+ instrument_per_piano_key[note_number].pitch_bends.append(pretty_midi.PitchBend(pb_midi, pb_time))
+
+ for inst in instrument_per_piano_key.values():
+ # only add instruments with notes
+ if len(inst.notes) > 0:
+ mid.instruments.append(inst)
+
+ return mid
+
+
+def get_infered_onsets(onsets: np.array, frames: np.array, n_diff: int = 2) -> np.array:
+ """Infer onsets from large changes in frame amplitudes.
+
+ Args:
+ onsets: Array of note onset predictions.
+ frames: Audio frames.
+ n_diff: Differences used to detect onsets.
+
+ Returns:
+ The maximum between the predicted onsets and its differences.
+ """
+ diffs = []
+ for n in range(1, n_diff + 1):
+ frames_appended = np.concatenate([np.zeros((n, frames.shape[1])), frames])
+ diffs.append(frames_appended[n:, :] - frames_appended[:-n, :])
+ frame_diff = np.min(diffs, axis=0)
+ frame_diff[frame_diff < 0] = 0
+ frame_diff[:n_diff, :] = 0
+ frame_diff = np.max(onsets) * frame_diff / np.max(frame_diff) # rescale to have the same max as onsets
+
+ max_onsets_diff = np.max([onsets, frame_diff], axis=0) # use the max of the predicted onsets and the differences
+
+ return max_onsets_diff
+
+
+def constrain_frequency(
+ onsets: np.array, frames: np.array, max_freq: Optional[float], min_freq: Optional[float]
+) -> Tuple[np.array, np.array]:
+ """Zero out activations above or below the max/min frequencies
+
+ Args:
+ onsets: Onset activation matrix (n_times, n_freqs)
+ frames: Frame activation matrix (n_times, n_freqs)
+ max_freq: The maximum frequency to keep.
+ min_freq: the minimum frequency to keep.
+
+ Returns:
+ The onset and frame activation matrices, with frequencies outside the min and max
+ frequency set to 0.
+ """
+ if max_freq is not None:
+ max_freq_idx = int(np.round(librosa.hz_to_midi(max_freq) - MIDI_OFFSET))
+ onsets[:, max_freq_idx:] = 0
+ frames[:, max_freq_idx:] = 0
+ if min_freq is not None:
+ min_freq_idx = int(np.round(librosa.hz_to_midi(min_freq) - MIDI_OFFSET))
+ onsets[:, :min_freq_idx] = 0
+ frames[:, :min_freq_idx] = 0
+
+ return onsets, frames
+
+
+def model_frames_to_time(n_frames: int) -> np.ndarray:
+ original_times = librosa.core.frames_to_time(
+ np.arange(n_frames),
+ sr=AUDIO_SAMPLE_RATE,
+ hop_length=FFT_HOP,
+ )
+ window_numbers = np.floor(np.arange(n_frames) / ANNOT_N_FRAMES)
+ window_offset = (FFT_HOP / AUDIO_SAMPLE_RATE) * (
+ ANNOT_N_FRAMES - (AUDIO_N_SAMPLES / FFT_HOP)
+ ) + 0.0018 # this is a magic number, but it's needed for this to align properly
+ times = original_times - (window_offset * window_numbers)
+ return times
+
+
+def output_to_notes_polyphonic(
+ frames: np.array,
+ onsets: np.array,
+ onset_thresh: float,
+ frame_thresh: float,
+ min_note_len: int,
+ infer_onsets: bool,
+ max_freq: Optional[float],
+ min_freq: Optional[float],
+ melodia_trick: bool = True,
+ energy_tol: int = 11,
+) -> List[Tuple[int, int, int, float]]:
+ """Decode raw model output to polyphonic note events
+
+ Args:
+ frames: Frame activation matrix (n_times, n_freqs).
+ onsets: Onset activation matrix (n_times, n_freqs).
+ onset_thresh: Minimum amplitude of an onset activation to be considered an onset.
+ frame_thresh: Minimum amplitude of a frame activation for a note to remain "on".
+ min_note_len: Minimum allowed note length in frames.
+ infer_onsets: If True, add additional onsets when there are large differences in frame amplitudes.
+ max_freq: Maximum allowed output frequency, in Hz.
+ min_freq: Minimum allowed output frequency, in Hz.
+ melodia_trick : Whether to use the melodia trick to better detect notes.
+ energy_tol: Drop notes below this energy.
+
+ Returns:
+ list of tuples [(start_time_frames, end_time_frames, pitch_midi, amplitude)]
+ representing the note events, where amplitude is a number between 0 and 1
+ """
+ if onset_thresh > np.max(onsets):
+ print("warning - adjusted onset_thresh")
+ onset_thresh = np.max(onsets) - (0.5 * np.std(onsets))
+
+ n_frames = frames.shape[0]
+
+ onsets, frames = constrain_frequency(onsets, frames, max_freq, min_freq)
+ # use onsets inferred from frames in addition to the predicted onsets
+ if infer_onsets:
+ onsets = get_infered_onsets(onsets, frames)
+
+ peak_thresh_mat = np.zeros(onsets.shape)
+ peaks = scipy.signal.argrelmax(onsets, axis=0)
+ peak_thresh_mat[peaks] = onsets[peaks]
+
+ onset_idx = np.where(peak_thresh_mat >= onset_thresh)
+ onset_time_idx = onset_idx[0][::-1] # sort to go backwards in time
+ onset_freq_idx = onset_idx[1][::-1] # sort to go backwards in time
+
+ remaining_energy = np.zeros(frames.shape)
+ removed_frames = []
+ remaining_energy[:, :] = frames[:, :]
+
+ # loop over onsets
+ note_events = []
+ for note_start_idx, freq_idx in zip(onset_time_idx, onset_freq_idx):
+ # if we're too close to the end of the audio, continue
+ if note_start_idx >= n_frames - 1:
+ continue
+
+ # find time index at this frequency band where the frames drop below an energy threshold
+ i = note_start_idx + 1
+ k = 0 # number of frames since energy dropped below threshold
+ while i < n_frames - 1 and k < energy_tol:
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+ i += 1
+
+ i -= k # go back to frame above threshold
+
+ # if the note is too short, skip it
+ if i - note_start_idx <= min_note_len:
+ continue
+
+ remaining_energy[note_start_idx:i, freq_idx] = 0
+ for j in range(note_start_idx, i):
+ removed_frames.append([j, freq_idx])
+
+ # add the note
+ amplitude = np.mean(frames[note_start_idx:i, freq_idx])
+ note_events.append(
+ (
+ note_start_idx,
+ i,
+ freq_idx + MIDI_OFFSET,
+ amplitude,
+ )
+ )
+
+ if melodia_trick:
+
+ energy_shape = remaining_energy.shape
+ # delete semitone neighbors of already used notes
+ for j, freq_idx in removed_frames:
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[j, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[j, freq_idx - 1] = 0
+
+ while np.max(remaining_energy) > frame_thresh:
+ i_mid, freq_idx = np.unravel_index(np.argmax(remaining_energy), energy_shape)
+ remaining_energy[i_mid, freq_idx] = 0
+
+ # forward pass
+ i = i_mid + 1
+ k = 0
+ while i < n_frames - 1 and k < energy_tol:
+ remaining_energy[i, freq_idx] = 0
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[i, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[i, freq_idx - 1] = 0
+
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+
+ i += 1
+
+ i_end = i - 1 - k # go back to frame above threshold
+
+ # backward pass
+ i = i_mid - 1
+ k = 0
+ while i > 0 and k < energy_tol:
+ remaining_energy[i, freq_idx] = 0
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[i, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[i, freq_idx - 1] = 0
+
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+
+ i -= 1
+
+ i_start = i + 1 + k # go back to frame above threshold
+ assert i_start >= 0, "{}".format(i_start)
+ assert i_end < n_frames
+
+ if i_end - i_start <= min_note_len:
+ # note is too short or too quiet, skip it and remove the energy
+ continue
+
+ # add the note
+ amplitude = np.mean(frames[i_start:i_end, freq_idx])
+ note_events.append(
+ (
+ i_start,
+ i_end,
+ freq_idx + MIDI_OFFSET,
+ amplitude,
+ )
+ )
+
+ return note_events
diff --git a/basic_pitch/predict.py b/basic_pitch/predict.py
new file mode 100644
index 0000000..146dc7d
--- /dev/null
+++ b/basic_pitch/predict.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import pathlib
+
+from basic_pitch import ICASSP_2022_MODEL_PATH
+
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+
+def main() -> None:
+ """Handle command line arguments. Entrypoint for this script."""
+ parser = argparse.ArgumentParser(description="Predict midi from audio.")
+ parser.add_argument("output_dir", type=str, help="directory to save outputs")
+ parser.add_argument("audio_paths", type=str, nargs="+", help="Space separated paths to the input audio files.")
+ parser.add_argument(
+ "--model_path",
+ type=str,
+ default=ICASSP_2022_MODEL_PATH,
+ help="path to the saved model directory. Defaults to a ICASSP 2022 model",
+ )
+ parser.add_argument(
+ "--save-midi",
+ action="store_true",
+ default=True,
+ help="Create a MIDI file.",
+ )
+ parser.add_argument(
+ "--sonify-midi",
+ action="store_true",
+ help="Create an audio .wav file which sonifies the MIDI outputs.",
+ )
+ parser.add_argument(
+ "--save-model-outputs",
+ action="store_true",
+ help="Save the raw model output as an npz file.",
+ )
+ parser.add_argument(
+ "--save-note-events",
+ action="store_true",
+ help="Save the predicted note events as a csv file.",
+ )
+ parser.add_argument(
+ "--onset-threshold",
+ type=float,
+ default=0.5,
+ help="The minimum likelihood for an onset to occur, between 0 and 1.",
+ )
+ parser.add_argument(
+ "--frame-threshold",
+ type=float,
+ default=0.3,
+ help="The minimum likelihood for a frame to sustain, between 0 and 1.",
+ )
+ parser.add_argument(
+ "--minimum-note-length",
+ type=float,
+ default=58,
+ help="The minimum allowed note length, in miliseconds.",
+ )
+ parser.add_argument(
+ "--minimum-frequency",
+ type=float,
+ default=None,
+ help="The minimum allowed note frequency, in Hz.",
+ )
+ parser.add_argument(
+ "--maximum-frequency",
+ type=float,
+ default=None,
+ help="The maximum allowed note frequency, in Hz.",
+ )
+ parser.add_argument("--debug-file", default=None, help="Optional file for debug output for inference.")
+ #
+ parser.add_argument("--no-melodia", default=False, action="store_true", help="Skip the melodia trick.")
+ args = parser.parse_args()
+
+ print("")
+ print("✨✨✨✨✨✨✨✨✨")
+ print("✨ Basic Pitch ✨")
+ print("✨✨✨✨✨✨✨✨✨")
+
+ # tensorflow is very slow to import
+ # this import is here so that the help messages print faster
+ from basic_pitch.inference import predict_and_save, verify_output_dir, verify_input_path
+
+ output_dir = pathlib.Path(args.output_dir)
+ verify_output_dir(output_dir)
+
+ audio_path_list = [pathlib.Path(audio_path) for audio_path in args.audio_paths]
+ for audio_path in audio_path_list:
+ verify_input_path(audio_path)
+
+ predict_and_save(
+ audio_path_list,
+ output_dir,
+ args.save_midi,
+ args.sonify_midi,
+ args.save_model_outputs,
+ args.save_note_events,
+ pathlib.Path(args.model_path),
+ args.onset_threshold,
+ args.frame_threshold,
+ args.minimum_note_length,
+ args.minimum_frequency,
+ args.maximum_frequency,
+ not args.no_melodia,
+ pathlib.Path(args.debug_file) if args.debug_file else None,
+ )
+
+ print("\n✨ Done ✨\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/basic_pitch/saved_models/icassp_2022/nmp/saved_model.pb b/basic_pitch/saved_models/icassp_2022/nmp/saved_model.pb
new file mode 100644
index 0000000..3a22fc9
Binary files /dev/null and b/basic_pitch/saved_models/icassp_2022/nmp/saved_model.pb differ
diff --git a/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.data-00000-of-00001 b/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.data-00000-of-00001
new file mode 100644
index 0000000..a473eb0
Binary files /dev/null and b/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.data-00000-of-00001 differ
diff --git a/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.index b/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.index
new file mode 100644
index 0000000..dba4ca4
Binary files /dev/null and b/basic_pitch/saved_models/icassp_2022/nmp/variables/variables.index differ
diff --git a/catalog-info.yaml b/catalog-info.yaml
new file mode 100644
index 0000000..e38bb99
--- /dev/null
+++ b/catalog-info.yaml
@@ -0,0 +1,7 @@
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+ name: basic-pitch
+spec:
+ type: resource
+ owner: MIQ
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..199f963
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,65 @@
+[metadata]
+name = basic-pitch
+version = attr: basic_pitch.__version__
+description = attr: basic_pitch.__description__
+long_description = file: README.md LICENSE
+author = attr: basic_pitch.__author__
+author_email = attr: basic_pitch.__author_email__
+maintainer = attr: basic_pitch.__maintainer__
+maintainer_email = attr: basic_pitch.__maintainer_email__
+keywords =
+license = Apache 2.0
+classifiers =
+ Development Status :: 5 - Production/Stable
+ Natural Language :: English
+ Operating System :: POSIX :: Linux
+ Operating System :: MacOS :: MacOS X
+ Operating System :: Microsoft :: Windows
+ Programming Language :: Python
+ Programming Language :: Python :: 3.7
+ Programming Language :: Python :: 3.8
+ Programming Language :: Python :: 3.9
+ Programming Language :: Python :: 3.10
+ Programming Language :: Python :: Implementation :: CPython
+
+[options]
+zip_safe = False
+packages = find:
+include_package_data = True
+install_requires =
+ librosa>=0.8.0
+ mir_eval>=0.6
+ numpy>=1.19.2,<1.20
+ pretty_midi>=0.2.9
+ resampy>=0.2.2
+ scipy>=1.4.1
+ tensorflow>=2.4.1,<2.7.0
+ typing_extensions
+
+[options.entry_points]
+console_scripts =
+ basic-pitch = basic_pitch.predict:main
+
+[options.extras_require]
+test =
+ coverage>=5.0.2
+ pytest>=6.1.1
+ pytest-mock
+docs =
+ mkdocs>=1.0.4
+dev =
+ basic_pitch[test,docs]
+ bump2version>=1.0.1
+ mypy
+ tox
+
+
+[bumpversion]
+current_version = 0.0.1
+commit = True
+tag = True
+
+[bumpversion:file:basic_pitch/__init__.py]
+
+[bdist_wheel]
+universal = 1
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..005210b
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+
+if __name__ == "__main__":
+ setup()
diff --git a/tests/resources/vocadito_10.wav b/tests/resources/vocadito_10.wav
new file mode 100644
index 0000000..147817e
Binary files /dev/null and b/tests/resources/vocadito_10.wav differ
diff --git a/tests/resources/vocadito_14.wav b/tests/resources/vocadito_14.wav
new file mode 100644
index 0000000..ee55d88
Binary files /dev/null and b/tests/resources/vocadito_14.wav differ
diff --git a/tests/test_inference.py b/tests/test_inference.py
new file mode 100644
index 0000000..23a477e
--- /dev/null
+++ b/tests/test_inference.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pathlib
+import tempfile
+import unittest
+
+import librosa
+import pretty_midi
+import numpy as np
+
+from basic_pitch import ICASSP_2022_MODEL_PATH, inference
+from basic_pitch.constants import ANNOTATIONS_N_SEMITONES
+
+RESOURCES_PATH = pathlib.Path(__file__).parent / "resources"
+
+
+class TestPredict(unittest.TestCase):
+ def test_predict(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ model_output, midi_data, note_events = inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ )
+ assert set(model_output.keys()) == set(["note", "onset", "contour"])
+ assert model_output["note"].shape == model_output["onset"].shape
+ assert isinstance(midi_data, pretty_midi.PrettyMIDI)
+ lowest_supported_midi = 21
+ note_pitch_min = [n[2] >= lowest_supported_midi for n in note_events]
+ note_pitch_max = [n[2] <= lowest_supported_midi + ANNOTATIONS_N_SEMITONES for n in note_events]
+ assert all(note_pitch_min)
+ assert all(note_pitch_max)
+ assert isinstance(note_events, list)
+
+ def test_predict_with_saves(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ with tempfile.TemporaryDirectory() as tmpdir:
+ inference.predict_and_save(
+ [test_audio_path],
+ tmpdir,
+ True,
+ True,
+ True,
+ True,
+ )
+ expected_midi_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.mid")
+ expected_csv_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.csv")
+ expected_npz_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.npz")
+ expected_sonif_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.wav")
+
+ for output_path in [expected_midi_path, expected_csv_path, expected_npz_path, expected_sonif_path]:
+ assert os.path.exists(output_path)
+
+ def test_predict_onset_threshold(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ for onset_threshold in [0, 0.3, 0.8, 1]:
+ inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ onset_threshold=onset_threshold,
+ )
+
+ def test_predict_frame_threshold(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ for frame_threshold in [0, 0.3, 0.8, 1]:
+ inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ frame_threshold=frame_threshold,
+ )
+
+ def test_predict_min_note_length(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ for minimum_note_length in [10, 100, 1000]:
+ _, _, note_events = inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ minimum_note_length=minimum_note_length,
+ )
+ min_len_s = minimum_note_length / 1000.0
+ note_lengths = [n[1] - n[0] >= min_len_s for n in note_events]
+ assert all(note_lengths)
+
+ def test_predict_min_freq(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ for minimum_frequency in [40, 80, 200, 2000]:
+ _, _, note_events = inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ minimum_frequency=minimum_frequency,
+ )
+ min_freq_midi = np.round(librosa.hz_to_midi(minimum_frequency))
+ note_pitch = [n[2] >= min_freq_midi for n in note_events]
+ assert all(note_pitch)
+
+ def test_predict_max_freq(self) -> None:
+ test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
+ for maximum_frequency in [40, 80, 200, 2000]:
+ _, _, note_events = inference.predict(
+ test_audio_path,
+ ICASSP_2022_MODEL_PATH,
+ maximum_frequency=maximum_frequency,
+ )
+ max_freq_midi = np.round(librosa.hz_to_midi(maximum_frequency))
+ note_pitch = [n[2] <= max_freq_midi for n in note_events]
+ assert all(note_pitch)
diff --git a/tests/test_nn.py b/tests/test_nn.py
new file mode 100644
index 0000000..7e2b1c0
--- /dev/null
+++ b/tests/test_nn.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterator, Tuple
+import unittest
+
+import numpy as np
+import tensorflow as tf
+
+from basic_pitch import models, nn
+from basic_pitch.constants import (
+ ANNOT_N_FRAMES,
+ ANNOTATIONS_N_SEMITONES,
+ AUDIO_N_SAMPLES,
+ N_FREQ_BINS_CONTOURS,
+)
+
+BATCH_SIZE = 3
+
+tfkl = tf.keras.layers
+
+
+class TestHarmonicStacking(unittest.TestCase):
+ def _audio_data_gen(self) -> Iterator[Tuple[np.array, np.array]]:
+ while True:
+ audio = np.random.uniform(size=(BATCH_SIZE, AUDIO_N_SAMPLES, 1)).astype(np.float32)
+ output = np.random.uniform(size=(BATCH_SIZE, ANNOT_N_FRAMES, ANNOTATIONS_N_SEMITONES * 3, 1)).astype(
+ np.float32
+ )
+ yield (audio, output)
+
+ def _dummy_dataset(self) -> tf.data.Dataset:
+ ds = tf.data.Dataset.from_generator(
+ self._audio_data_gen,
+ (tf.float32, tf.float32),
+ (
+ tf.TensorShape([BATCH_SIZE, AUDIO_N_SAMPLES, 1]),
+ tf.TensorShape([BATCH_SIZE, ANNOT_N_FRAMES, ANNOTATIONS_N_SEMITONES * 3, 1]),
+ ),
+ )
+ return ds
+
+ def test_defaults(self) -> None:
+ inputs = tf.keras.Input(shape=(AUDIO_N_SAMPLES, 1))
+ x = models.get_cqt(inputs, 5, True)
+ conv = nn.HarmonicStacking(3, [1, 2, 3, 4, 5], N_FREQ_BINS_CONTOURS)(x)
+ outputs = tfkl.Conv2D(1, (3, 3), padding="same")(conv)
+ model = tf.keras.Model(inputs=inputs, outputs=outputs)
+
+ train_ds = self._dummy_dataset()
+ valid_ds = self._dummy_dataset()
+
+ model.compile(
+ loss="binary_crossentropy",
+ optimizer=tf.keras.optimizers.Adam(0.1),
+ sample_weight_mode=None,
+ )
+
+ model.fit(
+ train_ds,
+ epochs=2,
+ steps_per_epoch=2,
+ validation_data=valid_ds,
+ validation_steps=1,
+ )
+
+ for element in train_ds:
+ in_vals = element[0]
+ out_vals = model(in_vals)
+ assert out_vals.shape == (
+ BATCH_SIZE,
+ ANNOT_N_FRAMES,
+ ANNOTATIONS_N_SEMITONES * 3,
+ 1,
+ )
+ break
+
+ def test_fractions(self) -> None:
+ inputs = tf.keras.Input(shape=(AUDIO_N_SAMPLES, 1))
+ x = models.get_cqt(inputs, 2, True)
+ conv = nn.HarmonicStacking(3, [0.5, 1, 2], N_FREQ_BINS_CONTOURS)(x)
+ outputs = tfkl.Conv2D(1, (3, 3), padding="same")(conv)
+ model = tf.keras.Model(inputs=inputs, outputs=outputs)
+
+ train_ds = self._dummy_dataset()
+ valid_ds = self._dummy_dataset()
+
+ model.compile(
+ loss="binary_crossentropy",
+ optimizer=tf.keras.optimizers.Adam(0.1),
+ sample_weight_mode=None,
+ )
+
+ model.fit(
+ train_ds,
+ epochs=2,
+ steps_per_epoch=2,
+ validation_data=valid_ds,
+ validation_steps=1,
+ )
+
+ for element in train_ds:
+ inputs = element[0]
+ output = model(inputs)
+ assert output.shape == (
+ BATCH_SIZE,
+ ANNOT_N_FRAMES,
+ ANNOTATIONS_N_SEMITONES * 3,
+ 1,
+ )
+ break
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..e140288
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,56 @@
+[tox]
+envlist = py37,py38,py39,manifest,check-formatting,lint,mypy
+skipsdist = True
+usedevelop = True
+
+[testenv]
+deps = -e .[dev]
+commands =
+ pytest tests {posargs}
+setenv =
+ SOURCE = {toxinidir}/basic_pitch
+ TEST_SOURCE = {toxinidir}/tests
+ LINE_LENGTH = "120"
+
+[testenv:manifest]
+deps = check-manifest
+skip_install = true
+commands = check-manifest --ignore 'tests/*'
+
+[testenv:check-formatting]
+basepython = python3.8
+deps = black
+skip_install = true
+commands =
+ black {env:SOURCE} tests --line-length {env:LINE_LENGTH} --diff --check
+
+[testenv:format]
+basepython = python3.8
+deps = black
+skip_install = true
+commands =
+ black {env:SOURCE} tests --line-length {env:LINE_LENGTH}
+
+[testenv:lint]
+basepython = python3.8
+deps = flake8
+skip_install = true
+commands = flake8
+
+[testenv:mypy]
+basepython = python3.8
+deps =
+ mypy
+commands = mypy basic_pitch/layers tests --strict --ignore-missing-imports --allow-subclassing-any
+
+[flake8]
+show-source = true
+max-line-length = 120
+exclude = .venv,.tox,.git,dist,doc,*.egg,build
+ignore = E203,E731,W503,
+
+[pytest]
+addopts = -v
+testpaths = tests
+
+