diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml new file mode 100644 index 00000000..80b1f18a --- /dev/null +++ b/.github/workflows/acceptance.yml @@ -0,0 +1,70 @@ +name: Acceptance tests + +# Builds the in-repo dogfood plugins and runs the acceptance (golden-file) +# self-tests via CTest. This is independent of Tracktion's main private build +# pipeline; it exists so the `pluginval test` feature has a public, reproducible +# regression check across the three desktop platforms. + +on: + push: + branches: [ master, develop, v2 ] + pull_request: + workflow_dispatch: + +concurrency: + group: acceptance-${{ github.ref }} + cancel-in-progress: true + +jobs: + acceptance: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + + env: + # Cache CPM's downloads (JUCE etc.) so reruns don't re-fetch them. + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cpm-cache + + steps: + - uses: actions/checkout@v4 + + - name: Cache CPM sources + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.cpm-cache + key: cpm-${{ matrix.os }}-${{ hashFiles('CMakeLists.txt', 'cmake/CPM.cmake') }} + restore-keys: cpm-${{ matrix.os }}- + + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + libasound2-dev libx11-dev libxext-dev libxinerama-dev libxrandr-dev \ + libxcursor-dev libxcomposite-dev libfreetype6-dev libfontconfig1-dev \ + libgl1-mesa-dev libcurl4-openssl-dev ladspa-sdk ninja-build xvfb + + # The acceptance tests host the dogfood plugins via JUCE's own VST3 hosting, + # so the embedded VST3 validator (and its heavy SDK build) isn't needed here. + - name: Configure + run: > + cmake -B build + -DCMAKE_BUILD_TYPE=Release + -DPLUGINVAL_BUILD_TEST_PLUGINS=ON + -DPLUGINVAL_VST3_VALIDATOR=OFF + + - name: Build + run: cmake --build build --config Release + + - name: Run acceptance tests (Linux) + if: runner.os == 'Linux' + working-directory: build + run: xvfb-run --auto-servernum ctest -C Release --output-on-failure -R pluginval.acceptance + + - name: Run acceptance tests (macOS / Windows) + if: runner.os != 'Linux' + working-directory: build + run: ctest -C Release --output-on-failure -R pluginval.acceptance diff --git a/CLAUDE.md b/CLAUDE.md index 97aa0061..d6f27fef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,7 @@ Replace `` with the ID from step 2. ``` pluginval/ -├── Source/ # Main application source code +├── source/ # Main application source code │ ├── Main.cpp # Application entry point │ ├── MainComponent.cpp/h # GUI main window component │ ├── Validator.cpp/h # Core validation orchestration @@ -93,6 +93,11 @@ pluginval/ │ ├── vst3validator/ # Embedded VST3 validator integration │ │ ├── VST3ValidatorRunner.h │ │ └── VST3ValidatorRunner.cpp +│ ├── acceptance/ # `pluginval test` golden-file subsystem (parallel to validate) +│ │ ├── TestConfig.cpp/h # snake_case JSON config struct (independent of PluginvalSettings) +│ │ ├── AcceptanceTest.cpp/h # plugin load + state + input + render; record-or-compare orchestrator +│ │ ├── ReferenceComparator.cpp/h # Comparator interface + registry + SampleComparator +│ │ └── TestReporter.cpp/h # text + JSON result, exit code │ └── tests/ # Individual test implementations │ ├── BasicTests.cpp # Core plugin tests (info, state, audio) │ ├── BusTests.cpp # Audio bus configuration tests @@ -108,6 +113,10 @@ pluginval/ ├── tests/ │ ├── AddPluginvalTests.cmake # CMake module for CTest integration │ ├── test_plugins/ # Test plugin files +│ │ ├── tone_generator/ # Deterministic dogfood generator (PLUGINVAL_BUILD_TEST_PLUGINS) +│ │ ├── gain/ # Deterministic dogfood gain effect (for the input.audio path) +│ │ └── playhead_probe/ # Writes the host transport to output (for the playhead path) +│ ├── acceptance/ # Acceptance self-tests: configs (*.json.in), inputs/, checked-in refs/ WAVs │ ├── mac_tests/ # macOS-specific tests │ └── windows_tests.bat # Windows test scripts ├── docs/ # Documentation @@ -150,6 +159,7 @@ cmake --build Builds/Debug --config Debug | `WITH_ADDRESS_SANITIZER` | Enable AddressSanitizer | OFF | | `WITH_THREAD_SANITIZER` | Enable ThreadSanitizer | OFF | | `VST2_SDK_DIR` | Path to VST2 SDK (env var) | - | +| `PLUGINVAL_BUILD_TEST_PLUGINS` | Build the in-repo dogfood plugins + acceptance CTest self-tests | OFF | ### Enabling VST2 Support @@ -258,6 +268,48 @@ settings set via a base64-encoded JSON argument (`--config-base64`), avoiding per-flag re-serialisation and command-line quoting hazards. `--help`/`--version` are handled by CLI11 (auto usage + a footer with the env-var/commands notes). +### Acceptance Testing (`pluginval test`) + +A **parallel subsystem** to validate, in `source/acceptance/`. It answers "does +this plugin produce the expected output for a known input + state?" — a +deterministic *render + golden-file comparison*, not a unit-test pass/fail. Full +spec: `tests/acceptance/Acceptance testing design.md`; end-user guide: +`docs/Acceptance testing.md`. + +- **CLI**: `pluginval test `. A new `Command::test` is recognised + by `settings_parser::dispatch()` (captures the positional config path into + `DispatchResult::testConfigPath`), `isCommandLine()` and `getFooterText()`; + `CommandLine.cpp`'s `performCommandLine()` has a `Command::test` branch that + runs the acceptance runner **synchronously on the message thread** and quits. +- **Config**: `acceptance::TestConfig` (`TestConfig.cpp/h`) — std-typed struct, + **snake_case JSON keys** mapped via explicit `to_json`/`from_json` (members + stay camelCase). It is **independent** of `PluginvalSettings` / the `--config` + layering: the test config is a positional argument loaded standalone. +- **Flow** (`AcceptanceTest.cpp`): load plugin → apply `state.file` then + `state.parameters` (normalised, matched by index / case-insensitive name or + paramID) → feed `input.audio`/`input.midi` or silence → if a `playhead` is + configured, point a fixed-tempo transport (`FixedPlayHead`, position advances + per block) at the plugin → render a fixed duration block-by-block (reusing the + `AudioProcessingTest` shape + the VST3-safe helpers in `TestUtilities.h`). If + no reference exists it **records** one (32-bit float WAV + `.wav.json` + sidecar manifest); otherwise it **compares** and writes a diff WAV on failure. + Exit `0`/`1`. +- **Comparators** (`ReferenceComparator.cpp/h`): pluggable `Comparator` + + `createComparator(name)` registry. v1 ships only `sample` (per-sample abs-diff + tolerance, default one 16-bit LSB = `1/32768`; `0` = bit-exact). Adding + `spectrum`/`crosscorr`/etc. is one registry entry, no config/runner changes. +- **Dogfood + self-tests**: three minimal deterministic `juce_add_plugin` targets + behind `PLUGINVAL_BUILD_TEST_PLUGINS` — `tests/test_plugins/tone_generator/` + (closed-form sine/square generator, phase resets on `prepareToPlay`), + `tests/test_plugins/gain/` (a gain effect, dogfoods the `input.audio` path) and + `tests/test_plugins/playhead_probe/` (writes the host transport to its output, + dogfoods the `playhead` path). `tests/acceptance/` holds checked-in configs + (`*.json.in`, the plugin paths + input dir substituted at configure time), + `inputs/` and reference WAVs, run via CTest (`pluginval.acceptance.*`: sine-440, + square-220, square-state, gain-half, playhead-120). Phase 2 items (config-array + multiplexing, automation, extra comparators, child-process isolation) are notes + only. + ### Test Framework Tests are self-registering. To find all tests, look for static instances: @@ -298,12 +350,12 @@ The VST3 validator (Steinberg's vstvalidator) is embedded into pluginval when bu 1. The VST3 SDK is fetched via CPM during CMake configure 2. The SDK's own `validator` target is built as a separate executable 3. A CMake script (`cmake/GenerateBinaryHeader.cmake`) converts the compiled binary into a C byte array header -4. `VST3ValidatorRunner` (`Source/vst3validator/`) extracts the embedded binary to a temp file on first use +4. `VST3ValidatorRunner` (`source/vst3validator/`) extracts the embedded binary to a temp file on first use 5. When the `VST3validator` test runs, it spawns the extracted validator as a subprocess **Key files:** - `cmake/GenerateBinaryHeader.cmake` — binary-to-C-header conversion script -- `Source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File` +- `source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File` **Disabling embedded validator:** ```bash @@ -486,7 +538,7 @@ Run internal tests via CLI: ## Common Tasks for AI Assistants ### Finding Where Tests Are Defined -- All test classes are in `Source/tests/*.cpp` +- All test classes are in `source/tests/*.cpp` - Search for `static.*Test.*Test;` to find registrations - Each test subclasses `PluginTest` diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a7eab78..c1e7c12a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,10 @@ endif() option(WITH_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) option(WITH_THREAD_SANITIZER "Enable Thread Sanitizer" OFF) +# Builds the in-repo dogfood plugins (e.g. the tone generator) used by the +# acceptance self-tests. Off for normal release builds. +option(PLUGINVAL_BUILD_TEST_PLUGINS "Build the in-repo test plugins used by the acceptance self-tests" OFF) + message(STATUS "Sanitizers: ASan=${WITH_ADDRESS_SANITIZER} TSan=${WITH_THREAD_SANITIZER}") if (WITH_ADDRESS_SANITIZER) if (MSVC) @@ -105,7 +109,7 @@ endif() juce_add_gui_app(pluginval BUNDLE_ID com.Tracktion.pluginval COMPANY_NAME Tracktion - ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/Source/binarydata/icon.png" + ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/source/binarydata/icon.png" HARDENED_RUNTIME_ENABLED TRUE HARDENED_RUNTIME_OPTIONS com.apple.security.cs.allow-unsigned-executable-memory com.apple.security.cs.disable-library-validation com.apple.security.get-task-allow) @@ -118,42 +122,50 @@ set_target_properties(pluginval PROPERTIES CXX_VISIBILITY_PRESET hidden) set(SourceFiles - Source/CommandLine.h - Source/CrashHandler.h - Source/MainComponent.h - Source/PluginTests.h - Source/PluginvalSettings.h - Source/SettingsParser.h - Source/SettingsSerializer.h - Source/TestUtilities.h - Source/Validator.h - Source/CommandLine.cpp - Source/CrashHandler.cpp - Source/Main.cpp - Source/MainComponent.cpp - Source/PluginTests.cpp - Source/SettingsParser.cpp - Source/SettingsSerializer.cpp - Source/tests/BasicTests.cpp - Source/tests/LocaleTest.cpp - Source/tests/BusTests.cpp - Source/tests/EditorTests.cpp - Source/tests/ExtremeTests.cpp - Source/tests/ParameterFuzzTests.cpp - Source/TestUtilities.cpp - Source/Validator.cpp) + source/CommandLine.h + source/CrashHandler.h + source/MainComponent.h + source/PluginTests.h + source/PluginvalSettings.h + source/SettingsParser.h + source/SettingsSerializer.h + source/TestUtilities.h + source/Validator.h + source/acceptance/TestConfig.h + source/acceptance/TestConfig.cpp + source/acceptance/AcceptanceTest.h + source/acceptance/AcceptanceTest.cpp + source/acceptance/ReferenceComparator.h + source/acceptance/ReferenceComparator.cpp + source/acceptance/TestReporter.h + source/acceptance/TestReporter.cpp + source/CommandLine.cpp + source/CrashHandler.cpp + source/Main.cpp + source/MainComponent.cpp + source/PluginTests.cpp + source/SettingsParser.cpp + source/SettingsSerializer.cpp + source/tests/BasicTests.cpp + source/tests/LocaleTest.cpp + source/tests/BusTests.cpp + source/tests/EditorTests.cpp + source/tests/ExtremeTests.cpp + source/tests/ParameterFuzzTests.cpp + source/TestUtilities.cpp + source/Validator.cpp) target_sources(pluginval PRIVATE ${SourceFiles}) -source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles}) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/source PREFIX Source FILES ${SourceFiles}) # Add VST3 validator runner sources to pluginval (extracts embedded binary at runtime) if(PLUGINVAL_VST3_VALIDATOR) set(VST3ValidatorFiles - Source/vst3validator/VST3ValidatorRunner.h - Source/vst3validator/VST3ValidatorRunner.cpp + source/vst3validator/VST3ValidatorRunner.h + source/vst3validator/VST3ValidatorRunner.cpp ) target_sources(pluginval PRIVATE ${VST3ValidatorFiles}) - source_group("Source/vst3validator" FILES ${VST3ValidatorFiles}) + source_group("source/vst3validator" FILES ${VST3ValidatorFiles}) endif() if (DEFINED ENV{VST2_SDK_DIR}) @@ -256,3 +268,12 @@ else() DEPENDS pluginval ${PLUGINVAL_TARGET} COMMENT "Run pluginval CLI with strict validation") endif() + +# In-repo dogfood plugins + acceptance self-tests. +if (PLUGINVAL_BUILD_TEST_PLUGINS) + enable_testing() + add_subdirectory(tests/test_plugins/tone_generator) + add_subdirectory(tests/test_plugins/gain) + add_subdirectory(tests/test_plugins/playhead_probe) + add_subdirectory(tests/acceptance) +endif() diff --git a/README.md b/README.md index 4ad3a92a..144a6bc9 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ This means you can check the exit code on your various CI and mark builds a fail - [Testing plugins with pluginval]() - [Debugging a failed validation]() - [Adding pluginval to CI]() + - [Acceptance testing]() ### Contributing If you would like to contribute to the project please do! It's very simple to add tests, simply: diff --git a/ROADMAP.md b/ROADMAP.md index e7d181b5..5a63ca77 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,7 +5,7 @@ There are lots of ways pluginval can be improved, some of these are listed below You can help get there by issuing PRs or [funding](FUNDING.md) development. - [x] Integration of real-time safety checking (via rtcheck) -- [ ] Improved command-line handling and json config file support +- [x] Improved command-line handling and json config file support - [ ] Acceptance testing using input files and reference output files - [ ] Improved stack trace/crash reporting - [ ] Automatic integration of Asan/Tsan on platforms that allow it diff --git a/SUBCOMMANDS_HANDOFF.md b/SUBCOMMANDS_HANDOFF.md deleted file mode 100644 index 3f6ef277..00000000 --- a/SUBCOMMANDS_HANDOFF.md +++ /dev/null @@ -1,89 +0,0 @@ -# HANDOFF: Restructure pluginval's CLI into subcommands - -## Goal -Turn the current "mode" flags into proper subcommands, each with its own -argument set, while keeping the old flat flags working as **deprecated aliases -for one release**: - -| New (target) | Old (keep as deprecated alias) | -|---|---| -| `pluginval validate [options] ` (the **default**) | `pluginval --validate ` / `pluginval ` | -| `pluginval run-tests` | `pluginval --run-tests` | -| `pluginval strictness-help [level]` | `pluginval --strictness-help [level]` | -| `pluginval --version` / `pluginval --help` | (unchanged) | - -All settings options (`--strictness-level`, `--config`, `--rtcheck`, …), -environment variables, and the JSON precedence pipeline belong to the `validate` -subcommand and are otherwise unchanged. - -This was deliberately deferred from the CLI11 refactor PR (#174). The pipeline -and `PluginvalSettings` were designed to be reused unchanged. - -## Read these first (don't trust this summary — verify against the files) -- `Source/SettingsParser.cpp/.h` — the parser. Key pieces to reuse: - - `preprocess()` — deprecation rewrite, macOS flag strip, implicit `--validate`, tokenise. - - `parseTokens(tokens, env)` — env → `--config` → CLI layering into one `PluginvalSettings`. **This is the `validate` body.** - - `configureApp(app, s)` — registers every option on a `CLI::App`, used for both the env and CLI passes. - - `createChildProcessCommandLine()` — the parent→child base64 handoff (`--config-base64 --validate `). - - `isCommandLine(tokens)` — what makes `shouldPerformCommandLine` return true. -- `Source/CommandLine.cpp` — `performCommandLine()` is the dispatcher today: - token-scans for `--run-tests` / `--strictness-help`, otherwise runs `parseTokens` for validate; `--help`/`--version` go through CLI11. Also `runUnitTests()`, `printStrictnessHelp()`. -- `Source/Main.cpp` — calls `shouldPerformCommandLine()` then `performCommandLine()` with `getCommandLineParameters()` (a single `juce::String`). -- `Source/CommandLineTests.cpp` — the test contract. - -## Recommended approach: a thin `argv[1]` verb dispatcher (not CLI11 subcommands) -The existing `parseTokens` pipeline (env/config/CLI layering via two `CLI::App` -passes) is the hard part and already works. Rather than re-express it inside -CLI11's native `add_subcommand` machinery (which complicates the env/config -layering because the options live on the subcommand), peel the verb off the -front and route: - -1. In a new `dispatch()` step (in `SettingsParser` or `CommandLine.cpp`): - - Tokenise the command line (reuse `preprocess` minus the implicit-validate step, or add a pre-step). - - Look at the first non-option token: - - `validate` → strip it, run the existing validate pipeline on the rest. - - `run-tests` → `runUnitTests()`. - - `strictness-help` → `printStrictnessHelp(level)`. - - otherwise → **default to validate** (this preserves `pluginval ` and `pluginval --strictness-level 5 `). -2. Each verb keeps its own small set of expected args. `validate` reuses - `configureApp`/`parseTokens` verbatim. `run-tests` takes none. - `strictness-help` takes an optional level. - -This keeps `parseTokens` and the precedence layering untouched — the subcommand -work is purely a routing layer in front of it. - -(If you prefer CLI11-native subcommands instead: put `configureApp` options on a -`validate` subcommand, and make `buildEnvArgv`/the env pass target that -subcommand's options. Doable, but more invasive for no functional gain here.) - -## Deprecated-alias behaviour (one release) -Keep the old flat forms working, but print a one-line notice to stderr pointing -at the new syntax, e.g.: -- `pluginval --validate x` → run validate; warn `"--validate is deprecated; use 'pluginval validate x'"`. -- `pluginval --run-tests` → warn `"use 'pluginval run-tests'"`. -- `pluginval --strictness-help` → warn `"use 'pluginval strictness-help'"`. -- `pluginval ` (bare path) → **no warning** (still the documented shorthand for `validate`). - -Gate the warnings behind detection of the old flag so the new subcommand form is -silent. Remove the aliases in the release after next; note it in `CHANGELIST.md`. - -## Files to touch -- `Source/SettingsParser.{h,cpp}` — add the verb dispatch + a `Command` result (validate/run-tests/strictness-help/help/version), or expose a `dispatch()` that returns which verb + the remaining tokens. Keep `parseTokens` as the validate body. -- `Source/CommandLine.cpp` — `performCommandLine()` routes on the verb; emit the deprecation notices for old flat flags. `shouldPerformCommandLine()` must also recognise the bare verbs (`validate`/`run-tests`/`strictness-help`) in addition to the old flags. -- `Source/CommandLineTests.cpp` — add tests: each subcommand; default-to-validate; bare-path shorthand; every deprecated alias still works (and warns); `run-tests`/`strictness-help` arg handling. -- `docs/Command line options.md` — regenerate (`pluginval --help`); CLI11 can show per-subcommand help if you go native, otherwise hand-format the verb list. -- `CHANGELIST.md` — note the subcommand syntax + the deprecation. -- `CLAUDE.md` — update the "CLI Settings Pipeline" section to mention the verb layer. - -## Gotchas / decisions to make -- **Child-process handoff.** `createChildProcessCommandLine()` emits `--config-base64 --validate `. Decide whether the child invocation becomes `validate --config-base64 …` or stays flat. Simplest: keep it flat and have the dispatcher treat a leading `--config-base64`/`--validate` as the (deprecated, unwarned-for-internal) validate path. Note `--config-base64` is now hardened to reject being combined with non-`--validate` options — keep that working under whichever form you choose. -- **`shouldPerformCommandLine`** is what flips pluginval into CLI (vs GUI) mode in `Main.cpp`. It must return true for `pluginval run-tests` etc., not just the old flags. -- **`--help` scope.** With the dispatcher, `pluginval --help` is the top-level help (list verbs + the validate options). Consider `pluginval validate --help` for the full option list. CLI11-native subcommands give this for free. -- **Implicit validate** currently lives in `preprocess`. With an explicit `validate` verb, make sure `pluginval ` (no verb) still resolves to validate, and `pluginval validate ` doesn't double-insert `--validate`. -- **Reconcile** a positional plugin path under `validate` (e.g. `pluginval validate `) with the existing `--validate ` option — pick one canonical form (recommend the positional for the new syntax, mapping it onto `s.validatePath`). - -## Verify -- `pluginval run-tests` passes the full unit suite (it must, it's how CI runs tests). -- `pluginval validate --strictness-level 10 ` and `pluginval ` both validate. -- Every deprecated alias produces identical behaviour to before (plus a notice). -- CI matrix green (Linux/macOS/Windows build + dependency). Remember `.github/workflows/build.yaml` uses `--run-tests` and `--strictness-level 10 --validate …`; update those to the new syntax **and** keep an alias test, or the deprecation will fire in CI. diff --git a/docs/Acceptance testing.md b/docs/Acceptance testing.md new file mode 100644 index 00000000..d3037cad --- /dev/null +++ b/docs/Acceptance testing.md @@ -0,0 +1,76 @@ +# Acceptance testing + +Acceptance testing answers a different question from normal validation. Instead of +"does this plugin conform to the host API and behave safely?" it asks **"does this +plugin still produce the output I expect for a known input and state?"** — a +deterministic *render + golden-file comparison*. It is ideal for catching +accidental DSP changes (a wrong coefficient, a refactor that shifts the output) in +your own plugins from CI. + +It is driven by a small JSON config and a dedicated command: + +``` +pluginval test myPlugin-default.json +``` + +- The **first** run renders the plugin and, finding no reference, **records** one + (a `.wav` plus a `.wav.json` manifest) next to the config and reports success. +- **Subsequent** runs render again and **compare** against that reference, + exiting `0` on a match and `1` on a mismatch (writing a diff `.wav` to help you + see what changed). + +So the workflow is: write a config, run once to record the reference, **commit the +config and the reference**, then let CI run the same command on every change. + +### A minimal config + +```json +{ + "name": "myReverb-default", + "plugin": "/path/to/MyReverb.vst3", + "input": { "audio": "inputs/drums.wav" }, + "reference": "refs/myReverb-default.wav", + "state": { "parameters": { "Mix": 0.5, "Decay": 0.8 } }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 2.0, + "comparison": { "sample": 1e-6 } +} +``` + +JSON keys are `snake_case`. The most useful fields: + +| Field | Notes | +|---|---| +| `plugin` | Path to the plugin (or an AU identifier). **Required.** | +| `input.audio` / `input.midi` | Input files to feed it. Omit both for silence (e.g. instruments driven only by MIDI, or generators). | +| `state.parameters` | A map of parameter name (or index) → **normalised** value (`0`–`1`), applied before rendering. | +| `state.file` | A binary `getStateInformation` blob to restore first (e.g. a captured preset). Applied before `state.parameters`. | +| `reference` | The golden `.wav`. Defaults to `.wav` next to the config. | +| `render_duration` | Seconds to render. If omitted, the input audio's length is used. | +| `playhead` | A fixed transport for tempo-dependent plugins: `{ "bpm": 120, "time_signature": { "numerator": 4, "denominator": 4 } }`. Omit it and the plugin gets no playhead. The position advances with the render. | +| `comparison` | How to compare. `{ "sample": }` is a per-sample absolute-difference tolerance (`0` = bit-exact); the default is one 16-bit LSB. | + +### Determinism matters + +Acceptance testing only works for output that is reproducible from a fixed input +and state. A plugin with free-running randomness can't be golden-tested reliably. +For the same reason, references are only safely **portable across platforms** when +you allow a tolerance — exact per-sample matches rarely survive different CPUs and +floating-point libraries. If a reference recorded on one OS fails on another, raise +the `sample` tolerance (or use a more tolerant comparison method as they are added). + +### Running from CI + +`pluginval test` is just a command that returns an exit code, so any CI system can +run it the same way it runs your other checks: + +```bash +pluginval test tests/acceptance/myReverb-default.json +``` + +A non-zero exit fails the build. Commit the config and its reference `.wav` +alongside your project so every run compares against the same golden file. + +For the complete schema, the comparator design and the planned roadmap, see the +[design document](<../tests/acceptance/Acceptance testing design.md>). diff --git a/docs/Command line options.md b/docs/Command line options.md index d87244b8..6c4c66a5 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -11,6 +11,9 @@ COMMANDS: "pluginval [options] " also work. run-tests Run the internal unit tests. strictness-help [level] List all tests that run at the given strictness level. + test Run a deterministic acceptance (golden-file) test + from a config. Records a reference on first run, + compares against it afterwards (exit 0/1). The flat flags --validate , --run-tests and --strictness-help [level] are deprecated aliases for the commands above and will be removed in a future diff --git a/Source/CommandLine.cpp b/source/CommandLine.cpp similarity index 92% rename from Source/CommandLine.cpp rename to source/CommandLine.cpp index 282c9b50..7b46af1c 100644 --- a/Source/CommandLine.cpp +++ b/source/CommandLine.cpp @@ -18,6 +18,7 @@ #include "PluginTests.h" #include "PluginvalSettings.h" #include "SettingsParser.h" +#include "acceptance/AcceptanceTest.h" #include #include @@ -236,6 +237,22 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co return; } + if (routed.command == settings_parser::Command::test) + { + if (routed.testConfigPath.isEmpty()) + { + exitWithError ("*** FAILED: No acceptance-test config specified (usage: pluginval test )"); + return; + } + + // The acceptance runner needs the message thread for the VST3-safe + // lifecycle helpers, and we are already on it here, so run synchronously. + const auto configFile = juce::File::getCurrentWorkingDirectory().getChildFile (routed.testConfigPath); + app.setApplicationReturnValue (acceptance::runTestFile (configFile)); + app.quit(); + return; + } + // Otherwise this is a validation run (positional plugin, or explicit/implicit // --validate). CLI11 handles --help/--version and parse errors. if (routed.deprecatedAlias) diff --git a/Source/CommandLine.h b/source/CommandLine.h similarity index 100% rename from Source/CommandLine.h rename to source/CommandLine.h diff --git a/Source/CommandLineTests.cpp b/source/CommandLineTests.cpp similarity index 78% rename from Source/CommandLineTests.cpp rename to source/CommandLineTests.cpp index b19a8106..e90772ad 100644 --- a/Source/CommandLineTests.cpp +++ b/source/CommandLineTests.cpp @@ -436,6 +436,124 @@ struct CommandLineTests : public juce::UnitTest expect (shouldPerformCommandLine ("strictness-help")); expect (shouldPerformCommandLine ("validate MyPlugin.vst3")); expect (shouldPerformCommandLine ("validate MyPluginID")); + expect (shouldPerformCommandLine ("test config.json")); + } + + beginTest ("Subcommand: test captures the positional config path"); + { + using settings_parser::Command; + + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json")); + expect (d.command == Command::test); + expect (! d.deprecatedAlias); + expectEquals (d.testConfigPath, juce::String ("config.json")); + } + { + // The config path is the first non-option token after the verb. + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test ./refs/sine.json")); + expect (d.command == Command::test); + expectEquals (d.testConfigPath, juce::String ("./refs/sine.json")); + } + { + // Missing positional -> empty path (the runner reports the usage error). + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test")); + expect (d.command == Command::test); + expect (d.testConfigPath.isEmpty()); + } + } + + beginTest ("Acceptance TestConfig JSON parsing (snake_case keys, defaults)"); + { + const auto json = R"({ + "name": "myReverb-default", + "plugin": "/path/to/Plugin.vst3", + "input": { "audio": "in.wav" }, + "state": { "parameters": { "Mix": 0.5, "3": 1.0 } }, + "sample_rate": 48000, + "block_size": 256, + "render_duration": 2.0 + })"; + + const auto config = nlohmann::json::parse (json).get(); + + expectEquals (config.getName(), juce::String ("myReverb-default")); + expectEquals (juce::String (config.plugin), juce::String ("/path/to/Plugin.vst3")); + expectEquals (juce::String (config.inputAudio), juce::String ("in.wav")); + expect (config.inputMidi.empty()); + expectEquals (config.sampleRate, 48000.0); + expectEquals (config.blockSize, 256); + expect (config.renderDuration.has_value()); + expectEquals (*config.renderDuration, 2.0); + expectEquals ((int) config.stateParameters.size(), 2); + expectEquals (config.stateParameters.at ("Mix"), 0.5); + expectEquals (config.stateParameters.at ("3"), 1.0); + + // Omitted comparison falls back to one 16-bit LSB. + const auto comparison = config.getComparison(); + expect (comparison.contains ("sample")); + expectEquals (comparison["sample"].get(), 1.0 / 32768.0); + } + + beginTest ("Acceptance TestConfig omitted render_duration and explicit comparison"); + { + const auto json = R"({ + "plugin": "Plugin.vst3", + "comparison": { "sample": 0.0 } + })"; + + const auto config = nlohmann::json::parse (json).get(); + expect (! config.renderDuration.has_value()); + expectEquals (config.getComparison()["sample"].get(), 0.0); + } + + beginTest ("Acceptance TestConfig playhead (object time signature)"); + { + // Absent playhead -> unset (no transport supplied to the plugin). + { + const auto config = nlohmann::json::parse (R"({ "plugin": "P.vst3" })").get(); + expect (! config.playhead.has_value()); + } + + // Present playhead -> parsed, with time_signature as an object. + { + const auto json = R"({ + "plugin": "P.vst3", + "playhead": { + "bpm": 90, + "time_signature": { "numerator": 6, "denominator": 8 }, + "start_ppq": 4.0 + } + })"; + + const auto config = nlohmann::json::parse (json).get(); + expect (config.playhead.has_value()); + expectEquals (config.playhead->bpm, 90.0); + expectEquals (config.playhead->timeSigNumerator, 6); + expectEquals (config.playhead->timeSigDenominator, 8); + expectEquals (config.playhead->startPpq, 4.0); + } + + // Time signature defaults to 4/4 when omitted. + { + const auto config = nlohmann::json::parse (R"({ "plugin": "P.vst3", "playhead": { "bpm": 100 } })") + .get(); + expect (config.playhead.has_value()); + expectEquals (config.playhead->timeSigNumerator, 4); + expectEquals (config.playhead->timeSigDenominator, 4); + } + + // An invalid (zero) denominator is rejected. + { + bool threw = false; + try + { + nlohmann::json::parse (R"({ "plugin": "P.vst3", "playhead": { "bpm": 120, "time_signature": { "numerator": 4, "denominator": 0 } } })") + .get(); + } + catch (const std::exception&) { threw = true; } + expect (threw, "expected a zero denominator to be rejected"); + } } } }; diff --git a/Source/CrashHandler.cpp b/source/CrashHandler.cpp similarity index 100% rename from Source/CrashHandler.cpp rename to source/CrashHandler.cpp diff --git a/Source/CrashHandler.h b/source/CrashHandler.h similarity index 100% rename from Source/CrashHandler.h rename to source/CrashHandler.h diff --git a/Source/Main.cpp b/source/Main.cpp similarity index 100% rename from Source/Main.cpp rename to source/Main.cpp diff --git a/Source/MainComponent.cpp b/source/MainComponent.cpp similarity index 100% rename from Source/MainComponent.cpp rename to source/MainComponent.cpp diff --git a/Source/MainComponent.h b/source/MainComponent.h similarity index 100% rename from Source/MainComponent.h rename to source/MainComponent.h diff --git a/Source/PluginTests.cpp b/source/PluginTests.cpp similarity index 100% rename from Source/PluginTests.cpp rename to source/PluginTests.cpp diff --git a/Source/PluginTests.h b/source/PluginTests.h similarity index 100% rename from Source/PluginTests.h rename to source/PluginTests.h diff --git a/Source/PluginvalLookAndFeel.h b/source/PluginvalLookAndFeel.h similarity index 100% rename from Source/PluginvalLookAndFeel.h rename to source/PluginvalLookAndFeel.h diff --git a/Source/PluginvalSettings.h b/source/PluginvalSettings.h similarity index 100% rename from Source/PluginvalSettings.h rename to source/PluginvalSettings.h diff --git a/Source/RTCheck.h b/source/RTCheck.h similarity index 100% rename from Source/RTCheck.h rename to source/RTCheck.h diff --git a/Source/SettingsParser.cpp b/source/SettingsParser.cpp similarity index 96% rename from Source/SettingsParser.cpp rename to source/SettingsParser.cpp index e93cb76e..61daea8d 100644 --- a/Source/SettingsParser.cpp +++ b/source/SettingsParser.cpp @@ -130,6 +130,7 @@ R"(Commands: validate [options] Validate the plugin at the given path or AU id (the default). run-tests Run the internal unit tests. strictness-help [level] List all tests that run at the given strictness level. + test Run a deterministic acceptance (golden-file) test from a config. The flat flags --validate , --run-tests and --strictness-help [level] are deprecated aliases for the commands above and will be removed in a future version. @@ -313,7 +314,7 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma { const auto& verb = tokens.getReference (0); - if (verb == "validate" || verb == "run-tests" || verb == "strictness-help") + if (verb == "validate" || verb == "run-tests" || verb == "strictness-help" || verb == "test") return true; } @@ -365,6 +366,23 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma result.strictnessLevel = levelAfter (tokensIn, 0); return result; } + + if (verb == "test") + { + result.command = Command::test; + + // The positional acceptance-test config (first non-option token after the verb). + for (int i = 1; i < tokensIn.size(); ++i) + { + if (! tokensIn.getReference (i).startsWith ("-")) + { + result.testConfigPath = tokensIn.getReference (i); + break; + } + } + + return result; + } } // 2. Deprecated flat command flags. diff --git a/Source/SettingsParser.h b/source/SettingsParser.h similarity index 95% rename from Source/SettingsParser.h rename to source/SettingsParser.h index 15089faa..1898fb4b 100644 --- a/Source/SettingsParser.h +++ b/source/SettingsParser.h @@ -69,7 +69,8 @@ namespace settings_parser { validate, /**< Validate a plugin (the default when no verb is given). */ runTests, /**< Run the internal unit tests. */ - strictnessHelp /**< List the tests that run at a given strictness level. */ + strictnessHelp, /**< List the tests that run at a given strictness level. */ + test /**< Run a deterministic acceptance (golden-file) test from a config. */ }; /** The outcome of routing tokens to a subcommand. */ @@ -79,6 +80,7 @@ namespace settings_parser juce::StringArray validateTokens; /**< Verb-stripped option tokens to feed parseTokens (validate only). */ bool deprecatedAlias = false; /**< A legacy flat flag (--validate/--run-tests/--strictness-help) was used. */ int strictnessLevel = 5; /**< The level for the strictnessHelp command. */ + juce::String testConfigPath; /**< The positional config.json for the test command. */ }; /** Routes tokenised input to a subcommand, peeling the leading verb. The diff --git a/Source/SettingsSerializer.cpp b/source/SettingsSerializer.cpp similarity index 100% rename from Source/SettingsSerializer.cpp rename to source/SettingsSerializer.cpp diff --git a/Source/SettingsSerializer.h b/source/SettingsSerializer.h similarity index 100% rename from Source/SettingsSerializer.h rename to source/SettingsSerializer.h diff --git a/Source/StrictnessInfoPopup.h b/source/StrictnessInfoPopup.h similarity index 100% rename from Source/StrictnessInfoPopup.h rename to source/StrictnessInfoPopup.h diff --git a/Source/TestUtilities.cpp b/source/TestUtilities.cpp similarity index 100% rename from Source/TestUtilities.cpp rename to source/TestUtilities.cpp diff --git a/Source/TestUtilities.h b/source/TestUtilities.h similarity index 100% rename from Source/TestUtilities.h rename to source/TestUtilities.h diff --git a/Source/Validator.cpp b/source/Validator.cpp similarity index 100% rename from Source/Validator.cpp rename to source/Validator.cpp diff --git a/Source/Validator.h b/source/Validator.h similarity index 100% rename from Source/Validator.h rename to source/Validator.h diff --git a/source/acceptance/AcceptanceTest.cpp b/source/acceptance/AcceptanceTest.cpp new file mode 100644 index 00000000..c05ed61c --- /dev/null +++ b/source/acceptance/AcceptanceTest.cpp @@ -0,0 +1,500 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "AcceptanceTest.h" +#include "../TestUtilities.h" + +#include + +#include +#include +#include + +namespace acceptance +{ + +//============================================================================== +namespace +{ + juce::String archString() + { + #if defined (__aarch64__) || defined (_M_ARM64) + return "arm64"; + #elif defined (__x86_64__) || defined (_M_X64) + return "x86_64"; + #else + return "unknown"; + #endif + } + + //============================================================================== + /** A fixed-tempo transport whose position advances with the render. Constructed + from the config's playhead block and pointed at the plugin for the duration + of the render; setSamplePosition() is called before each processBlock. */ + class FixedPlayHead : public juce::AudioPlayHead + { + public: + FixedPlayHead (const TestConfig::PlayheadConfig& config, double sr) + : cfg (config), sampleRate (sr) + { + } + + void setSamplePosition (juce::int64 sample) noexcept { currentSample = sample; } + + juce::Optional getPosition() const override + { + const double seconds = (double) currentSample / sampleRate; + + // PPQ is measured in quarter notes; a bar is numerator * (4/denominator) of them. + const double quarterNotesPerBar = cfg.timeSigNumerator * 4.0 / cfg.timeSigDenominator; + const double ppq = cfg.startPpq + seconds * (cfg.bpm / 60.0); + + PositionInfo info; + info.setBpm (cfg.bpm); + info.setTimeSignature (TimeSignature { cfg.timeSigNumerator, cfg.timeSigDenominator }); + info.setTimeInSamples (currentSample); + info.setTimeInSeconds (seconds); + info.setPpqPosition (ppq); + info.setPpqPositionOfLastBarStart (std::floor (ppq / quarterNotesPerBar) * quarterNotesPerBar); + info.setIsPlaying (true); + return info; + } + + private: + const TestConfig::PlayheadConfig cfg; + const double sampleRate; + juce::int64 currentSample = 0; + }; + + //============================================================================== + std::unique_ptr loadPlugin (juce::AudioPluginFormatManager& formatManager, + const juce::String& pathOrID, + double sampleRate, int blockSize) + { + juce::KnownPluginList list; + juce::OwnedArray found; + list.scanAndAddDragAndDroppedFiles (formatManager, juce::StringArray (pathOrID), found); + + if (found.isEmpty()) + throw std::runtime_error (("no plugin found at: " + pathOrID + + " (missing/damaged binary, an incompatible format, or an AU not registered with macOS)").toStdString()); + + juce::String error; + auto instance = std::unique_ptr ( + formatManager.createPluginInstance (*found.getFirst(), sampleRate, blockSize, error)); + + if (instance == nullptr) + throw std::runtime_error (("failed to create plugin instance: " + error).toStdString()); + + return instance; + } + + //============================================================================== + void applyState (juce::AudioPluginInstance& instance, const TestConfig& config) + { + // Binary state first (the full plugin state), then the parameter map as + // overrides (see the precedence rule in the spec). + if (const auto stateFile = config.getStateFile(); stateFile != juce::File()) + { + if (! stateFile.existsAsFile()) + throw std::runtime_error (("state.file not found: " + stateFile.getFullPathName()).toStdString()); + + juce::MemoryBlock state; + stateFile.loadFileAsData (state); + callSetStateInformationOnMessageThreadIfVST3 (instance, state); + } + + for (const auto& [key, value] : config.stateParameters) + { + const juce::String name (key); + juce::AudioProcessorParameter* match = nullptr; + + if (name.containsOnly ("0123456789") && name.isNotEmpty()) + { + match = instance.getParameters()[name.getIntValue()]; + } + else + { + // Match the display name (case-insensitively), or the JUCE paramID + // when the host exposes parameters as AudioProcessorParameterWithID. + for (auto* p : instance.getParameters()) + { + if (auto* withID = dynamic_cast (p)) + if (withID->paramID.equalsIgnoreCase (name)) + { + match = p; + break; + } + + if (p->getName (512).equalsIgnoreCase (name)) + { + match = p; + break; + } + } + } + + if (match == nullptr) + throw std::runtime_error (("state.parameters: no parameter named or indexed \"" + name + "\"").toStdString()); + + match->setValueNotifyingHost ((float) value); + } + } + + //============================================================================== + juce::AudioBuffer readWav (juce::AudioFormatManager& formatManager, const juce::File& file) + { + std::unique_ptr reader (formatManager.createReaderFor (file)); + + if (reader == nullptr) + throw std::runtime_error (("could not read audio file: " + file.getFullPathName()).toStdString()); + + juce::AudioBuffer buffer ((int) reader->numChannels, (int) reader->lengthInSamples); + reader->read (&buffer, 0, (int) reader->lengthInSamples, 0, true, true); + return buffer; + } + + void writeFloatWav (const juce::File& file, const juce::AudioBuffer& buffer, double sampleRate) + { + file.getParentDirectory().createDirectory(); + file.deleteFile(); + + std::unique_ptr stream (file.createOutputStream()); + + if (stream == nullptr) + throw std::runtime_error (("could not open for writing: " + file.getFullPathName()).toStdString()); + + juce::WavAudioFormat wav; + const auto options = juce::AudioFormatWriterOptions() + .withSampleRate (sampleRate) + .withNumChannels (buffer.getNumChannels()) + .withBitsPerSample (32) + .withSampleFormat (juce::AudioFormatWriterOptions::SampleFormat::floatingPoint); + + std::unique_ptr writer (wav.createWriterFor (stream, options)); + + if (writer == nullptr) + throw std::runtime_error (("could not create WAV writer for: " + file.getFullPathName()).toStdString()); + + writer->writeFromAudioSampleBuffer (buffer, 0, buffer.getNumSamples()); + } + + //============================================================================== + /** Builds a single MidiBuffer with events at absolute sample positions. */ + juce::MidiBuffer loadMidi (const juce::File& file, double sampleRate) + { + juce::FileInputStream stream (file); + + if (! stream.openedOk()) + throw std::runtime_error (("could not read MIDI file: " + file.getFullPathName()).toStdString()); + + juce::MidiFile midiFile; + + if (! midiFile.readFrom (stream)) + throw std::runtime_error (("could not parse MIDI file: " + file.getFullPathName()).toStdString()); + + midiFile.convertTimestampTicksToSeconds(); + + juce::MidiBuffer buffer; + + for (int t = 0; t < midiFile.getNumTracks(); ++t) + if (const auto* track = midiFile.getTrack (t)) + for (const auto* event : *track) + { + const auto sample = (int) std::lround (event->message.getTimeStamp() * sampleRate); + buffer.addEvent (event->message, sample); + } + + return buffer; + } + + //============================================================================== + juce::String configHash (const TestConfig& config) + { + auto j = nlohmann::json {}; + to_json (j, config); + j.erase ("reference"); // the hash must be independent of where the reference lives + + // A small, stable (cross-platform) FNV-1a 64-bit hash. The hash is only + // used for an informational "stale reference" warning, so juce_cryptography + // (MD5) isn't worth pulling in. + const auto dump = j.dump(); + std::uint64_t hash = 1469598103934665603ull; + + for (const auto c : dump) + { + hash ^= (std::uint64_t) (unsigned char) c; + hash *= 1099511628211ull; + } + + return juce::String::toHexString ((juce::int64) hash); + } +} + +//============================================================================== +RenderedAudio renderPlugin (const TestConfig& config) +{ + const auto sampleRate = config.sampleRate; + const auto blockSize = juce::jmax (1, config.blockSize); + + juce::AudioPluginFormatManager formatManager; + #if JUCE_VERSION >= 0x08000B + juce::addDefaultFormatsToManager (formatManager); + #else + formatManager.addDefaultFormats(); + #endif + + auto instance = loadPlugin (formatManager, config.getPluginPathOrID(), sampleRate, blockSize); + const auto description = instance->getPluginDescription(); + + // 1. State (file then parameter overrides), applied before prepareToPlay. + applyState (*instance, config); + + // 2. Input audio / MIDI (or silence). + juce::AudioFormatManager audioFormats; + audioFormats.registerBasicFormats(); + + juce::AudioBuffer inputAudio; + if (const auto audioFile = config.getInputAudioFile(); audioFile != juce::File()) + { + if (! audioFile.existsAsFile()) + throw std::runtime_error (("input.audio not found: " + audioFile.getFullPathName()).toStdString()); + + inputAudio = readWav (audioFormats, audioFile); + } + + juce::MidiBuffer midi; + if (const auto midiFile = config.getInputMidiFile(); midiFile != juce::File()) + { + if (! midiFile.existsAsFile()) + throw std::runtime_error (("input.midi not found: " + midiFile.getFullPathName()).toStdString()); + + midi = loadMidi (midiFile, sampleRate); + } + + // 3. Render length. + int numSamples = 0; + if (config.renderDuration) + numSamples = (int) std::lround (*config.renderDuration * sampleRate); + else if (inputAudio.getNumSamples() > 0) + numSamples = inputAudio.getNumSamples(); + + if (numSamples <= 0) + throw std::runtime_error ("render_duration is required when there is no input.audio"); + + // 4. Optional fixed transport for time-dependent plugins. + std::unique_ptr playHead; + if (config.playhead) + { + playHead = std::make_unique (*config.playhead, sampleRate); + instance->setPlayHead (playHead.get()); + } + + // 5. Prepare and render block by block. + callPrepareToPlayOnMessageThreadIfVST3 (*instance, sampleRate, blockSize); + + const int numInputChannels = instance->getTotalNumInputChannels(); + const int numOutputChannels = juce::jmax (1, instance->getTotalNumOutputChannels()); + const int channelsRequired = juce::jmax (numInputChannels, numOutputChannels); + + juce::AudioBuffer output (numOutputChannels, numSamples); + output.clear(); + + juce::AudioBuffer block (channelsRequired, blockSize); + + for (int pos = 0; pos < numSamples; pos += blockSize) + { + const int thisBlock = juce::jmin (blockSize, numSamples - pos); + + if (playHead != nullptr) + playHead->setSamplePosition (pos); + + block.clear(); + + // Copy the input audio slice into the block (silence past its end). + for (int c = 0; c < juce::jmin (numInputChannels, inputAudio.getNumChannels()); ++c) + { + const int available = juce::jmax (0, juce::jmin (thisBlock, inputAudio.getNumSamples() - pos)); + if (available > 0) + block.copyFrom (c, 0, inputAudio, c, pos, available); + } + + juce::MidiBuffer blockMidi; + blockMidi.addEvents (midi, pos, thisBlock, -pos); + + juce::AudioBuffer proc (block.getArrayOfWritePointers(), channelsRequired, thisBlock); + instance->processBlock (proc, blockMidi); + + for (int c = 0; c < numOutputChannels; ++c) + output.copyFrom (c, pos, proc, c, 0, thisBlock); + } + + instance->setPlayHead (nullptr); // playHead is about to be destroyed + callReleaseResourcesOnMessageThreadIfVST3 (*instance); + instance.reset(); + + return { std::move (output), sampleRate, blockSize, description }; +} + +//============================================================================== +TestResult runTest (const TestConfig& config) +{ + const auto name = config.getName(); + + try + { + const auto rendered = renderPlugin (config); + const auto referenceFile = config.getReferenceFile(); + + TestResult result; + result.name = name; + result.referenceFile = referenceFile; + + // Record mode: no reference yet -> write the float WAV + JSON sidecar. + if (! referenceFile.existsAsFile()) + { + writeFloatWav (referenceFile, rendered.buffer, rendered.sampleRate); + + nlohmann::json manifest; + manifest["plugin"] = { + { "name", rendered.description.name.toStdString() }, + { "manufacturer", rendered.description.manufacturerName.toStdString() }, + { "version", rendered.description.version.toStdString() }, + { "format", rendered.description.pluginFormatName.toStdString() }, + { "uid", rendered.description.createIdentifierString().toStdString() } + }; + manifest["render"] = { + { "sample_rate", rendered.sampleRate }, + { "block_size", rendered.blockSize }, + { "num_channels", rendered.buffer.getNumChannels() }, + { "num_samples", rendered.buffer.getNumSamples() }, + { "length_seconds", rendered.buffer.getNumSamples() / rendered.sampleRate } + }; + manifest["pluginval_version"] = VERSION; + manifest["config_hash"] = configHash (config).toStdString(); + manifest["created_on"] = { + { "os", juce::SystemStats::getOperatingSystemName().toStdString() }, + { "arch", archString().toStdString() }, + { "date", juce::Time::getCurrentTime().toISO8601 (true).toStdString() } + }; + + config.getReferenceSidecarFile().replaceWithText (manifest.dump (2)); + + result.outcome = TestResult::Outcome::referenceCreated; + return result; + } + + // Compare mode: warn (don't fail) if the stored config hash differs. + if (const auto sidecar = config.getReferenceSidecarFile(); sidecar.existsAsFile()) + { + try + { + const auto manifest = nlohmann::json::parse (sidecar.loadFileAsString().toStdString()); + if (manifest.contains ("config_hash") + && manifest["config_hash"].get() != configHash (config).toStdString()) + { + std::cout << " WARNING: reference was recorded from a different config (stale?): " + << sidecar.getFullPathName() << std::endl; + } + } + catch (const std::exception&) { /* a malformed sidecar is non-fatal */ } + } + + juce::AudioFormatManager formats; + formats.registerBasicFormats(); + const auto reference = readWav (formats, referenceFile); + + bool allPassed = true; + + const auto comparisonConfig = config.getComparison(); + + for (auto it = comparisonConfig.begin(); it != comparisonConfig.end(); ++it) + { + const juce::String method (it.key()); + auto comparator = createComparator (method); + + if (comparator == nullptr) + return TestResult::makeError (name, "unknown comparison method: " + method); + + const auto result2 = comparator->compare (reference, rendered.buffer, it.value()); + allPassed = allPassed && result2.passed; + result.comparisons.emplace_back (method, result2); + } + + result.outcome = allPassed ? TestResult::Outcome::passed : TestResult::Outcome::failed; + + if (! allPassed) + { + // Write a diff WAV (output - reference) over the overlapping region. + const int channels = juce::jmin (reference.getNumChannels(), rendered.buffer.getNumChannels()); + const int samples = juce::jmin (reference.getNumSamples(), rendered.buffer.getNumSamples()); + + if (channels > 0 && samples > 0) + { + juce::AudioBuffer diff (channels, samples); + + for (int c = 0; c < channels; ++c) + { + diff.copyFrom (c, 0, rendered.buffer, c, 0, samples); + diff.addFrom (c, 0, reference, c, 0, samples, -1.0f); + } + + const auto diffFile = config.getDiffFile(); + writeFloatWav (diffFile, diff, rendered.sampleRate); + result.diffFile = diffFile; + } + } + + return result; + } + catch (const std::exception& e) + { + return TestResult::makeError (name, e.what()); + } +} + +//============================================================================== +int runTestFile (const juce::File& configFile) +{ + std::vector configs; + + try + { + configs = TestConfig::loadFromFile (configFile); + } + catch (const std::exception& e) + { + const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), e.what()); + reporter::report (result); + return reporter::exitCode (result); + } + + if (configs.empty()) + { + const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), "no test configs in file"); + reporter::report (result); + return reporter::exitCode (result); + } + + // v1 runs the first entry only; the parser already accepts arrays for the + // phase-2 multiplexing extension. + if (configs.size() > 1) + std::cout << "Note: " << configs.size() << " configs found; v1 runs the first only." << std::endl; + + const auto result = runTest (configs.front()); + reporter::report (result); + return reporter::exitCode (result); +} + +} // namespace acceptance diff --git a/source/acceptance/AcceptanceTest.h b/source/acceptance/AcceptanceTest.h new file mode 100644 index 00000000..fe666619 --- /dev/null +++ b/source/acceptance/AcceptanceTest.h @@ -0,0 +1,56 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "TestConfig.h" +#include "TestReporter.h" + +#include + +namespace acceptance +{ + +//============================================================================== +/** A rendered buffer plus the metadata needed for the reference manifest. */ +struct RenderedAudio +{ + juce::AudioBuffer buffer; + double sampleRate = 0.0; + int blockSize = 0; + juce::PluginDescription description; +}; + +//============================================================================== +/** Loads the plugin, applies state (file then parameters), feeds the input + (audio / MIDI / silence) and renders a fixed duration into a single buffer. + + Must be called on the message thread (it uses the VST3-safe lifecycle + helpers and creates the plugin instance directly). Throws std::runtime_error + on any setup / render failure. +*/ +RenderedAudio renderPlugin (const TestConfig&); + +//============================================================================== +/** Runs a single resolved config end-to-end: render, then record-or-compare, + producing a TestResult. Never throws - setup failures become an error result. */ +TestResult runTest (const TestConfig&); + +/** Loads the config file (first entry only for v1), runs it, reports the result + to stdout and returns the process exit code (0 success, 1 failure). + + Must be called on the message thread. */ +int runTestFile (const juce::File& configFile); + +} // namespace acceptance diff --git a/source/acceptance/ReferenceComparator.cpp b/source/acceptance/ReferenceComparator.cpp new file mode 100644 index 00000000..9853267c --- /dev/null +++ b/source/acceptance/ReferenceComparator.cpp @@ -0,0 +1,122 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "ReferenceComparator.h" + +#include + +namespace acceptance +{ + +//============================================================================== +/** + Per-sample absolute-difference comparator. + + config is the value under the "sample" key: a bare number giving the + tolerance (0 == bit-exact), or an object { "tolerance": }. Channel + and length mismatches fail outright. +*/ +struct SampleComparator : public Comparator +{ + juce::String getName() const override { return "sample"; } + + static double toleranceFrom (const nlohmann::json& config) + { + if (config.is_number()) + return config.get(); + + if (config.is_object()) + if (auto t = config.find ("tolerance"); t != config.end() && t->is_number()) + return t->get(); + + return 1.0 / 32768.0; + } + + ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::json& config) override + { + const double tolerance = toleranceFrom (config); + + ComparisonResult r; + r.details["tolerance"] = tolerance; + + if (reference.getNumChannels() != output.getNumChannels() + || reference.getNumSamples() != output.getNumSamples()) + { + r.passed = false; + r.summary = "size mismatch: reference " + + juce::String (reference.getNumChannels()) + "ch x " + juce::String (reference.getNumSamples()) + " samples, " + + "output " + juce::String (output.getNumChannels()) + "ch x " + juce::String (output.getNumSamples()) + " samples"; + r.details["reference_channels"] = reference.getNumChannels(); + r.details["reference_samples"] = reference.getNumSamples(); + r.details["output_channels"] = output.getNumChannels(); + r.details["output_samples"] = output.getNumSamples(); + return r; + } + + double maxAbsDiff = 0.0; + int firstFailChannel = -1, firstFailSample = -1; + + for (int c = 0; c < reference.getNumChannels(); ++c) + { + const auto* ref = reference.getReadPointer (c); + const auto* out = output.getReadPointer (c); + + for (int s = 0; s < reference.getNumSamples(); ++s) + { + const double diff = std::abs ((double) out[s] - (double) ref[s]); + + if (diff > maxAbsDiff) + maxAbsDiff = diff; + + if (diff > tolerance && firstFailChannel < 0) + { + firstFailChannel = c; + firstFailSample = s; + } + } + } + + r.score = maxAbsDiff; + r.passed = maxAbsDiff <= tolerance; + r.details["max_abs_diff"] = maxAbsDiff; + + if (r.passed) + { + r.summary = "max abs diff " + juce::String (maxAbsDiff) + " <= tolerance " + juce::String (tolerance); + } + else + { + r.summary = "max abs diff " + juce::String (maxAbsDiff) + " > tolerance " + juce::String (tolerance) + + " (first at channel " + juce::String (firstFailChannel) + ", sample " + juce::String (firstFailSample) + ")"; + r.details["first_fail_channel"] = firstFailChannel; + r.details["first_fail_sample"] = firstFailSample; + } + + return r; + } +}; + +//============================================================================== +std::unique_ptr createComparator (const juce::String& name) +{ + if (name == "sample") + return std::make_unique(); + + // Future: "peakrms", "spectrum", "crosscorr", "fingerprint" register here. + return {}; +} + +} // namespace acceptance diff --git a/source/acceptance/ReferenceComparator.h b/source/acceptance/ReferenceComparator.h new file mode 100644 index 00000000..680c9d45 --- /dev/null +++ b/source/acceptance/ReferenceComparator.h @@ -0,0 +1,60 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include + +#include + +namespace acceptance +{ + +//============================================================================== +/** The outcome of a single comparator run. */ +struct ComparisonResult +{ + bool passed = false; + double score = 0.0; /**< Method-specific metric, e.g. max abs diff. */ + juce::String summary; /**< Human-readable one-liner. */ + nlohmann::json details; /**< Structured detail for the machine-readable report. */ +}; + +//============================================================================== +/** + Pluggable comparison method. v1 ships only "sample"; further methods + (spectrum, crosscorr, fingerprint) register later with no changes to config + parsing or the runner. +*/ +struct Comparator +{ + virtual ~Comparator() = default; + + /** The method name, e.g. "sample". */ + virtual juce::String getName() const = 0; + + /** Compares a freshly rendered buffer against the reference. config is the + value sitting under this comparator's key in the config's "comparison" + map (a bare number for "sample", or an object for richer methods). */ + virtual ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::json& config) = 0; +}; + +//============================================================================== +/** Creates a comparator by name, or nullptr if the name is unknown. */ +std::unique_ptr createComparator (const juce::String& name); + +} // namespace acceptance diff --git a/source/acceptance/TestConfig.cpp b/source/acceptance/TestConfig.cpp new file mode 100644 index 00000000..94b69042 --- /dev/null +++ b/source/acceptance/TestConfig.cpp @@ -0,0 +1,236 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "TestConfig.h" + +#include + +namespace acceptance +{ + +//============================================================================== +namespace +{ + /** The default comparison when none is supplied: one 16-bit LSB. */ + nlohmann::json defaultComparison() + { + return nlohmann::json { { "sample", 1.0 / 32768.0 } }; + } + + /** Reads a string member from a (possibly absent) nested object. */ + std::string getNestedString (const nlohmann::json& j, const char* outer, const char* inner) + { + if (auto o = j.find (outer); o != j.end() && o->is_object()) + if (auto i = o->find (inner); i != o->end() && i->is_string()) + return i->get(); + + return {}; + } +} + +//============================================================================== +void to_json (nlohmann::json& j, const TestConfig& c) +{ + j = nlohmann::json::object(); + j["name"] = c.name; + j["plugin"] = c.plugin; + + if (! c.inputAudio.empty() || ! c.inputMidi.empty()) + { + auto input = nlohmann::json::object(); + if (! c.inputAudio.empty()) input["audio"] = c.inputAudio; + if (! c.inputMidi.empty()) input["midi"] = c.inputMidi; + j["input"] = input; + } + + if (! c.reference.empty()) + j["reference"] = c.reference; + + if (! c.stateFile.empty() || ! c.stateParameters.empty()) + { + auto state = nlohmann::json::object(); + if (! c.stateFile.empty()) state["file"] = c.stateFile; + if (! c.stateParameters.empty()) state["parameters"] = c.stateParameters; + j["state"] = state; + } + + j["sample_rate"] = c.sampleRate; + j["block_size"] = c.blockSize; + + if (c.renderDuration) + j["render_duration"] = *c.renderDuration; + + if (! c.comparison.is_null()) + j["comparison"] = c.comparison; + + if (c.playhead) + { + j["playhead"] = { + { "bpm", c.playhead->bpm }, + { "time_signature", { { "numerator", c.playhead->timeSigNumerator }, + { "denominator", c.playhead->timeSigDenominator } } }, + { "start_ppq", c.playhead->startPpq } + }; + } +} + +void from_json (const nlohmann::json& j, TestConfig& c) +{ + c.name = j.value ("name", std::string()); + c.plugin = j.value ("plugin", std::string()); + c.reference = j.value ("reference", std::string()); + + c.inputAudio = getNestedString (j, "input", "audio"); + c.inputMidi = getNestedString (j, "input", "midi"); + + c.stateFile = getNestedString (j, "state", "file"); + + if (auto state = j.find ("state"); state != j.end() && state->is_object()) + if (auto params = state->find ("parameters"); params != state->end() && params->is_object()) + c.stateParameters = params->get>(); + + c.sampleRate = j.value ("sample_rate", 44100.0); + c.blockSize = j.value ("block_size", 512); + + if (auto d = j.find ("render_duration"); d != j.end() && d->is_number()) + c.renderDuration = d->get(); + + if (auto comp = j.find ("comparison"); comp != j.end() && ! comp->is_null()) + c.comparison = *comp; + + if (auto ph = j.find ("playhead"); ph != j.end() && ph->is_object()) + { + TestConfig::PlayheadConfig p; + p.bpm = ph->value ("bpm", 120.0); + p.startPpq = ph->value ("start_ppq", 0.0); + + if (auto ts = ph->find ("time_signature"); ts != ph->end() && ts->is_object()) + { + p.timeSigNumerator = ts->value ("numerator", 4); + p.timeSigDenominator = ts->value ("denominator", 4); + } + + if (p.timeSigNumerator <= 0 || p.timeSigDenominator <= 0) + throw std::runtime_error ("playhead.time_signature numerator and denominator must be positive"); + + c.playhead = p; + } +} + +//============================================================================== +nlohmann::json TestConfig::getComparison() const +{ + return comparison.is_null() ? defaultComparison() : comparison; +} + +juce::String TestConfig::getPluginPathOrID() const +{ + const juce::String raw (plugin); + + // Resolve relative/home paths against the working directory; leave absolute + // paths and bare component IDs (no '.' or '~') untouched. Mirrors + // settings_parser::resolvePluginPath. + if (raw.contains ("~") || raw.contains (".")) + return juce::File::getCurrentWorkingDirectory().getChildFile (raw).getFullPathName(); + + return raw; +} + +juce::File TestConfig::resolveAgainstConfigDir (const std::string& path) const +{ + if (path.empty()) + return {}; + + const juce::String s (path); + + if (juce::File::isAbsolutePath (s)) + return juce::File (s); + + const auto base = configDir == juce::File() ? juce::File::getCurrentWorkingDirectory() : configDir; + return base.getChildFile (s); +} + +juce::String TestConfig::getName() const +{ + if (! name.empty()) + return juce::String (name); + + return getReferenceFile().getFileNameWithoutExtension(); +} + +juce::File TestConfig::getReferenceFile() const +{ + if (! reference.empty()) + return resolveAgainstConfigDir (reference); + + const auto base = configDir == juce::File() ? juce::File::getCurrentWorkingDirectory() : configDir; + return base.getChildFile (juce::String (name) + ".wav"); +} + +juce::File TestConfig::getReferenceSidecarFile() const +{ + const auto ref = getReferenceFile(); + return ref.getSiblingFile (ref.getFileName() + ".json"); +} + +juce::File TestConfig::getDiffFile() const +{ + const auto ref = getReferenceFile(); + return ref.getSiblingFile (ref.getFileNameWithoutExtension() + "-diff.wav"); +} + +juce::File TestConfig::getInputAudioFile() const { return resolveAgainstConfigDir (inputAudio); } +juce::File TestConfig::getInputMidiFile() const { return resolveAgainstConfigDir (inputMidi); } +juce::File TestConfig::getStateFile() const { return resolveAgainstConfigDir (stateFile); } + +//============================================================================== +std::vector TestConfig::loadFromFile (const juce::File& file) +{ + if (! file.existsAsFile()) + throw std::runtime_error (("test config not found: " + file.getFullPathName()).toStdString()); + + nlohmann::json j; + + try + { + j = nlohmann::json::parse (file.loadFileAsString().toStdString()); + } + catch (const std::exception& e) + { + throw std::runtime_error (("failed to parse test config " + file.getFullPathName() + ": " + e.what()).toStdString()); + } + + std::vector configs; + + const auto addOne = [&] (const nlohmann::json& entry) + { + auto c = entry.get(); + c.configDir = file.getParentDirectory(); + configs.push_back (std::move (c)); + }; + + if (j.is_array()) + { + for (const auto& entry : j) + addOne (entry); + } + else + { + addOne (j); + } + + return configs; +} + +} // namespace acceptance diff --git a/source/acceptance/TestConfig.h b/source/acceptance/TestConfig.h new file mode 100644 index 00000000..3db74b3f --- /dev/null +++ b/source/acceptance/TestConfig.h @@ -0,0 +1,116 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** + The definition of a single acceptance test. + + This is a plain, std-typed struct deserialised from the positional + of the "pluginval test" command. JSON keys are snake_case + (e.g. sample_rate); the C++ members stay JUCE-style camelCase, so the mapping + is done explicitly in the per-struct to_json / from_json (see TestConfig.cpp) + rather than the bare NLOHMANN macro. + + This config is completely independent of PluginvalSettings / the --config + settings layering used by the validate command. +*/ +struct TestConfig +{ + /** A fixed transport supplied to the plugin during the render, for + time-dependent plugins (tempo-synced LFOs, arpeggiators, ...). The tempo + and time signature are constant; the position advances with the render. */ + struct PlayheadConfig + { + double bpm = 120.0; + int timeSigNumerator = 4; + int timeSigDenominator = 4; + double startPpq = 0.0; /**< Transport position (in quarter notes) at sample 0. */ + }; + + std::string name; /**< Labels results; derives the default reference path. */ + std::string plugin; /**< Plugin path or AU id. Required. */ + std::string inputAudio; /**< input.audio: path to an input audio file, or empty for silence. */ + std::string inputMidi; /**< input.midi: path to a .mid file, or empty for none. */ + std::string reference; /**< The golden file. Empty -> derived from name. */ + std::string stateFile; /**< state.file: binary getStateInformation blob. */ + std::map stateParameters; /**< state.parameters: name-or-index -> normalised value. */ + double sampleRate = 44100.0; + int blockSize = 512; + std::optional renderDuration; /**< Seconds. Unset -> derive from the input length. */ + nlohmann::json comparison; /**< Map of comparator name -> sub-config. Empty -> default. */ + std::optional playhead; /**< Fixed transport. Unset -> no playhead supplied to the plugin. */ + + //============================================================================== + /** The directory the config was loaded from. Relative reference / input / + state paths resolve against this; the plugin path resolves against the + working directory (per the spec). Not part of the JSON. */ + juce::File configDir; + + //============================================================================== + /** Returns the resolved comparison map, substituting the default + ({ "sample": 1/32768 }) when none was supplied. */ + nlohmann::json getComparison() const; + + /** The plugin path or AU id, resolved against the working directory. */ + juce::String getPluginPathOrID() const; + + /** The golden reference file (explicit, or /.wav). */ + juce::File getReferenceFile() const; + + /** The reference's JSON sidecar (.json). */ + juce::File getReferenceSidecarFile() const; + + /** The diff WAV written next to the reference on a comparison failure. */ + juce::File getDiffFile() const; + + /** Resolved input audio file, or an empty File if none. */ + juce::File getInputAudioFile() const; + + /** Resolved input MIDI file, or an empty File if none. */ + juce::File getInputMidiFile() const; + + /** Resolved state file, or an empty File if none. */ + juce::File getStateFile() const; + + /** The effective test name (explicit, or the reference's basename). */ + juce::String getName() const; + + //============================================================================== + /** Loads one or more configs from a JSON file. The top level may be a single + object or an array of objects (phase 2 multiplexing); v1 callers use the + first entry. Throws std::runtime_error on a load / parse error. */ + static std::vector loadFromFile (const juce::File&); + +private: + juce::File resolveAgainstConfigDir (const std::string& path) const; +}; + +//============================================================================== +void to_json (nlohmann::json&, const TestConfig&); +void from_json (const nlohmann::json&, TestConfig&); + +} // namespace acceptance diff --git a/source/acceptance/TestReporter.cpp b/source/acceptance/TestReporter.cpp new file mode 100644 index 00000000..a95c100e --- /dev/null +++ b/source/acceptance/TestReporter.cpp @@ -0,0 +1,110 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "TestReporter.h" + +#include + +namespace acceptance::reporter +{ + +//============================================================================== +static const char* toString (TestResult::Outcome o) +{ + switch (o) + { + case TestResult::Outcome::referenceCreated: return "reference-created"; + case TestResult::Outcome::passed: return "passed"; + case TestResult::Outcome::failed: return "failed"; + case TestResult::Outcome::error: return "error"; + } + + return "error"; +} + +nlohmann::json toJson (const TestResult& r) +{ + nlohmann::json j; + j["name"] = r.name.toStdString(); + j["outcome"] = toString (r.outcome); + + if (r.message.isNotEmpty()) + j["message"] = r.message.toStdString(); + + if (r.referenceFile != juce::File()) + j["reference"] = r.referenceFile.getFullPathName().toStdString(); + + if (r.diffFile != juce::File()) + j["diff"] = r.diffFile.getFullPathName().toStdString(); + + if (! r.comparisons.empty()) + { + auto comparisons = nlohmann::json::object(); + + for (const auto& [method, result] : r.comparisons) + { + nlohmann::json c; + c["passed"] = result.passed; + c["score"] = result.score; + c["summary"] = result.summary.toStdString(); + if (! result.details.is_null()) + c["details"] = result.details; + + comparisons[method.toStdString()] = c; + } + + j["comparisons"] = comparisons; + } + + return j; +} + +void report (const TestResult& r) +{ + switch (r.outcome) + { + case TestResult::Outcome::referenceCreated: + std::cout << "Reference created: " << r.referenceFile.getFullPathName() << std::endl; + break; + + case TestResult::Outcome::passed: + std::cout << "PASSED: " << r.name << std::endl; + break; + + case TestResult::Outcome::failed: + std::cout << "*** FAILED: " << r.name << std::endl; + break; + + case TestResult::Outcome::error: + std::cout << "*** ERROR: " << r.name << ": " << r.message << std::endl; + break; + } + + for (const auto& [method, result] : r.comparisons) + std::cout << " [" << method << "] " << (result.passed ? "ok" : "FAIL") << ": " << result.summary << std::endl; + + if (r.diffFile != juce::File()) + std::cout << " diff written to: " << r.diffFile.getFullPathName() << std::endl; + + // The structured result for machine consumers. + std::cout << toJson (r).dump (2) << std::endl; +} + +int exitCode (const TestResult& r) +{ + return (r.outcome == TestResult::Outcome::referenceCreated + || r.outcome == TestResult::Outcome::passed) ? 0 : 1; +} + +} // namespace acceptance::reporter diff --git a/source/acceptance/TestReporter.h b/source/acceptance/TestReporter.h new file mode 100644 index 00000000..10266b77 --- /dev/null +++ b/source/acceptance/TestReporter.h @@ -0,0 +1,73 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "ReferenceComparator.h" + +#include +#include + +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** The result of running one acceptance test. */ +struct TestResult +{ + enum class Outcome + { + referenceCreated, /**< No reference existed, so one was recorded. Treated as success. */ + passed, /**< Reference existed and every comparator passed. */ + failed, /**< Reference existed and at least one comparator failed. */ + error /**< Setup failed (plugin load, render, IO, bad config, ...). */ + }; + + Outcome outcome = Outcome::error; + juce::String name; + juce::String message; /**< Error text, or an overall summary. */ + + std::vector> comparisons; /**< method name -> result. */ + + juce::File referenceFile; + juce::File diffFile; /**< Written only on failure. */ + + //============================================================================== + static TestResult makeError (const juce::String& name, const juce::String& message) + { + TestResult r; + r.outcome = Outcome::error; + r.name = name; + r.message = message; + return r; + } +}; + +//============================================================================== +namespace reporter +{ + /** Structured, machine-readable representation of a result. */ + nlohmann::json toJson (const TestResult&); + + /** Prints a human-readable summary followed by the JSON to stdout. */ + void report (const TestResult&); + + /** 0 for referenceCreated / passed, 1 for failed / error. */ + int exitCode (const TestResult&); +} + +} // namespace acceptance diff --git a/Source/binarydata/icon.png b/source/binarydata/icon.png similarity index 100% rename from Source/binarydata/icon.png rename to source/binarydata/icon.png diff --git a/Source/binarydata/icon.svg b/source/binarydata/icon.svg similarity index 100% rename from Source/binarydata/icon.svg rename to source/binarydata/icon.svg diff --git a/Source/tests/BasicTests.cpp b/source/tests/BasicTests.cpp similarity index 100% rename from Source/tests/BasicTests.cpp rename to source/tests/BasicTests.cpp diff --git a/Source/tests/BusTests.cpp b/source/tests/BusTests.cpp similarity index 100% rename from Source/tests/BusTests.cpp rename to source/tests/BusTests.cpp diff --git a/Source/tests/EditorTests.cpp b/source/tests/EditorTests.cpp similarity index 100% rename from Source/tests/EditorTests.cpp rename to source/tests/EditorTests.cpp diff --git a/Source/tests/ExtremeTests.cpp b/source/tests/ExtremeTests.cpp similarity index 100% rename from Source/tests/ExtremeTests.cpp rename to source/tests/ExtremeTests.cpp diff --git a/Source/tests/LocaleTest.cpp b/source/tests/LocaleTest.cpp similarity index 100% rename from Source/tests/LocaleTest.cpp rename to source/tests/LocaleTest.cpp diff --git a/Source/tests/ParameterFuzzTests.cpp b/source/tests/ParameterFuzzTests.cpp similarity index 100% rename from Source/tests/ParameterFuzzTests.cpp rename to source/tests/ParameterFuzzTests.cpp diff --git a/Source/vst3validator/VST3ValidatorRunner.cpp b/source/vst3validator/VST3ValidatorRunner.cpp similarity index 100% rename from Source/vst3validator/VST3ValidatorRunner.cpp rename to source/vst3validator/VST3ValidatorRunner.cpp diff --git a/Source/vst3validator/VST3ValidatorRunner.h b/source/vst3validator/VST3ValidatorRunner.h similarity index 100% rename from Source/vst3validator/VST3ValidatorRunner.h rename to source/vst3validator/VST3ValidatorRunner.h diff --git a/tests/acceptance/Acceptance testing design.md b/tests/acceptance/Acceptance testing design.md new file mode 100644 index 00000000..0103cc73 --- /dev/null +++ b/tests/acceptance/Acceptance testing design.md @@ -0,0 +1,398 @@ +# Acceptance testing (design) + +> Status: **Phase 1 implemented.** The `pluginval test ` mode +> described here is built (`source/acceptance/`, wired into the CLI dispatcher) +> with the `sample` comparator, record-or-compare, float-WAV + JSON-sidecar +> references, a dogfood tone-generator plugin (`tests/test_plugins/tone_generator/`, +> behind `PLUGINVAL_BUILD_TEST_PLUGINS`) and CTest self-tests +> (`tests/acceptance/`). The Phase 2 items in §10 remain stubs/notes. This +> document is the reference spec for both the implementation and future extension. + +## 1. Overview + +The existing `validate` mode is a graded **unit-test suite** (`PluginTests : +juce::UnitTest`, self-registering `PluginTest` instances, pass/fail via +`expect`). It answers "does this plugin conform to the host API and behave +safely?" + +Acceptance testing answers a different question: **"does this plugin produce the +output I expect for a known input and state?"** It is a deterministic +*render + golden-file comparison*: + +1. Load a plugin, apply a known state / parameter set. +2. Feed a known input (audio and/or MIDI, or silence). +3. Render a fixed duration of audio. +4. If no reference exists, **record** one. If a reference exists, **compare** + the freshly rendered output against it and emit a pass/fail verdict. + +Because it is a fundamentally different activity from `validate`, it is built as +a **parallel subsystem** rather than as another `PluginTest`. It reuses the +plugin-loading, lifecycle, and render infrastructure but has its own CLI +command, config schema, and result model. + +### Scope and honest caveats + +- Acceptance testing is only meaningful for plugins that are **deterministic** + given a fixed input + state. A plugin with free-running internal randomness + cannot be golden-tested reliably. The planned playhead / seed controls help, + but cannot make a non-deterministic plugin deterministic. +- The **cross-platform / cross-version portability** goal (matching a reference + produced on a different OS, CPU, or plugin version) is realistic only with + tolerant comparison methods. Exact / per-sample comparison rarely survives + SIMD and platform floating-point differences. This is precisely why the + comparator is a pluggable abstraction (see §5): spectrum, cross-correlation + and fingerprint methods are what make portable references viable. + +## 2. Command-line interface + +A new subcommand sits alongside the existing verbs: + +``` +pluginval test +``` + +- `` is the **positional** acceptance-test definition (see §4). It + is parsed by its own loader. +- If the config's reference file does not exist, it is **created** (record + mode) and the command reports success. +- If the reference exists, the rendered output is **compared** against it and + the command exits `0` (match) or `1` (mismatch), consistent with `validate`. + +### Relationship to `--config` (important) + +Do **not** confuse the acceptance-test config with the existing `--config` +flag. They are unrelated: + +| | `--config file.json` | `pluginval test file.json` | +|---|---|---| +| Purpose | A *settings layer* for a **validate** run | The full **acceptance-test definition** | +| Schema | `PluginvalSettings` (strictness, timeouts, sample-rate lists, …) | Plugin + input + reference + state + comparison | +| Merge behaviour | `merge_patch`ed into the settings layering | Loaded standalone | +| How supplied | `--config` option | Positional argument to the `test` verb | + +The acceptance-test definition is **never** routed through the +`PluginvalSettings` / `--config` layering. + +## 3. Integration with the CLI pipeline + +The CLI was rewritten (PRs #175 and #176) onto **CLI11** + an **nlohmann/json** +settings pipeline with a subcommand dispatcher. Acceptance testing plugs into +that dispatcher; it does **not** touch the validate settings-layering at all. + +Required edits (small and localised): + +| Location | Edit | +|---|---| +| `SettingsParser.h` — `enum class Command` | add `test` | +| `SettingsParser.cpp` — `dispatch()` | recognise the verb `test`, peel it, capture the positional config path(s) | +| `SettingsParser.cpp` — `isCommandLine()` | recognise `test` so CLI mode is triggered | +| `SettingsParser.cpp` — `getFooterText()` | add `test` to the `Commands:` help block | +| `CommandLine.cpp` — `performCommandLine()` | add a `Command::test` branch that runs the acceptance runner and async-quits | + +Because `test` is brand new there are **no deprecated-alias** concerns. + +### Process isolation + +For v1, acceptance tests run **in-process**: record and compare both need the +rendered buffer back in the calling process, so in-process is the natural +choice. Crash isolation can be layered on later by mirroring the validate +child-process handoff (`createChildProcessCommandLine` in `SettingsParser.cpp`, +which passes an authoritative base64-JSON blob via `--config-base64`); the +equivalent would be `test --config-base64 ` with the child writing the +reference / diff to disk and returning an exit code. + +## 4. Config schema + +The config is a plain, std-typed struct that follows the same **nlohmann/json +(de)serialisation pattern** as `PluginvalSettings.h`, plus a `toX()` boundary +conversion to JUCE types. **JSON keys are `snake_case`** (e.g. `sample_rate`). +Because the C++ members stay JUCE-style `camelCase`, the snake_case keys are +mapped explicitly via per-struct `to_json` / `from_json` (rather than the bare +`NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT` macro, which would emit the +member names verbatim). Missing keys still fall back to the defaults. + +```jsonc +{ + "name": "myReverb-default", + "plugin": "/path/to/Plugin.vst3", // path or AU identifier string + "input": { "audio": "in.wav", "midi": "in.mid" }, // either / both / omitted -> silence + "reference": "refs/myReverb-default.wav", // optional; default derived from "name" + "state": { "file": "preset.state" }, // OR "parameters": { "Mix": 0.5, "3": 1.0 } + "sample_rate": 48000, + "block_size": 512, + "render_duration": 2.0, // seconds; omitted -> use input length + "comparison": { "sample": 1e-6 }, // omitted -> default 1/32768 (16-bit LSB) + "playhead": { "bpm": 120, "time_signature": { "numerator": 4, "denominator": 4 } }, + "automation": [ /* phase 2 */ ] +} +``` + +### Field reference + +| Field | Type | Default | Notes | +|---|---|---|---| +| `name` | string | basename of config file | Used to derive the default reference path and to label results. | +| `plugin` | string | — (required) | Absolute/relative path, or an AU identifier string. Relative paths resolve against the working directory. | +| `input.audio` | string | none | Path to an input audio file (WAV/AIFF/FLAC via `AudioFormatManager`). Used for effects. | +| `input.midi` | string | none | Path to a `.mid` file (`juce::MidiFile`). Used for instruments. | +| `reference` | string | `/.wav` | The golden file. Absent -> record mode; present -> compare mode. | +| `state.file` | string | none | Binary blob from `getStateInformation`, applied via the VST3-safe helper. | +| `state.parameters` | object | none | `name-or-index -> normalised value`. Applied after `state.file` if both are given. | +| `sample_rate` | number | 44100 | Single value (not a list, unlike validate). | +| `block_size` | number | 512 | Single value. | +| `render_duration` | number | input length | Seconds. `num_samples = round(duration * sample_rate)`. Input shorter than the duration is padded with silence; longer is truncated. | +| `comparison` | object | `{ "sample": 0.0000305 }` | Map of comparator name -> its sub-config. See §5. | +| `playhead` | object | none | Fixed transport for time-dependent plugins. `{ "bpm": , "time_signature": { "numerator": N, "denominator": D }, "start_ppq": }`. `time_signature` defaults to 4/4, `start_ppq` to 0. Omitted -> no playhead is set (the plugin sees `getPlayHead() == nullptr`). The tempo / time signature are constant; the position advances with the render. | +| `automation` | array | none | **Phase 2.** Parameter changes scheduled at sample positions. | + +### State precedence + +If both `state.file` and `state.parameters` are present, the binary state is +applied **first** (it represents the full plugin state), then the parameter map +is applied on top as overrides. + +### Multiplexing (future) + +The loader accepts **either a single object or a top-level array** of configs. +v1 may execute only the single / first entry; phase 2 iterates the array to +multiplex many test cases from one file. Designing the parser for arrays now +costs nothing and avoids a schema break later. Each array entry should carry a +`name` so its reference path and result are uniquely identifiable. + +## 5. Comparator abstraction (the core extension point) + +Comparison is **pluggable**. The config selects one or more methods; v1 ships +only `sample`, and additional methods register later with **zero** changes to +config parsing or the runner. + +```cpp +struct ComparisonResult +{ + bool passed = false; + double score = 0.0; // method-specific metric (e.g. max abs diff) + juce::String summary; // human-readable + nlohmann::json details; // structured, for the machine-readable report +}; + +struct Comparator +{ + virtual ~Comparator() = default; + virtual juce::String getName() const = 0; // "sample", "spectrum", ... + virtual ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::json& config) = 0; // value under comparison[name] +}; +``` + +- A small **registry** maps method name -> factory. +- `config` is whatever sits under the comparator's key — a bare number + (`1e-6`) for `sample`, or an object for richer methods + (e.g. `"spectrum": { "tolerance": 0.05, "fftSize": 2048 }`). +- When multiple methods are listed, **all must pass** (logical AND); each is + reported separately. + +### `comparison` defaults + +When `comparison` is omitted, the default is: + +```json +"comparison": { "sample": 0.0000305 } // 1.0 / 32768.0, i.e. one 16-bit LSB +``` + +### Roadmap of comparators + +| Method | Phase | Description | +|---|---|---| +| `sample` | **v1** | Per-sample absolute-difference tolerance (`0` = bit-exact) plus a length check. | +| `peakrms` | future | Peak and RMS difference thresholds. | +| `spectrum` | future | FFT-magnitude comparison with tolerance; robust to small phase/SIMD differences. | +| `crosscorr` | future | Cross-correlation, tolerant to small latency offsets. | +| `fingerprint` | future | Acoustic fingerprint match; most robust for cross-platform/version. | + +The `spectrum` / `crosscorr` / `fingerprint` methods are what make the +**cross-platform-portable** reference goal practical. + +## 6. Reference artifact format + +A reference is **two files**: + +1. **`.wav`** — the rendered output as a **32-bit float WAV** (preserves + full precision; it is the source of truth for comparison). +2. **`.wav.json`** — a sidecar manifest: + +```jsonc +{ + "plugin": { "name": "...", "manufacturer": "...", "version": "...", + "format": "VST3", "uid": "..." }, + "render": { "sample_rate": 48000, "block_size": 512, "num_channels": 2, + "num_samples": 96000, "length_seconds": 2.0 }, + "pluginval_version": "2.0.0", + "config_hash": "…", // hash of the resolved config (excluding the reference path) + "created_on": { "os": "macOS", "arch": "arm64", "date": "2026-…" } // informational only +} +``` + +- `created_on` is **informational only** — it is never used for matching, because + references are meant to be portable and shared. +- `config_hash` lets the runner detect a **stale** reference (one produced from a + different config than the one now being run) and warn. +- The default reference path deliberately contains **no platform/arch**, since + references are intended to be portable and checked into a repo. + +### Diff output on failure + +When a comparison fails, the runner also writes a **diff WAV** (`output − reference`) +next to the result to aid debugging. + +## 7. Module layout + +``` +source/acceptance/ + TestConfig.h/.cpp // std-typed struct + NLOHMANN_DEFINE_TYPE..._WITH_DEFAULT + // + toRenderSpec() boundary conversion (mirrors PluginvalSettings.h) + AcceptanceTest.h/.cpp // resolve + load plugin, apply state, feed input, render -> buffer + ReferenceComparator.h/.cpp // Comparator interface + registry + SampleComparator; record-or-compare + TestReporter.h/.cpp // text + JSON result, exit code +``` + +Reused infrastructure: + +- Plugin loading / scanning: `AudioPluginFormatManager::createPluginInstance` + and `KnownPluginList` (as in `PluginTests.cpp`). +- VST3-safe lifecycle helpers in `TestUtilities.h`: + `callPrepareToPlayOnMessageThreadIfVST3`, + `callSetStateInformationOnMessageThreadIfVST3`, etc. (VST3 requires several + operations on the message thread). +- The render-loop shape from the `AudioProcessingTest` in + `source/tests/BasicTests.cpp` (`prepareToPlay` -> `processBlock` loop with + `AudioBuffer` + `MidiBuffer`). + +Build wiring: add the new `.cpp` files to the `SourceFiles` list in +`CMakeLists.txt`, and add acceptance-mode cases to `source/CommandLineTests.cpp` +following the existing test style. + +## 8. Dogfood test plugin (tone generator) + +To develop and self-test the acceptance feature we need a **device under test +that is fully deterministic** — something whose output is known exactly, doesn't +depend on a third-party binary, and is stable across runs and platforms. We +build a tiny in-repo **tone-generator plugin** for this purpose ("dogfooding" +the feature with our own plugin). + +### What it is + +A minimal JUCE plugin (built with `juce_add_plugin`) that synthesises a simple, +deterministic tone: + +- **Parameters** (so we can exercise state / parameter application): + - `waveform` — choice: `sine`, `square` (extendable to `saw`, `triangle`). + - `frequency` — Hz (e.g. 20–20000, default 440). + - `gain` — linear or dB (default −6 dB). +- **Determinism**: the oscillator **phase resets to 0 on `prepareToPlay`** and + the waveform is computed in closed form from the sample index, so the same + config always renders the identical buffer (ideal for the `sample` + comparator, including bit-exact). No randomness, no denormal-sensitive + feedback paths. +- **No audio input required**: it is a generator, so acceptance configs for it + use silence input (or omit `input`) and a fixed `render_duration`. +- Produces a VST3 on all platforms (and an AU on macOS) so the same plugin + exercises both formats. + +### Where it lives + +A new CMake target alongside the existing `tests/test_plugins/`, e.g. +`pluginval_tone_generator`, built on demand (guarded behind an option such as +`PLUGINVAL_BUILD_TEST_PLUGINS`, off for normal release builds). Suggested +layout: + +``` +tests/test_plugins/tone_generator/ + CMakeLists.txt // juce_add_plugin target + ToneGeneratorPlugin.h/.cpp +``` + +### How it self-tests the acceptance feature + +Check in a handful of acceptance configs plus their recorded reference WAVs and +wire them into CTest, so CI both validates the tone generator's stability and +exercises the full record/compare path: + +``` +tests/acceptance/ + sine-440.json.in // tone gen, waveform=sine, state.parameters + square-220.json.in // tone gen, waveform=square, state.parameters, bit-exact + square-state.json.in // tone gen, state.file blob + a gain parameter override + gain-half.json.in // gain effect, input.audio = a full-height sine, bit-exact + playhead-120.json.in // playhead probe, fixed transport (bpm 120, 4/4), bit-exact + inputs/sine-full.wav // checked-in input for gain-half (±1.0 sine) + refs/.wav (+ .wav.json) // checked-in references + sidecar manifests + refs/square-state.state // checked-in getStateInformation blob +``` + +The configs are checked-in **templates** (`*.json.in`): the plugin artefact +paths and the input/reference directories are substituted at CMake configure +time (the tests can't hardcode a build-tree plugin path). Each `pluginval test +` CTest case asserts exit code `0` against the checked-in reference. +Because the dogfood plugins are closed-form deterministic, these references are +stable enough to commit — a first smoke test of cross-platform portability for +the `sample` comparator and the baseline for future comparators (`spectrum`, +`crosscorr`, …). + +The five cases cover the distinct render paths: + +- **`sine-440` / `square-220`** — the `state.parameters` (name/index → normalised + value) path. `square-220` compares bit-exact (`"sample": 0`). +- **`square-state`** — the binary `state.file` (`setStateInformation`) path, + plus a `state.parameters` `gain` override on top, exercising the + state-then-parameters precedence of §4. The blob is captured once and checked + in alongside its reference WAV. +- **`gain-half`** — the **`input.audio`** effect path: a second dogfood plugin + (`tests/test_plugins/gain/`, a deterministic gain effect) gains a checked-in + full-height (±1.0) sine by 0.5 and is compared bit-exact (0.5 is exact in + float). It also omits `render_duration`, so the render length is derived from + the input file. +- **`playhead-120`** — the **`playhead`** path: a third dogfood plugin + (`tests/test_plugins/playhead_probe/`) writes the host transport into its + output (channel 0 = `ppqPosition`, channel 1 = tempo), so the recorded + reference is a direct check that the fixed transport reached the plugin. If the + playhead regressed, the probe would output silence and the compare would fail. + +## 9. Execution flow + +1. Parse `pluginval test ` into one or more `TestConfig`. +2. Resolve the plugin (path or ID); create the instance at the config's + `sample_rate` / `block_size`. +3. Apply `state.file` then `state.parameters`. +4. Load `input.audio` and/or `input.midi`; otherwise use silence. +5. If a `playhead` is configured, point a fixed-tempo transport at the plugin + (its position advances each block). +6. `prepareToPlay`, render `render_duration` worth of blocks, accumulating the + output into a single buffer. +7. **No reference exists** -> write the float WAV + manifest (record mode); + report "reference created". +8. **Reference exists** -> run each configured comparator; report each verdict + and the overall pass/fail; on failure write a diff WAV; exit `0` / `1`. + +## 10. Phasing + +**Phase 1 (initial implementation)** + +- `pluginval test ` subcommand wired into `settings_parser`. +- Single-config (object) execution; parser already accepts arrays. +- Plugin load + state (file and/or parameter map) + file-based audio/MIDI input + (or silence). +- Fixed-duration render in-process. +- Record-or-compare with float-WAV + JSON-sidecar references. +- `sample` comparator only (default tolerance = one 16-bit LSB). +- Text + JSON result reporting; diff WAV on failure. +- Fixed `playhead` (tempo / time signature) for time-dependent plugins. +- **Dogfood test plugins** (§8) plus CTest self-tests that run the full + record/compare path against checked-in references. + +**Phase 2 and beyond** + +- Multiplexed execution of config arrays. +- Parameter `automation` timelines. +- Additional comparators: `peakrms`, `spectrum`, `crosscorr`, `fingerprint`. +- Synthesised inputs (`input.generator`: noise / sine, with seed). +- Optional child-process isolation mirroring the validate handoff. diff --git a/tests/acceptance/CMakeLists.txt b/tests/acceptance/CMakeLists.txt new file mode 100644 index 00000000..5d1f350c --- /dev/null +++ b/tests/acceptance/CMakeLists.txt @@ -0,0 +1,39 @@ +# Acceptance self-tests: drive the dogfood plugins through the full record/compare +# path against checked-in reference WAVs. +# +# The configs are checked-in templates (*.json.in). At configure time the dogfood +# plugin artefact paths, the input directory and the (source-tree) reference +# directory are substituted in. An artefact path may contain a generator +# expression (e.g. $ on multi-config generators), so we resolve it via +# file(GENERATE). + +get_target_property(tone_artefact pluginval_tone_generator_VST3 JUCE_PLUGIN_ARTEFACT_FILE) +get_target_property(gain_artefact pluginval_gain_VST3 JUCE_PLUGIN_ARTEFACT_FILE) +get_target_property(probe_artefact pluginval_playhead_probe_VST3 JUCE_PLUGIN_ARTEFACT_FILE) + +set(ACCEPTANCE_REF_DIR "${CMAKE_CURRENT_SOURCE_DIR}/refs") +set(ACCEPTANCE_INPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/inputs") +set(TONE_GENERATOR_PLUGIN "${tone_artefact}") +set(GAIN_PLUGIN "${gain_artefact}") +set(PLAYHEAD_PROBE_PLUGIN "${probe_artefact}") + +set(acceptance_cases + sine-440 # state.parameters path (waveform/frequency/gain by name) + square-220 # state.parameters path, bit-exact square wave + square-state # state.file (setStateInformation) + parameter override precedence + gain-half # input.audio effect path: gain a full-height sine, length from input + playhead-120) # fixed-playhead path: probe writes the host transport into the output + +foreach(case ${acceptance_cases}) + # Step 1: @-substitute the reference dir (and the possibly-genex plugin path). + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${case}.json.in" + "${CMAKE_CURRENT_BINARY_DIR}/${case}.json.gen" @ONLY) + + # Step 2: resolve any generator expression left in the plugin path. The output + # path is per-config so multi-config generators don't collide on one file. + file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/${case}.json.gen") + + add_test(NAME "pluginval.acceptance.${case}" + COMMAND pluginval test "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json") +endforeach() diff --git a/tests/acceptance/gain-half.json.in b/tests/acceptance/gain-half.json.in new file mode 100644 index 00000000..1fa0970f --- /dev/null +++ b/tests/acceptance/gain-half.json.in @@ -0,0 +1,16 @@ +{ + "name": "gain-half", + "plugin": "@GAIN_PLUGIN@", + "input": { "audio": "@ACCEPTANCE_INPUT_DIR@/sine-full.wav" }, + "reference": "@ACCEPTANCE_REF_DIR@/gain-half.wav", + "state": { + "parameters": { + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/inputs/sine-full.wav b/tests/acceptance/inputs/sine-full.wav new file mode 100644 index 00000000..6cf57322 Binary files /dev/null and b/tests/acceptance/inputs/sine-full.wav differ diff --git a/tests/acceptance/playhead-120.json.in b/tests/acceptance/playhead-120.json.in new file mode 100644 index 00000000..d8aacd6f --- /dev/null +++ b/tests/acceptance/playhead-120.json.in @@ -0,0 +1,15 @@ +{ + "name": "playhead-120", + "plugin": "@PLAYHEAD_PROBE_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/playhead-120.wav", + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "playhead": { + "bpm": 120, + "time_signature": { "numerator": 4, "denominator": 4 } + }, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/refs/gain-half.wav b/tests/acceptance/refs/gain-half.wav new file mode 100644 index 00000000..8d419d31 Binary files /dev/null and b/tests/acceptance/refs/gain-half.wav differ diff --git a/tests/acceptance/refs/gain-half.wav.json b/tests/acceptance/refs/gain-half.wav.json new file mode 100644 index 00000000..303838c4 --- /dev/null +++ b/tests/acceptance/refs/gain-half.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "e1622f157cf37917", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T14:22:49.718+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Gain", + "uid": "VST3-pluginval Gain-327cfb94-d8aa82c7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/playhead-120.wav b/tests/acceptance/refs/playhead-120.wav new file mode 100644 index 00000000..a98a8764 Binary files /dev/null and b/tests/acceptance/refs/playhead-120.wav differ diff --git a/tests/acceptance/refs/playhead-120.wav.json b/tests/acceptance/refs/playhead-120.wav.json new file mode 100644 index 00000000..680f2bce --- /dev/null +++ b/tests/acceptance/refs/playhead-120.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "6c38c7c43e1f8520", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T15:51:01.553+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Playhead Probe", + "uid": "VST3-pluginval Playhead Probe-f5ce8b6f-d8b37cc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/sine-440.wav b/tests/acceptance/refs/sine-440.wav new file mode 100644 index 00000000..8d419d31 Binary files /dev/null and b/tests/acceptance/refs/sine-440.wav differ diff --git a/tests/acceptance/refs/sine-440.wav.json b/tests/acceptance/refs/sine-440.wav.json new file mode 100644 index 00000000..cf674ea2 --- /dev/null +++ b/tests/acceptance/refs/sine-440.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "815cbbe305c6d5b0", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.117+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/square-220.wav b/tests/acceptance/refs/square-220.wav new file mode 100644 index 00000000..4032e1df Binary files /dev/null and b/tests/acceptance/refs/square-220.wav differ diff --git a/tests/acceptance/refs/square-220.wav.json b/tests/acceptance/refs/square-220.wav.json new file mode 100644 index 00000000..9168e12d --- /dev/null +++ b/tests/acceptance/refs/square-220.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "a1c34639b14ec668", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.377+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/square-state.state b/tests/acceptance/refs/square-state.state new file mode 100644 index 00000000..5a668c94 Binary files /dev/null and b/tests/acceptance/refs/square-state.state differ diff --git a/tests/acceptance/refs/square-state.wav b/tests/acceptance/refs/square-state.wav new file mode 100644 index 00000000..9dcab4c2 Binary files /dev/null and b/tests/acceptance/refs/square-state.wav differ diff --git a/tests/acceptance/refs/square-state.wav.json b/tests/acceptance/refs/square-state.wav.json new file mode 100644 index 00000000..787b1083 --- /dev/null +++ b/tests/acceptance/refs/square-state.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "bb5f4e275caed1db", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.606+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/sine-440.json.in b/tests/acceptance/sine-440.json.in new file mode 100644 index 00000000..35650ef6 --- /dev/null +++ b/tests/acceptance/sine-440.json.in @@ -0,0 +1,15 @@ +{ + "name": "sine-440", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/sine-440.wav", + "state": { + "parameters": { + "waveform": 0.0, + "frequency": 0.021021021021021, + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25 +} diff --git a/tests/acceptance/square-220.json.in b/tests/acceptance/square-220.json.in new file mode 100644 index 00000000..d0706873 --- /dev/null +++ b/tests/acceptance/square-220.json.in @@ -0,0 +1,18 @@ +{ + "name": "square-220", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/square-220.wav", + "state": { + "parameters": { + "waveform": 1.0, + "frequency": 0.010010010010010, + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/square-state.json.in b/tests/acceptance/square-state.json.in new file mode 100644 index 00000000..c30efebd --- /dev/null +++ b/tests/acceptance/square-state.json.in @@ -0,0 +1,17 @@ +{ + "name": "square-state", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/square-state.wav", + "state": { + "file": "@ACCEPTANCE_REF_DIR@/square-state.state", + "parameters": { + "gain": 0.25 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/test_plugins/gain/CMakeLists.txt b/tests/test_plugins/gain/CMakeLists.txt new file mode 100644 index 00000000..ec2f43d3 --- /dev/null +++ b/tests/test_plugins/gain/CMakeLists.txt @@ -0,0 +1,30 @@ +# A minimal, fully deterministic gain effect used to dogfood the acceptance +# feature's audio-input path. Built only when PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_gain + PRODUCT_NAME "pluginval Gain" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalGain + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Pgn1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_gain PRIVATE + GainPlugin.cpp + GainPlugin.h) + +target_compile_features(pluginval_gain PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_gain PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_gain PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/gain/GainPlugin.cpp b/tests/test_plugins/gain/GainPlugin.cpp new file mode 100644 index 00000000..fb3af1de --- /dev/null +++ b/tests/test_plugins/gain/GainPlugin.cpp @@ -0,0 +1,56 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "GainPlugin.h" + +//============================================================================== +GainProcessor::GainProcessor() + : juce::AudioProcessor (BusesProperties().withInput ("Input", juce::AudioChannelSet::stereo(), true) + .withOutput ("Output", juce::AudioChannelSet::stereo(), true)) +{ + // Default gain of 1.0 passes the input through unchanged. + addParameter (gain = new juce::AudioParameterFloat ("gain", "Gain", + juce::NormalisableRange (0.0f, 1.0f), 1.0f)); +} + +//============================================================================== +void GainProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + const float g = gain->get(); + + for (int c = 0; c < buffer.getNumChannels(); ++c) + buffer.applyGain (c, 0, buffer.getNumSamples(), g); +} + +//============================================================================== +void GainProcessor::getStateInformation (juce::MemoryBlock& destData) +{ + // getValue() is a private override on the concrete type, so go via the base. + juce::MemoryOutputStream mos (destData, false); + mos.writeFloat (static_cast (gain)->getValue()); +} + +void GainProcessor::setStateInformation (const void* data, int sizeInBytes) +{ + juce::MemoryInputStream mis (data, (size_t) sizeInBytes, false); + + if (mis.getNumBytesRemaining() >= (int) sizeof (float)) + gain->setValueNotifyingHost (mis.readFloat()); +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new GainProcessor(); +} diff --git a/tests/test_plugins/gain/GainPlugin.h b/tests/test_plugins/gain/GainPlugin.h new file mode 100644 index 00000000..27446fac --- /dev/null +++ b/tests/test_plugins/gain/GainPlugin.h @@ -0,0 +1,68 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, fully deterministic gain effect used to dogfood the acceptance + feature's audio-input path. + + Unlike the tone generator (a pure generator), this is an effect: it + multiplies its input by a single linear gain parameter. Feeding it a known + input file and comparing the gained output exercises the input.audio render + path. Multiplying by an exact-in-float gain (e.g. 0.5) keeps it bit-exact. +*/ +class GainProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + GainProcessor(); + ~GainProcessor() override = default; + + //============================================================================== + void prepareToPlay (double, int) override {} + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Gain"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override; + void setStateInformation (const void*, int) override; + +private: + //============================================================================== + juce::AudioParameterFloat* gain = nullptr; + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (GainProcessor) +}; diff --git a/tests/test_plugins/playhead_probe/CMakeLists.txt b/tests/test_plugins/playhead_probe/CMakeLists.txt new file mode 100644 index 00000000..c8c3660d --- /dev/null +++ b/tests/test_plugins/playhead_probe/CMakeLists.txt @@ -0,0 +1,33 @@ +# A minimal, deterministic plugin that writes the host transport into its output, +# used to dogfood the acceptance feature's fixed-playhead support. Built only when +# PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_playhead_probe + PRODUCT_NAME "pluginval Playhead Probe" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalPlayheadProbe + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Pph1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + VST3_CATEGORIES Generator + AU_MAIN_TYPE kAudioUnitType_Generator + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_playhead_probe PRIVATE + PlayheadProbePlugin.cpp + PlayheadProbePlugin.h) + +target_compile_features(pluginval_playhead_probe PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_playhead_probe PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_playhead_probe PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp new file mode 100644 index 00000000..b94eb2b9 --- /dev/null +++ b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp @@ -0,0 +1,55 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "PlayheadProbePlugin.h" + +//============================================================================== +void PlayheadProbeProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + buffer.clear(); + + auto* playHead = getPlayHead(); + + if (playHead == nullptr) + return; + + const auto position = playHead->getPosition(); + + if (! position.hasValue()) + return; + + double ppq = 0.0, bpm = 0.0; + + if (const auto v = position->getPpqPosition()) + ppq = *v; + + if (const auto v = position->getBpm()) + bpm = *v; + + const int numSamples = buffer.getNumSamples(); + + // Channel 0: the block's ppqPosition (a per-block staircase that advances with + // the transport). Channel 1: the tempo, scaled into a sane range. + if (buffer.getNumChannels() > 0) + juce::FloatVectorOperations::fill (buffer.getWritePointer (0), (float) ppq, numSamples); + + if (buffer.getNumChannels() > 1) + juce::FloatVectorOperations::fill (buffer.getWritePointer (1), (float) (bpm / 1000.0), numSamples); +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new PlayheadProbeProcessor(); +} diff --git a/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h new file mode 100644 index 00000000..d3f03781 --- /dev/null +++ b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h @@ -0,0 +1,69 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, deterministic plugin that writes the host transport into its + output so the acceptance feature's fixed-playhead support can be verified. + + Each block it reads getPlayHead()->getPosition() and writes the block's + ppqPosition into channel 0 and the tempo (scaled) into channel 1. With no + playhead (or no position) it outputs silence - so a recorded reference is a + direct check that the transport actually reached the plugin. +*/ +class PlayheadProbeProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + PlayheadProbeProcessor() + : juce::AudioProcessor (BusesProperties().withOutput ("Output", juce::AudioChannelSet::stereo(), true)) + { + } + + ~PlayheadProbeProcessor() override = default; + + //============================================================================== + void prepareToPlay (double, int) override {} + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Playhead Probe"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override {} + void setStateInformation (const void*, int) override {} + +private: + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (PlayheadProbeProcessor) +}; diff --git a/tests/test_plugins/tone_generator/CMakeLists.txt b/tests/test_plugins/tone_generator/CMakeLists.txt new file mode 100644 index 00000000..16a1e7c8 --- /dev/null +++ b/tests/test_plugins/tone_generator/CMakeLists.txt @@ -0,0 +1,32 @@ +# A minimal, fully deterministic tone-generator plugin used to dogfood the +# acceptance-testing feature. Built only when PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_tone_generator + PRODUCT_NAME "pluginval Tone Generator" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalToneGenerator + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Ptg1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + VST3_CATEGORIES Generator + AU_MAIN_TYPE kAudioUnitType_Generator + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_tone_generator PRIVATE + ToneGeneratorPlugin.cpp + ToneGeneratorPlugin.h) + +target_compile_features(pluginval_tone_generator PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_tone_generator PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_tone_generator PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp new file mode 100644 index 00000000..ecc0565e --- /dev/null +++ b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp @@ -0,0 +1,114 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "ToneGeneratorPlugin.h" + +namespace +{ + constexpr double twoPi = 2.0 * 3.14159265358979323846; +} + +//============================================================================== +ToneGeneratorProcessor::ToneGeneratorProcessor() + : juce::AudioProcessor (BusesProperties().withOutput ("Output", juce::AudioChannelSet::stereo(), true)) +{ + // Parameter indices are stable (0, 1, 2) so acceptance configs can address + // them by index as well as by name. + addParameter (waveform = new juce::AudioParameterChoice ("waveform", "Waveform", + juce::StringArray { "sine", "square" }, 0)); + addParameter (frequency = new juce::AudioParameterFloat ("frequency", "Frequency", + juce::NormalisableRange (20.0f, 20000.0f), 440.0f)); + addParameter (gain = new juce::AudioParameterFloat ("gain", "Gain", + juce::NormalisableRange (0.0f, 1.0f), 0.5f)); +} + +//============================================================================== +void ToneGeneratorProcessor::prepareToPlay (double sampleRate, int) +{ + currentSampleRate = sampleRate; + sampleIndex = 0; // phase resets so renders are reproducible +} + +void ToneGeneratorProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + buffer.clear(); + + const auto numSamples = buffer.getNumSamples(); + const auto numChannels = buffer.getNumChannels(); + const bool isSquare = waveform->getIndex() == (int) Waveform::square; + const double freq = (double) frequency->get(); + const float g = gain->get(); + + if (numChannels > 0) + { + auto* channel0 = buffer.getWritePointer (0); + + for (int s = 0; s < numSamples; ++s) + { + const double phase = twoPi * freq * (double) (sampleIndex + s) / currentSampleRate; + + float value; + if (isSquare) + { + // Closed-form square: +1 for the first half of each cycle, -1 for the second. + const double fractional = phase / twoPi - std::floor (phase / twoPi); + value = fractional < 0.5 ? 1.0f : -1.0f; + } + else + { + value = (float) std::sin (phase); + } + + channel0[s] = value * g; + } + + // The tone is mono; copy it to every other output channel. + for (int c = 1; c < numChannels; ++c) + buffer.copyFrom (c, 0, buffer, 0, 0, numSamples); + } + + sampleIndex += numSamples; +} + +//============================================================================== +void ToneGeneratorProcessor::getStateInformation (juce::MemoryBlock& destData) +{ + // A tiny, explicit binary state: the three parameters' normalised values. + // This is the blob that a state.file acceptance test restores. getValue() is + // a private override on the concrete parameter types, so go via the base. + const auto normalised = [] (juce::AudioProcessorParameter* p) { return p->getValue(); }; + + juce::MemoryOutputStream mos (destData, false); + mos.writeFloat (normalised (waveform)); + mos.writeFloat (normalised (frequency)); + mos.writeFloat (normalised (gain)); +} + +void ToneGeneratorProcessor::setStateInformation (const void* data, int sizeInBytes) +{ + juce::MemoryInputStream mis (data, (size_t) sizeInBytes, false); + + if (mis.getNumBytesRemaining() >= (int) (3 * sizeof (float))) + { + waveform->setValueNotifyingHost (mis.readFloat()); + frequency->setValueNotifyingHost (mis.readFloat()); + gain->setValueNotifyingHost (mis.readFloat()); + } +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new ToneGeneratorProcessor(); +} diff --git a/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h new file mode 100644 index 00000000..04a6033d --- /dev/null +++ b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h @@ -0,0 +1,76 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, fully deterministic tone-generator plugin used to dogfood the + acceptance-testing feature. + + The output is computed in closed form from a sample index that resets to 0 + on prepareToPlay(), so the same configuration always renders the identical + buffer (ideal for the bit-exact / sample comparator). There is no randomness + and no denormal-sensitive feedback path. Audio input and MIDI are ignored - + it is a pure generator. +*/ +class ToneGeneratorProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + enum class Waveform { sine = 0, square = 1 }; + + ToneGeneratorProcessor(); + ~ToneGeneratorProcessor() override = default; + + //============================================================================== + void prepareToPlay (double sampleRate, int maximumExpectedSamplesPerBlock) override; + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Tone Generator"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override; + void setStateInformation (const void*, int) override; + +private: + //============================================================================== + juce::AudioParameterChoice* waveform = nullptr; + juce::AudioParameterFloat* frequency = nullptr; + juce::AudioParameterFloat* gain = nullptr; + + double currentSampleRate = 44100.0; + juce::int64 sampleIndex = 0; // resets to 0 on prepareToPlay for determinism + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ToneGeneratorProcessor) +};