Tracktion · drowaudio · Jun 8, 2026 · Jun 16, 2026 · Jun 16, 2026 · Jun 16, 2026
diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml
@@ -0,0 +1,70 @@
+name: Acceptance tests
+
+# Builds the in-repo dogfood plugins and runs the acceptance (golden-file)
+# self-tests via CTest. This is independent of Tracktion's main private build
+# pipeline; it exists so the `pluginval test` feature has a public, reproducible
+# regression check across the three desktop platforms.
+
+on:
+  push:
+    branches: [ master, develop, v2 ]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: acceptance-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  acceptance:
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+
+    env:
+      # Cache CPM's downloads (JUCE etc.) so reruns don't re-fetch them.
+      CPM_SOURCE_CACHE: ${{ github.workspace }}/.cpm-cache
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache CPM sources
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.cpm-cache
+          key: cpm-${{ matrix.os }}-${{ hashFiles('CMakeLists.txt', 'cmake/CPM.cmake') }}
+          restore-keys: cpm-${{ matrix.os }}-
+
+      - name: Install Linux dependencies
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libasound2-dev libx11-dev libxext-dev libxinerama-dev libxrandr-dev \
+            libxcursor-dev libxcomposite-dev libfreetype6-dev libfontconfig1-dev \
+            libgl1-mesa-dev libcurl4-openssl-dev ladspa-sdk ninja-build xvfb
+
+      # The acceptance tests host the dogfood plugins via JUCE's own VST3 hosting,
+      # so the embedded VST3 validator (and its heavy SDK build) isn't needed here.
+      - name: Configure
+        run: >
+          cmake -B build
+          -DCMAKE_BUILD_TYPE=Release
+          -DPLUGINVAL_BUILD_TEST_PLUGINS=ON
+          -DPLUGINVAL_VST3_VALIDATOR=OFF
+
+      - name: Build
+        run: cmake --build build --config Release
+
+      - name: Run acceptance tests (Linux)
+        if: runner.os == 'Linux'
+        working-directory: build
+        run: xvfb-run --auto-servernum ctest -C Release --output-on-failure -R pluginval.acceptance
+
+      - name: Run acceptance tests (macOS / Windows)
+        if: runner.os != 'Linux'
+        working-directory: build
+        run: ctest -C Release --output-on-failure -R pluginval.acceptance
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -75,7 +75,7 @@ Replace `<run_id>` with the ID from step 2.
 
 ```
 pluginval/
-├── Source/                    # Main application source code
+├── source/                    # Main application source code
 │   ├── Main.cpp              # Application entry point
 │   ├── MainComponent.cpp/h   # GUI main window component
 │   ├── Validator.cpp/h       # Core validation orchestration
@@ -93,6 +93,11 @@ pluginval/
 │   ├── vst3validator/        # Embedded VST3 validator integration
 │   │   ├── VST3ValidatorRunner.h
 │   │   └── VST3ValidatorRunner.cpp
+│   ├── acceptance/           # `pluginval test` golden-file subsystem (parallel to validate)
+│   │   ├── TestConfig.cpp/h        # snake_case JSON config struct (independent of PluginvalSettings)
+│   │   ├── AcceptanceTest.cpp/h    # plugin load + state + input + render; record-or-compare orchestrator
+│   │   ├── ReferenceComparator.cpp/h # Comparator interface + registry + SampleComparator
+│   │   └── TestReporter.cpp/h      # text + JSON result, exit code
 │   └── tests/                # Individual test implementations
 │       ├── BasicTests.cpp    # Core plugin tests (info, state, audio)
 │       ├── BusTests.cpp      # Audio bus configuration tests
@@ -108,6 +113,10 @@ pluginval/
 ├── tests/
 │   ├── AddPluginvalTests.cmake  # CMake module for CTest integration
 │   ├── test_plugins/         # Test plugin files
+│   │   ├── tone_generator/   # Deterministic dogfood generator (PLUGINVAL_BUILD_TEST_PLUGINS)
+│   │   ├── gain/             # Deterministic dogfood gain effect (for the input.audio path)
+│   │   └── playhead_probe/   # Writes the host transport to output (for the playhead path)
+│   ├── acceptance/           # Acceptance self-tests: configs (*.json.in), inputs/, checked-in refs/ WAVs
 │   ├── mac_tests/            # macOS-specific tests
 │   └── windows_tests.bat     # Windows test scripts
 ├── docs/                     # Documentation
@@ -150,6 +159,7 @@ cmake --build Builds/Debug --config Debug
 | `WITH_ADDRESS_SANITIZER` | Enable AddressSanitizer | OFF |
 | `WITH_THREAD_SANITIZER` | Enable ThreadSanitizer | OFF |
 | `VST2_SDK_DIR` | Path to VST2 SDK (env var) | - |
+| `PLUGINVAL_BUILD_TEST_PLUGINS` | Build the in-repo dogfood plugins + acceptance CTest self-tests | OFF |
 
 ### Enabling VST2 Support
 
@@ -258,6 +268,48 @@ settings set via a base64-encoded JSON argument (`--config-base64`), avoiding
 per-flag re-serialisation and command-line quoting hazards. `--help`/`--version`
 are handled by CLI11 (auto usage + a footer with the env-var/commands notes).
 
+### Acceptance Testing (`pluginval test`)
+
+A **parallel subsystem** to validate, in `source/acceptance/`. It answers "does
+this plugin produce the expected output for a known input + state?" — a
+deterministic *render + golden-file comparison*, not a unit-test pass/fail. Full
+spec: `tests/acceptance/Acceptance testing design.md`; end-user guide:
+`docs/Acceptance testing.md`.
+
+- **CLI**: `pluginval test <config.json>`. A new `Command::test` is recognised
+  by `settings_parser::dispatch()` (captures the positional config path into
+  `DispatchResult::testConfigPath`), `isCommandLine()` and `getFooterText()`;
+  `CommandLine.cpp`'s `performCommandLine()` has a `Command::test` branch that
+  runs the acceptance runner **synchronously on the message thread** and quits.
+- **Config**: `acceptance::TestConfig` (`TestConfig.cpp/h`) — std-typed struct,
+  **snake_case JSON keys** mapped via explicit `to_json`/`from_json` (members
+  stay camelCase). It is **independent** of `PluginvalSettings` / the `--config`
+  layering: the test config is a positional argument loaded standalone.
+- **Flow** (`AcceptanceTest.cpp`): load plugin → apply `state.file` then
+  `state.parameters` (normalised, matched by index / case-insensitive name or
+  paramID) → feed `input.audio`/`input.midi` or silence → if a `playhead` is
+  configured, point a fixed-tempo transport (`FixedPlayHead`, position advances
+  per block) at the plugin → render a fixed duration block-by-block (reusing the
+  `AudioProcessingTest` shape + the VST3-safe helpers in `TestUtilities.h`). If
+  no reference exists it **records** one (32-bit float WAV + `<name>.wav.json`
+  sidecar manifest); otherwise it **compares** and writes a diff WAV on failure.
+  Exit `0`/`1`.
+- **Comparators** (`ReferenceComparator.cpp/h`): pluggable `Comparator` +
+  `createComparator(name)` registry. v1 ships only `sample` (per-sample abs-diff
+  tolerance, default one 16-bit LSB = `1/32768`; `0` = bit-exact). Adding
+  `spectrum`/`crosscorr`/etc. is one registry entry, no config/runner changes.
+- **Dogfood + self-tests**: three minimal deterministic `juce_add_plugin` targets
+  behind `PLUGINVAL_BUILD_TEST_PLUGINS` — `tests/test_plugins/tone_generator/`
+  (closed-form sine/square generator, phase resets on `prepareToPlay`),
+  `tests/test_plugins/gain/` (a gain effect, dogfoods the `input.audio` path) and
+  `tests/test_plugins/playhead_probe/` (writes the host transport to its output,
+  dogfoods the `playhead` path). `tests/acceptance/` holds checked-in configs
+  (`*.json.in`, the plugin paths + input dir substituted at configure time),
+  `inputs/` and reference WAVs, run via CTest (`pluginval.acceptance.*`: sine-440,
+  square-220, square-state, gain-half, playhead-120). Phase 2 items (config-array
+  multiplexing, automation, extra comparators, child-process isolation) are notes
+  only.
+
 ### Test Framework
 
 Tests are self-registering. To find all tests, look for static instances:
@@ -298,12 +350,12 @@ The VST3 validator (Steinberg's vstvalidator) is embedded into pluginval when bu
 1. The VST3 SDK is fetched via CPM during CMake configure
 2. The SDK's own `validator` target is built as a separate executable
 3. A CMake script (`cmake/GenerateBinaryHeader.cmake`) converts the compiled binary into a C byte array header
-4. `VST3ValidatorRunner` (`Source/vst3validator/`) extracts the embedded binary to a temp file on first use
+4. `VST3ValidatorRunner` (`source/vst3validator/`) extracts the embedded binary to a temp file on first use
 5. When the `VST3validator` test runs, it spawns the extracted validator as a subprocess
 
 **Key files:**
 - `cmake/GenerateBinaryHeader.cmake` — binary-to-C-header conversion script
-- `Source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File`
+- `source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File`
 
 **Disabling embedded validator:**
 ```bash
@@ -486,7 +538,7 @@ Run internal tests via CLI:
 ## Common Tasks for AI Assistants
 
 ### Finding Where Tests Are Defined
-- All test classes are in `Source/tests/*.cpp`
+- All test classes are in `source/tests/*.cpp`
 - Search for `static.*Test.*Test;` to find registrations
 - Each test subclasses `PluginTest`
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -28,6 +28,10 @@ endif()
 option(WITH_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF)
 option(WITH_THREAD_SANITIZER "Enable Thread Sanitizer" OFF)
 
+# Builds the in-repo dogfood plugins (e.g. the tone generator) used by the
+# acceptance self-tests. Off for normal release builds.
+option(PLUGINVAL_BUILD_TEST_PLUGINS "Build the in-repo test plugins used by the acceptance self-tests" OFF)
+
 message(STATUS "Sanitizers: ASan=${WITH_ADDRESS_SANITIZER} TSan=${WITH_THREAD_SANITIZER}")
 if (WITH_ADDRESS_SANITIZER)
     if (MSVC)
@@ -105,7 +109,7 @@ endif()
 juce_add_gui_app(pluginval
     BUNDLE_ID                       com.Tracktion.pluginval
     COMPANY_NAME                    Tracktion
-    ICON_BIG                        "${CMAKE_CURRENT_SOURCE_DIR}/Source/binarydata/icon.png"
+    ICON_BIG                        "${CMAKE_CURRENT_SOURCE_DIR}/source/binarydata/icon.png"
     HARDENED_RUNTIME_ENABLED        TRUE
     HARDENED_RUNTIME_OPTIONS        com.apple.security.cs.allow-unsigned-executable-memory com.apple.security.cs.disable-library-validation com.apple.security.get-task-allow)
 
@@ -118,42 +122,50 @@ set_target_properties(pluginval PROPERTIES
     CXX_VISIBILITY_PRESET hidden)
 
 set(SourceFiles
-    Source/CommandLine.h
-    Source/CrashHandler.h
-    Source/MainComponent.h
-    Source/PluginTests.h
-    Source/PluginvalSettings.h
-    Source/SettingsParser.h
-    Source/SettingsSerializer.h
-    Source/TestUtilities.h
-    Source/Validator.h
-    Source/CommandLine.cpp
-    Source/CrashHandler.cpp
-    Source/Main.cpp
-    Source/MainComponent.cpp
-    Source/PluginTests.cpp
-    Source/SettingsParser.cpp
-    Source/SettingsSerializer.cpp
-    Source/tests/BasicTests.cpp
-    Source/tests/LocaleTest.cpp
-    Source/tests/BusTests.cpp
-    Source/tests/EditorTests.cpp
-    Source/tests/ExtremeTests.cpp
-    Source/tests/ParameterFuzzTests.cpp
-    Source/TestUtilities.cpp
-    Source/Validator.cpp)
+    source/CommandLine.h
+    source/CrashHandler.h
+    source/MainComponent.h
+    source/PluginTests.h
+    source/PluginvalSettings.h
+    source/SettingsParser.h
+    source/SettingsSerializer.h
+    source/TestUtilities.h
+    source/Validator.h
+    source/acceptance/TestConfig.h
+    source/acceptance/TestConfig.cpp
+    source/acceptance/AcceptanceTest.h
+    source/acceptance/AcceptanceTest.cpp
+    source/acceptance/ReferenceComparator.h
+    source/acceptance/ReferenceComparator.cpp
+    source/acceptance/TestReporter.h
+    source/acceptance/TestReporter.cpp
+    source/CommandLine.cpp
+    source/CrashHandler.cpp
+    source/Main.cpp
+    source/MainComponent.cpp
+    source/PluginTests.cpp
+    source/SettingsParser.cpp
+    source/SettingsSerializer.cpp
+    source/tests/BasicTests.cpp
+    source/tests/LocaleTest.cpp
+    source/tests/BusTests.cpp
+    source/tests/EditorTests.cpp
+    source/tests/ExtremeTests.cpp
+    source/tests/ParameterFuzzTests.cpp
+    source/TestUtilities.cpp
+    source/Validator.cpp)
 
 target_sources(pluginval PRIVATE ${SourceFiles})
-source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles})
+source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/source PREFIX Source FILES ${SourceFiles})
 
 # Add VST3 validator runner sources to pluginval (extracts embedded binary at runtime)
 if(PLUGINVAL_VST3_VALIDATOR)
     set(VST3ValidatorFiles
-        Source/vst3validator/VST3ValidatorRunner.h
-        Source/vst3validator/VST3ValidatorRunner.cpp
+        source/vst3validator/VST3ValidatorRunner.h
+        source/vst3validator/VST3ValidatorRunner.cpp
     )
     target_sources(pluginval PRIVATE ${VST3ValidatorFiles})
-    source_group("Source/vst3validator" FILES ${VST3ValidatorFiles})
+    source_group("source/vst3validator" FILES ${VST3ValidatorFiles})
 endif()
 
 if (DEFINED ENV{VST2_SDK_DIR})
@@ -256,3 +268,12 @@ else()
             DEPENDS pluginval ${PLUGINVAL_TARGET}
             COMMENT "Run pluginval CLI with strict validation")
 endif()
+
+# In-repo dogfood plugins + acceptance self-tests.
+if (PLUGINVAL_BUILD_TEST_PLUGINS)
+    enable_testing()
+    add_subdirectory(tests/test_plugins/tone_generator)
+    add_subdirectory(tests/test_plugins/gain)
+    add_subdirectory(tests/test_plugins/playhead_probe)
+    add_subdirectory(tests/acceptance)
+endif()
diff --git a/README.md b/README.md
@@ -102,6 +102,7 @@ This means you can check the exit code on your various CI and mark builds a fail
  - [Testing plugins with pluginval](<docs/Testing plugins with pluginval.md>)
  - [Debugging a failed validation](<docs/Debugging a failed validation.md>)
  - [Adding pluginval to CI](<docs/Adding pluginval to CI.md>)
+ - [Acceptance testing](<docs/Acceptance testing.md>)
 
 ### Contributing
 If you would like to contribute to the project please do! It's very simple to add tests, simply:

diff --git a/ROADMAP.md b/ROADMAP.md
@@ -5,7 +5,7 @@ There are lots of ways pluginval can be improved, some of these are listed below
 You can help get there by issuing PRs or [funding](FUNDING.md) development.
 
 - [x] Integration of real-time safety checking (via rtcheck)
-- [ ] Improved command-line handling and json config file support
+- [x] Improved command-line handling and json config file support
 - [ ] Acceptance testing using input files and reference output files
 - [ ] Improved stack trace/crash reporting
 - [ ] Automatic integration of Asan/Tsan on platforms that allow it